1199 lines
47 KiB
Python
1199 lines
47 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
"""``/v1/videos*`` endpoint helpers peeled from ``api_server.py``.
|
|
|
|
PUT HERE:
|
|
- Multipart form parsing, upload limits, job run/cleanup, response factories,
|
|
app-state runtime context, and other route-adjacent ``/v1/videos*`` logic
|
|
that used to live in ``api_server.py``.
|
|
- May import ``video_api_utils`` for decode/encode (helpers → utils OK).
|
|
|
|
DO NOT PUT HERE:
|
|
- Shared media encode/decode backends — keep those in ``video_api_utils``.
|
|
|
|
LONGEVITY:
|
|
- This package ``helpers.py`` is the **longer home** for video generation
|
|
endpoint logic through the helper (P0.2) + router (P0.3) stages.
|
|
- The later P1.3 video modality PR (#5227) may further split this file (e.g. form /
|
|
jobs) and move ``video_api_utils`` into ``media.py`` here; until then keep
|
|
growing here, not in root utils.
|
|
|
|
See ``video/generation/README.md`` (utils vs helpers, no overlap).
|
|
"""
|
|
|
|
import asyncio
|
|
import io
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import time
|
|
from collections.abc import Mapping, Sequence
|
|
from contextlib import suppress
|
|
from http import HTTPStatus
|
|
from numbers import Integral
|
|
from pathlib import Path
|
|
from typing import Any, Literal, cast
|
|
|
|
from fastapi import File, Form, HTTPException, Request, UploadFile
|
|
from PIL import Image
|
|
from vllm.entrypoints.launchers.launcher import terminate_if_errored
|
|
from vllm.entrypoints.serve import create_error_response
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
|
|
|
from vllm_omni.diffusion.models.interface import ReferenceVideoDecodeSpec
|
|
from vllm_omni.entrypoints.openai.app_state import Omnivideo
|
|
from vllm_omni.entrypoints.openai.errors import InvalidInputReferenceError
|
|
from vllm_omni.entrypoints.openai.protocol.videos import (
|
|
SecondStr,
|
|
SizeStr,
|
|
VideoAction,
|
|
VideoError,
|
|
VideoGenerationRequest,
|
|
VideoGenerationStatus,
|
|
VideoResponse,
|
|
)
|
|
from vllm_omni.entrypoints.openai.serving_video import (
|
|
LatentEditInput,
|
|
OmniOpenAIServingVideo,
|
|
ReferenceAudio,
|
|
ReferenceImage,
|
|
ReferenceVideo,
|
|
_stage_diffusion_model_class_name,
|
|
)
|
|
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
|
|
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE
|
|
from vllm_omni.entrypoints.openai.utils import get_stage_type
|
|
from vllm_omni.entrypoints.openai.video_api_utils import (
|
|
VideoFrames,
|
|
_decode_image_bytes,
|
|
_validate_image_pixel_limit,
|
|
decode_audio_url,
|
|
decode_input_reference,
|
|
)
|
|
from vllm_omni.errors import OmniClientError
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
|
|
|
|
MINIMAX_H3_MAX_REFERENCE_VIDEO_BYTES = 50 * 1024 * 1024
|
|
|
|
MINIMAX_H3_MAX_REFERENCE_AUDIO_BYTES = 15 * 1024 * 1024
|
|
|
|
MINIMAX_H3_MAX_REFERENCE_COUNT = 12
|
|
|
|
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
|
|
|
|
MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES = frozenset({".mp4", ".mov"})
|
|
|
|
MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES = frozenset({".wav", ".mp3"})
|
|
|
|
CONTROL_REFERENCE_IMAGE_SUFFIXES = frozenset({".bmp", ".gif", ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp"})
|
|
CONTROL_REFERENCE_VIDEO_SUFFIXES = frozenset({".mkv", ".mov", ".mp4", ".webm"})
|
|
CONTROL_REFERENCE_MAX_BYTES = 512 * 1024 * 1024
|
|
|
|
LATENT_EDIT_SOURCE_MAX_BYTES = 512 * 1024 * 1024
|
|
LATENT_EDIT_MASK_FILE_MAX_BYTES = 8 * 1024 * 1024
|
|
|
|
VIDEO_SYNC_TIMEOUT_S = float(os.environ.get("VLLM_OMNI_VIDEO_SYNC_TIMEOUT", 600.0))
|
|
|
|
|
|
def _resolve_video_runtime_context(raw_request: Request) -> tuple[str | None, list[Any] | None]:
|
|
app_model_name = None
|
|
serving_models = getattr(raw_request.app.state, "openai_serving_models", None)
|
|
if serving_models and getattr(serving_models, "base_model_paths", None):
|
|
base_paths = serving_models.base_model_paths
|
|
if base_paths:
|
|
app_model_name = base_paths[0].name
|
|
|
|
app_stage_configs = getattr(raw_request.app.state, "stage_configs", None)
|
|
return app_model_name, app_stage_configs
|
|
|
|
|
|
def _parse_form_json(value: str | None, expected_type: type | None = None) -> Any:
|
|
if value is None or value == "":
|
|
return None
|
|
try:
|
|
parsed = json.loads(value)
|
|
except json.JSONDecodeError as exc:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Invalid JSON in form field.",
|
|
) from exc
|
|
if expected_type is not None and not isinstance(parsed, expected_type):
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"Invalid JSON in form field: expected {expected_type.__name__}, got {type(parsed).__name__}.",
|
|
)
|
|
return parsed
|
|
|
|
|
|
def _parse_latent_edit_mask_json(
|
|
value: str,
|
|
*,
|
|
field_name: str,
|
|
) -> Any:
|
|
"""Decode one request mask; Stage 0 owns its semantic validation."""
|
|
try:
|
|
parsed = _parse_form_json(value)
|
|
# Bound container nesting at the parsing boundary without duplicating
|
|
# Stage 0's numeric, range, or shape validation.
|
|
pending = [(parsed, 0)]
|
|
while pending:
|
|
item, depth = pending.pop()
|
|
if not isinstance(item, (list, dict)):
|
|
continue
|
|
if depth >= 64:
|
|
raise ValueError("JSON nesting exceeds 64 levels")
|
|
children = item.values() if isinstance(item, dict) else item
|
|
pending.extend((child, depth + 1) for child in children)
|
|
return parsed
|
|
except (RecursionError, ValueError) as exc:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} contains invalid JSON or is nested too deeply.",
|
|
) from exc
|
|
|
|
|
|
async def _read_latent_edit_mask_json(
|
|
value: UploadFile | None,
|
|
*,
|
|
field_name: str,
|
|
) -> Any | None:
|
|
"""Read a mask from a bounded UTF-8 JSON file part."""
|
|
if value is None:
|
|
return None
|
|
|
|
try:
|
|
declared_size = value.size
|
|
if isinstance(declared_size, Integral) and int(declared_size) > LATENT_EDIT_MASK_FILE_MAX_BYTES:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} JSON exceeds the {LATENT_EDIT_MASK_FILE_MAX_BYTES // (1024 * 1024)} MiB limit.",
|
|
)
|
|
|
|
payload = await value.read(LATENT_EDIT_MASK_FILE_MAX_BYTES + 1)
|
|
if len(payload) > LATENT_EDIT_MASK_FILE_MAX_BYTES:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} JSON exceeds the {LATENT_EDIT_MASK_FILE_MAX_BYTES // (1024 * 1024)} MiB limit.",
|
|
)
|
|
try:
|
|
text = payload.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} JSON file must be UTF-8 encoded.",
|
|
) from exc
|
|
if not text:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} JSON file must not be empty.",
|
|
)
|
|
parsed = _parse_latent_edit_mask_json(text, field_name=field_name)
|
|
if parsed is None:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} JSON file must contain a mask value.",
|
|
)
|
|
return parsed
|
|
finally:
|
|
await value.close()
|
|
|
|
|
|
def _config_get(config: Any, key: str, default: Any = None) -> Any:
|
|
if isinstance(config, dict):
|
|
return config.get(key, default)
|
|
if hasattr(config, "get"):
|
|
try:
|
|
return config.get(key, default)
|
|
except Exception:
|
|
pass
|
|
return getattr(config, key, default)
|
|
|
|
|
|
def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
|
|
if not stage_configs:
|
|
return []
|
|
|
|
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
|
|
|
model_classes: list[type] = []
|
|
for stage_cfg in stage_configs:
|
|
if get_stage_type(stage_cfg) != "diffusion":
|
|
continue
|
|
model_class_name = _stage_diffusion_model_class_name(stage_cfg)
|
|
if not model_class_name:
|
|
continue
|
|
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
|
if model_cls is not None:
|
|
model_classes.append(model_cls)
|
|
return model_classes
|
|
|
|
|
|
def _normalize_reference_video_decode_spec(spec: ReferenceVideoDecodeSpec) -> ReferenceVideoDecodeSpec:
|
|
max_frames = spec.max_frames
|
|
if max_frames is not None:
|
|
try:
|
|
max_frames = int(max_frames)
|
|
except (TypeError, ValueError) as exc:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Invalid reference video decode spec: max_frames must be an integer.",
|
|
) from exc
|
|
if max_frames <= 0:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Invalid reference video decode spec: max_frames must be positive.",
|
|
)
|
|
|
|
keep = str(spec.keep or "first").strip().lower()
|
|
if keep not in {"first", "last"}:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Invalid reference video decode spec: keep must be either 'first' or 'last'.",
|
|
)
|
|
return ReferenceVideoDecodeSpec(max_frames=max_frames, keep=cast(Literal["first", "last"], keep))
|
|
|
|
|
|
def _reference_video_decode_spec(
|
|
req: VideoGenerationRequest,
|
|
stage_configs: list[Any] | None,
|
|
) -> ReferenceVideoDecodeSpec:
|
|
video_params = req.resolve_video_params()
|
|
extra_params = req.extra_params if isinstance(req.extra_params, dict) else {}
|
|
for model_cls in _diffusion_model_classes(stage_configs):
|
|
resolver = getattr(model_cls, "reference_video_decode_spec", None)
|
|
if resolver is None:
|
|
continue
|
|
try:
|
|
spec = resolver(num_frames=video_params.num_frames, extra_args=extra_params)
|
|
except (TypeError, ValueError) as exc:
|
|
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value, detail=str(exc)) from exc
|
|
if spec is not None:
|
|
return _normalize_reference_video_decode_spec(spec)
|
|
return ReferenceVideoDecodeSpec(max_frames=video_params.num_frames, keep="first")
|
|
|
|
|
|
def video_response_from_request(model_name: str, req: VideoGenerationRequest) -> VideoResponse:
|
|
video_params = req.resolve_video_params()
|
|
duration_s = None
|
|
if req.seconds is not None:
|
|
duration_s = float(req.seconds)
|
|
elif video_params.num_frames is not None and video_params.fps is not None:
|
|
duration_s = video_params.num_frames / video_params.fps
|
|
|
|
resp = VideoResponse(
|
|
model=model_name,
|
|
status=VideoGenerationStatus.QUEUED,
|
|
size=req.size,
|
|
prompt=req.prompt,
|
|
quality=req.quality or "default",
|
|
fps=video_params.fps,
|
|
num_frames=video_params.num_frames,
|
|
duration_s=duration_s,
|
|
)
|
|
resp.seconds = str(req.seconds or resp.seconds)
|
|
return resp
|
|
|
|
|
|
def _status_code_for_video_failure(error: VideoError | None) -> int:
|
|
if error is None:
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
|
|
if isinstance(error.code, int):
|
|
if 400 <= error.code < 600:
|
|
return error.code
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
|
|
if error.code == "HTTPException":
|
|
status_text, _, _ = error.message.partition(":")
|
|
try:
|
|
status_code = int(status_text)
|
|
except ValueError:
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
if 400 <= status_code < 600:
|
|
return status_code
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
|
|
if error.code == "EngineDeadError":
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
if error.code == "EngineGenerateError":
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
|
|
return HTTPStatus.INTERNAL_SERVER_ERROR.value
|
|
|
|
|
|
def _video_error_from_exception(exc: Exception) -> VideoError:
|
|
if isinstance(exc, HTTPException):
|
|
message = str(exc.detail) if exc.detail else str(exc)
|
|
return VideoError(code=exc.status_code, message=message)
|
|
|
|
if isinstance(exc, OmniClientError):
|
|
return VideoError(code=exc.status_code, message=exc.message)
|
|
|
|
if isinstance(exc, (EngineGenerateError, EngineDeadError)):
|
|
err = create_error_response(exc)
|
|
return VideoError(code=err.error.code, message=err.error.message)
|
|
|
|
return VideoError(
|
|
code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
|
message=str(exc),
|
|
)
|
|
|
|
|
|
async def _cleanup_video(video_id: str):
|
|
try:
|
|
await STORAGE_MANAGER.delete(video_id)
|
|
except Exception:
|
|
logger.warning("Failed to cleanup partial video file '%s'", video_id)
|
|
|
|
|
|
def _cleanup_video_references(
|
|
reference_video: ReferenceVideo | None,
|
|
reference_audio: ReferenceAudio | None,
|
|
control_path: str | None = None,
|
|
latent_edit_input: LatentEditInput | None = None,
|
|
) -> None:
|
|
if reference_video is not None:
|
|
for path in reference_video.cleanup_paths:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
if reference_audio is not None:
|
|
cleanup_paths = reference_audio.cleanup_paths or tuple(_reference_list(reference_audio.path))
|
|
for path in cleanup_paths:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
if control_path is not None and os.path.exists(control_path):
|
|
os.unlink(control_path)
|
|
if latent_edit_input is not None:
|
|
for path in latent_edit_input.cleanup_paths:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
|
|
|
|
def _unpack_video_generation_result(
|
|
result: Sequence[object],
|
|
) -> tuple[bytes, dict[str, float], float, VideoAction | None, dict[str, object]]:
|
|
video_metadata: dict[str, object] = {}
|
|
if len(result) == 5:
|
|
video_bytes, stage_durations, peak_memory_mb, action, raw_metadata = result
|
|
if isinstance(raw_metadata, dict):
|
|
video_metadata = {str(key): value for key, value in raw_metadata.items()}
|
|
else:
|
|
video_bytes, stage_durations, peak_memory_mb, action = result
|
|
return (
|
|
cast(bytes, video_bytes),
|
|
cast(dict[str, float], stage_durations),
|
|
float(cast(float, peak_memory_mb)),
|
|
cast(VideoAction | None, action),
|
|
video_metadata,
|
|
)
|
|
|
|
|
|
async def _run_video_generation_job(
|
|
handler: OmniOpenAIServingVideo,
|
|
request: VideoGenerationRequest,
|
|
video_id: str,
|
|
reference_image: ReferenceImage | None = None,
|
|
reference_video: ReferenceVideo | None = None,
|
|
reference_audio: ReferenceAudio | None = None,
|
|
control_path: str | None = None,
|
|
app_state: Any | None = None,
|
|
latent_edit_input: LatentEditInput | None = None,
|
|
) -> None:
|
|
job = await VIDEO_STORE.get(video_id)
|
|
if job is None:
|
|
logger.warning("Video job %s missing before generation task started; skipping", video_id)
|
|
_cleanup_video_references(reference_video, reference_audio, control_path, latent_edit_input)
|
|
return
|
|
|
|
started_at = time.perf_counter()
|
|
try:
|
|
|
|
async def _mark_started() -> None:
|
|
await VIDEO_STORE.update_fields(video_id, {"status": VideoGenerationStatus.IN_PROGRESS})
|
|
|
|
video_bytes, stage_durations, peak_memory_mb, action, video_metadata = _unpack_video_generation_result(
|
|
await handler.generate_video_bytes(
|
|
request,
|
|
video_id,
|
|
reference_image=reference_image,
|
|
reference_video=reference_video,
|
|
reference_audio=reference_audio,
|
|
on_started=_mark_started,
|
|
latent_edit_input=latent_edit_input,
|
|
)
|
|
)
|
|
|
|
save_context = await STORAGE_MANAGER.save(video_bytes, video_id)
|
|
logger.info("Video request %s persisted %s output file.", video_id, save_context.key)
|
|
|
|
updated_fields = {
|
|
"status": VideoGenerationStatus.COMPLETED,
|
|
"progress": 100,
|
|
"file_name": f"{video_id}.{job.file_extension}",
|
|
"completed_at": save_context.created_at,
|
|
"inference_time_s": time.perf_counter() - started_at,
|
|
"stage_durations": stage_durations,
|
|
"peak_memory_mb": peak_memory_mb,
|
|
"action": action,
|
|
}
|
|
updated_fields.update({key: value for key, value in video_metadata.items() if value is not None})
|
|
if save_context.expires_at is not None:
|
|
updated_fields["expires_at"] = save_context.expires_at
|
|
|
|
await VIDEO_STORE.update_fields(video_id, updated_fields)
|
|
except (EngineGenerateError, EngineDeadError) as exc:
|
|
logger.exception("Video generation failed (engine error) for id=%s", video_id)
|
|
|
|
await _cleanup_video(video_id)
|
|
await VIDEO_STORE.update_fields(
|
|
video_id,
|
|
{
|
|
"status": VideoGenerationStatus.FAILED,
|
|
"completed_at": int(time.time()),
|
|
"error": _video_error_from_exception(exc),
|
|
"inference_time_s": time.perf_counter() - started_at,
|
|
},
|
|
)
|
|
# Background tasks can't propagate exceptions to FastAPI handlers.
|
|
# Actively signal shutdown when the engine is dead.
|
|
if app_state is not None and isinstance(exc, EngineDeadError):
|
|
terminate_if_errored(
|
|
server=app_state.server,
|
|
engine=app_state.engine_client,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("Video generation failed for id=%s", video_id)
|
|
|
|
await _cleanup_video(video_id)
|
|
await VIDEO_STORE.update_fields(
|
|
video_id,
|
|
{
|
|
"status": VideoGenerationStatus.FAILED,
|
|
"completed_at": int(time.time()),
|
|
"error": _video_error_from_exception(exc),
|
|
"inference_time_s": time.perf_counter() - started_at,
|
|
},
|
|
)
|
|
except asyncio.CancelledError:
|
|
await _cleanup_video(video_id)
|
|
await VIDEO_STORE.pop(video_id)
|
|
raise
|
|
finally:
|
|
_cleanup_video_references(reference_video, reference_audio, control_path, latent_edit_input)
|
|
|
|
|
|
async def _persist_uploaded_video_references(uploads: list[UploadFile]) -> list[str]:
|
|
paths: list[str] = []
|
|
try:
|
|
for upload in uploads:
|
|
suffix = Path(upload.filename or "").suffix.lower()
|
|
if suffix not in {".mkv", ".mov", ".mp4", ".webm"}:
|
|
suffix = ".mp4"
|
|
fd, path = tempfile.mkstemp(prefix="vllm_omni_video_reference_", suffix=suffix)
|
|
paths.append(path)
|
|
with os.fdopen(fd, "wb") as output:
|
|
while chunk := await upload.read(1024 * 1024):
|
|
output.write(chunk)
|
|
except Exception:
|
|
for path in paths:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
raise
|
|
return paths
|
|
|
|
|
|
async def _persist_uploaded_reference(
|
|
upload: UploadFile,
|
|
*,
|
|
field_name: str,
|
|
suffix: str,
|
|
max_bytes: int,
|
|
) -> str:
|
|
"""Stream one bounded upload to request-scoped local storage."""
|
|
declared_size = upload.size
|
|
if isinstance(declared_size, Integral) and int(declared_size) > max_bytes:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} exceeds the {max_bytes // (1024 * 1024)} MiB size limit.",
|
|
)
|
|
|
|
fd, path = tempfile.mkstemp(prefix=f"vllm_omni_{field_name}_", suffix=suffix)
|
|
size = 0
|
|
persisted = False
|
|
try:
|
|
with os.fdopen(fd, "wb") as output:
|
|
while chunk := await upload.read(min(1024 * 1024, max_bytes - size + 1)):
|
|
size += len(chunk)
|
|
if size > max_bytes:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} exceeds the {max_bytes // (1024 * 1024)} MiB size limit.",
|
|
)
|
|
output.write(chunk)
|
|
if size == 0:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} must not be empty.",
|
|
)
|
|
persisted = True
|
|
return path
|
|
finally:
|
|
if not persisted:
|
|
with suppress(OSError):
|
|
os.unlink(path)
|
|
|
|
|
|
async def _persist_uploaded_control_reference(
|
|
upload: UploadFile,
|
|
*,
|
|
max_bytes: int = CONTROL_REFERENCE_MAX_BYTES,
|
|
) -> str:
|
|
"""Stream one model control upload to request-scoped local storage."""
|
|
kind = _uploaded_media_kind(upload)
|
|
suffix = Path(upload.filename or "").suffix.lower()
|
|
supported_suffixes = CONTROL_REFERENCE_IMAGE_SUFFIXES | CONTROL_REFERENCE_VIDEO_SUFFIXES
|
|
if kind == "audio" or (suffix and suffix not in supported_suffixes):
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="control_reference must be an image or video file.",
|
|
)
|
|
if not suffix:
|
|
suffix = ".png" if kind == "image" else ".mp4"
|
|
return await _persist_uploaded_reference(
|
|
upload,
|
|
field_name="control_reference",
|
|
suffix=suffix,
|
|
max_bytes=max_bytes,
|
|
)
|
|
|
|
|
|
async def _persist_latent_edit_source(
|
|
upload: UploadFile,
|
|
*,
|
|
field_name: Literal["source_video", "source_audio"],
|
|
max_bytes: int = LATENT_EDIT_SOURCE_MAX_BYTES,
|
|
) -> str:
|
|
"""Stream one latent-edit source upload to request-scoped storage."""
|
|
suffix = Path(upload.filename or "").suffix.lower()
|
|
if field_name == "source_video":
|
|
allowed_suffixes = MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES
|
|
expected = "an MP4 or MOV file"
|
|
else:
|
|
allowed_suffixes = MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES
|
|
expected = "a WAV or MP3 file"
|
|
if suffix not in allowed_suffixes:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"{field_name} must be {expected}.",
|
|
)
|
|
|
|
return await _persist_uploaded_reference(
|
|
upload,
|
|
field_name=field_name,
|
|
suffix=suffix,
|
|
max_bytes=max_bytes,
|
|
)
|
|
|
|
|
|
def _validate_latent_edit_inputs(
|
|
handler: OmniOpenAIServingVideo,
|
|
*,
|
|
source_video: UploadFile | None,
|
|
source_audio: UploadFile | None,
|
|
video_noise_mask: Any | None,
|
|
audio_noise_mask: Any | None,
|
|
) -> bool:
|
|
"""Validate cross-field latent-edit requirements before persisting files."""
|
|
has_any_input = any(value is not None for value in (source_video, source_audio, video_noise_mask, audio_noise_mask))
|
|
if not has_any_input:
|
|
return False
|
|
if not handler.supports_latent_mask_editing:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Latent-mask editing is not supported by this model.",
|
|
)
|
|
if video_noise_mask is None and audio_noise_mask is None:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Latent-mask editing requires at least one of video_noise_mask or audio_noise_mask.",
|
|
)
|
|
if source_audio is not None and audio_noise_mask is None:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="source_audio requires an audio_noise_mask upload.",
|
|
)
|
|
return True
|
|
|
|
|
|
def _validate_control_upload(
|
|
handler: OmniOpenAIServingVideo,
|
|
request: VideoGenerationRequest,
|
|
control_reference: UploadFile | None,
|
|
control_type: str | None,
|
|
) -> str | None:
|
|
"""Validate a generic uploaded control against model capabilities."""
|
|
if control_reference is None:
|
|
if control_type is not None:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="control_type requires a control_reference upload.",
|
|
)
|
|
return None
|
|
if control_type is None or not control_type.strip():
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="control_reference requires control_type.",
|
|
)
|
|
|
|
normalized_type = control_type.strip().lower()
|
|
supported_types = handler.supported_control_upload_types
|
|
if normalized_type not in supported_types:
|
|
supported = ", ".join(sorted(supported_types)) or "none"
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=(
|
|
f"Control upload type '{normalized_type}' is not supported by this model. "
|
|
f"Supported control upload types: {supported}."
|
|
),
|
|
)
|
|
|
|
existing = (request.extra_params or {}).get(normalized_type)
|
|
if existing is not None:
|
|
if not isinstance(existing, Mapping) and existing is not True:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"extra_params.{normalized_type} must be an object when control_reference is uploaded.",
|
|
)
|
|
if isinstance(existing, Mapping) and any(existing.get(key) is not None for key in ("control", "control_path")):
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=(
|
|
"Provide either control_reference or "
|
|
f"extra_params.{normalized_type}.control/control_path, not both."
|
|
),
|
|
)
|
|
return normalized_type
|
|
|
|
|
|
def _attach_control_upload(
|
|
request: VideoGenerationRequest,
|
|
control_type: str,
|
|
control_path: str | None = None,
|
|
) -> None:
|
|
"""Declare an uploaded control and attach its persisted path when available."""
|
|
extra_params = dict(request.extra_params or {})
|
|
existing = extra_params.get(control_type)
|
|
control_params = dict(existing) if isinstance(existing, Mapping) else {}
|
|
if control_path is not None:
|
|
control_params["control_path"] = control_path
|
|
extra_params[control_type] = control_params
|
|
request.extra_params = extra_params
|
|
|
|
|
|
def _reference_list(value: Any) -> list[Any]:
|
|
if value is None:
|
|
return []
|
|
return list(value) if isinstance(value, list) else [value]
|
|
|
|
|
|
def _uploaded_media_kind(upload: UploadFile) -> str:
|
|
content_type = (upload.content_type or "").lower()
|
|
if content_type.startswith("image/"):
|
|
return "image"
|
|
if content_type.startswith("audio/"):
|
|
return "audio"
|
|
suffix = Path(upload.filename or "").suffix.lower()
|
|
if suffix in {".png", ".jpg", ".jpeg", ".webp", ".heic", ".heif"}:
|
|
return "image"
|
|
if suffix in {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg"}:
|
|
return "audio"
|
|
return "video"
|
|
|
|
|
|
def _minimax_h3_upload_limit(upload: UploadFile) -> int:
|
|
kind = _uploaded_media_kind(upload)
|
|
if kind == "image":
|
|
return MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES
|
|
if kind == "audio":
|
|
return MINIMAX_H3_MAX_REFERENCE_AUDIO_BYTES
|
|
return MINIMAX_H3_MAX_REFERENCE_VIDEO_BYTES
|
|
|
|
|
|
async def _read_upload_limited(upload: UploadFile, *, max_bytes: int | None = None) -> bytes:
|
|
"""Read an upload with an optional hard byte limit."""
|
|
if max_bytes is None:
|
|
return await upload.read()
|
|
|
|
declared_size = getattr(upload, "size", None)
|
|
if isinstance(declared_size, Integral) and int(declared_size) > max_bytes:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"Uploaded reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.",
|
|
)
|
|
|
|
chunks: list[bytes] = []
|
|
total = 0
|
|
while True:
|
|
chunk = await upload.read(min(1024 * 1024, max_bytes - total + 1))
|
|
if not chunk:
|
|
break
|
|
total += len(chunk)
|
|
if total > max_bytes:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"Uploaded reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.",
|
|
)
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
|
|
|
|
def _validate_minimax_h3_image_payload(
|
|
payload: bytes,
|
|
*,
|
|
filename: str | None,
|
|
allow_non_image: bool = False,
|
|
) -> None:
|
|
"""Validate a H3 image before converting it to a format-less PIL image."""
|
|
try:
|
|
with Image.open(io.BytesIO(payload)) as image:
|
|
image_format = str(image.format or "").lower()
|
|
if image_format in MINIMAX_H3_REFERENCE_IMAGE_FORMATS:
|
|
_validate_image_pixel_limit(image)
|
|
except InvalidInputReferenceError as exc:
|
|
raise HTTPException(HTTPStatus.BAD_REQUEST.value, detail=str(exc)) from exc
|
|
except Image.DecompressionBombError as exc:
|
|
raise HTTPException(
|
|
HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"Invalid uploaded image reference: {filename}; image exceeds the decoder pixel limit.",
|
|
) from exc
|
|
except (OSError, ValueError) as exc:
|
|
if allow_non_image:
|
|
return
|
|
raise HTTPException(400, detail=f"Invalid uploaded image reference: {filename}") from exc
|
|
|
|
if image_format not in MINIMAX_H3_REFERENCE_IMAGE_FORMATS:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=(
|
|
"MiniMax H3 reference images must use JPG, JPEG, PNG, WEBP, HEIC, or HEIF; "
|
|
f"got {image_format or 'unknown'}."
|
|
),
|
|
)
|
|
|
|
|
|
async def _persist_uploaded_media_references(
|
|
uploads: list[UploadFile],
|
|
) -> tuple[list[Image.Image], list[str], list[str]]:
|
|
"""Persist a mixed MiniMax H3 multipart reference list.
|
|
|
|
Images are decoded in memory; videos and audio remain files because H3's
|
|
reference encoders need the original container streams (including video
|
|
soundtracks).
|
|
"""
|
|
images: list[Image.Image] = []
|
|
videos: list[str] = []
|
|
audios: list[str] = []
|
|
paths: list[str] = []
|
|
if len(uploads) > MINIMAX_H3_MAX_REFERENCE_COUNT:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=f"MiniMax H3 accepts at most {MINIMAX_H3_MAX_REFERENCE_COUNT} total references.",
|
|
)
|
|
try:
|
|
for upload in uploads:
|
|
kind = _uploaded_media_kind(upload)
|
|
payload = await _read_upload_limited(upload, max_bytes=_minimax_h3_upload_limit(upload))
|
|
if kind == "image":
|
|
try:
|
|
_validate_minimax_h3_image_payload(payload, filename=upload.filename)
|
|
images.append(
|
|
_decode_image_bytes(
|
|
payload,
|
|
source=f"uploaded image reference: {upload.filename}",
|
|
)
|
|
)
|
|
except InvalidInputReferenceError as exc:
|
|
raise HTTPException(HTTPStatus.BAD_REQUEST.value, detail=str(exc)) from exc
|
|
continue
|
|
suffix = Path(upload.filename or "").suffix.lower()
|
|
if kind == "video" and suffix and suffix not in MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="MiniMax H3 reference videos must use an MP4 or MOV file.",
|
|
)
|
|
if kind == "audio" and suffix and suffix not in MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="MiniMax H3 reference audio must use a WAV or MP3 file.",
|
|
)
|
|
if not suffix or len(suffix) > 8:
|
|
suffix = ".mp3" if kind == "audio" else ".mp4"
|
|
fd, path = tempfile.mkstemp(prefix="vllm_omni_reference_", suffix=suffix)
|
|
paths.append(path)
|
|
with os.fdopen(fd, "wb") as output:
|
|
output.write(payload)
|
|
if kind == "audio":
|
|
audios.append(path)
|
|
else:
|
|
videos.append(path)
|
|
except Exception:
|
|
for path in paths:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
raise
|
|
return images, videos, audios
|
|
|
|
|
|
async def _parse_video_form(
|
|
raw_request: Request,
|
|
prompt: str = Form(...),
|
|
input_reference: UploadFile | None = File(default=None),
|
|
input_references: list[UploadFile] | None = File(default=None),
|
|
control_reference: UploadFile | None = File(default=None),
|
|
control_type: str | None = Form(default=None),
|
|
image_reference: str | None = Form(default=None),
|
|
video_reference: str | None = Form(default=None),
|
|
audio_reference: str | None = Form(default=None),
|
|
model: str | None = Form(default=None),
|
|
seconds: SecondStr | None = Form(default=None),
|
|
size: SizeStr | None = Form(default=None),
|
|
user: str | None = Form(default=None),
|
|
width: int | None = Form(default=None),
|
|
height: int | None = Form(default=None),
|
|
num_frames: int | None = Form(default=None),
|
|
fps: float | None = Form(default=None, ge=1, allow_inf_nan=False),
|
|
aspect_ratio: str | None = Form(default=None),
|
|
short_edge: int | None = Form(default=None, ge=1),
|
|
num_outputs_per_prompt: int = Form(default=1, ge=1, le=10),
|
|
start_time_seconds: float | None = Form(default=None, ge=0.0),
|
|
quality: str | None = Form(default=None),
|
|
num_inference_steps: int | None = Form(default=None),
|
|
guidance_scale: float | None = Form(default=None),
|
|
guidance_scale_2: float | None = Form(default=None),
|
|
boundary_ratio: float | None = Form(default=None),
|
|
flow_shift: float | None = Form(default=None),
|
|
true_cfg_scale: float | None = Form(default=None),
|
|
seed: int | None = Form(default=None),
|
|
generate_sound: bool | None = Form(default=None),
|
|
sound_duration: float | None = Form(default=None, gt=0.0),
|
|
negative_prompt: str | None = Form(default=None),
|
|
enable_frame_interpolation: bool | None = Form(default=None),
|
|
frame_interpolation_exp: int | None = Form(default=None, ge=1),
|
|
frame_interpolation_scale: float | None = Form(default=None, gt=0.0),
|
|
frame_interpolation_model_path: str | None = Form(default=None),
|
|
lora: str | None = Form(default=None),
|
|
extra_params: str | None = Form(default=None),
|
|
return_stage_metrics: bool | None = Form(default=None),
|
|
source_video: UploadFile | None = File(default=None),
|
|
source_audio: UploadFile | None = File(default=None),
|
|
video_noise_mask: UploadFile | None = File(default=None),
|
|
audio_noise_mask: UploadFile | None = File(default=None),
|
|
) -> tuple[
|
|
VideoGenerationRequest,
|
|
"OmniOpenAIServingVideo",
|
|
str,
|
|
ReferenceImage | None,
|
|
ReferenceVideo | None,
|
|
ReferenceAudio | None,
|
|
str | None,
|
|
LatentEditInput | None,
|
|
]:
|
|
"""FastAPI dependency that parses video form data, validates inputs,
|
|
resolves the handler, and decodes any reference image.
|
|
|
|
Used by both ``POST /v1/videos`` (async) and ``POST /v1/videos/sync``.
|
|
"""
|
|
input_references = input_references or []
|
|
input_reference_bytes: bytes | None = None
|
|
parsed_image_reference = _parse_form_json(image_reference)
|
|
parsed_video_reference = _parse_form_json(video_reference)
|
|
parsed_audio_reference = _parse_form_json(audio_reference)
|
|
parsed_video_noise_mask = await _read_latent_edit_mask_json(
|
|
video_noise_mask,
|
|
field_name="video_noise_mask",
|
|
)
|
|
parsed_audio_noise_mask = await _read_latent_edit_mask_json(
|
|
audio_noise_mask,
|
|
field_name="audio_noise_mask",
|
|
)
|
|
|
|
if input_references and any(
|
|
item is not None for item in (parsed_image_reference, parsed_video_reference, input_reference)
|
|
):
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail="Provide input_references alone, without input_reference, image_reference, or video_reference.",
|
|
)
|
|
if input_reference is not None and (parsed_image_reference is not None or parsed_video_reference is not None):
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=(
|
|
"Provide only one of input_reference, image_reference, or video_reference when using "
|
|
"input_reference; image_reference and video_reference may be combined."
|
|
),
|
|
)
|
|
|
|
request_data: dict[str, Any] = {
|
|
"prompt": prompt,
|
|
"model": model,
|
|
"seconds": seconds,
|
|
"size": size,
|
|
"image_reference": parsed_image_reference,
|
|
"video_reference": parsed_video_reference,
|
|
"audio_reference": parsed_audio_reference,
|
|
"user": user,
|
|
"width": width,
|
|
"height": height,
|
|
"num_frames": num_frames,
|
|
"fps": fps,
|
|
"aspect_ratio": aspect_ratio,
|
|
"short_edge": short_edge,
|
|
"num_outputs_per_prompt": num_outputs_per_prompt,
|
|
"start_time_seconds": start_time_seconds,
|
|
"quality": quality,
|
|
"num_inference_steps": num_inference_steps,
|
|
"guidance_scale": guidance_scale,
|
|
"guidance_scale_2": guidance_scale_2,
|
|
"boundary_ratio": boundary_ratio,
|
|
"flow_shift": flow_shift,
|
|
"true_cfg_scale": true_cfg_scale,
|
|
"seed": seed,
|
|
"generate_sound": generate_sound,
|
|
"sound_duration": sound_duration,
|
|
"negative_prompt": negative_prompt,
|
|
"enable_frame_interpolation": enable_frame_interpolation,
|
|
"frame_interpolation_exp": frame_interpolation_exp,
|
|
"frame_interpolation_scale": frame_interpolation_scale,
|
|
"frame_interpolation_model_path": frame_interpolation_model_path,
|
|
"lora": _parse_form_json(lora, expected_type=dict),
|
|
"extra_params": _parse_form_json(extra_params, expected_type=dict),
|
|
"return_stage_metrics": return_stage_metrics,
|
|
}
|
|
request_data = {k: v for k, v in request_data.items() if v is not None}
|
|
request = VideoGenerationRequest(**request_data)
|
|
|
|
handler = Omnivideo(raw_request)
|
|
if handler is None:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.SERVICE_UNAVAILABLE.value,
|
|
detail="Video generation handler not initialized.",
|
|
)
|
|
logger.info("Video generation handler: %s", type(handler).__name__)
|
|
try:
|
|
app_model_name, app_stage_configs = _resolve_video_runtime_context(raw_request)
|
|
effective_model_name = handler.model_name or app_model_name or request.model or "unknown"
|
|
if request.model is not None and effective_model_name is not None and request.model != effective_model_name:
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.BAD_REQUEST.value,
|
|
detail=(
|
|
f"Model mismatch: request specifies '{request.model}' but server is running "
|
|
f"'{effective_model_name}'."
|
|
),
|
|
)
|
|
handler.set_stage_configs_if_missing(app_stage_configs)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.exception("Video generation setup failed: %s", e)
|
|
raise HTTPException(
|
|
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
|
detail=f"Video generation setup failed: {str(e)}",
|
|
)
|
|
|
|
has_latent_edit = _validate_latent_edit_inputs(
|
|
handler,
|
|
source_video=source_video,
|
|
source_audio=source_audio,
|
|
video_noise_mask=parsed_video_noise_mask,
|
|
audio_noise_mask=parsed_audio_noise_mask,
|
|
)
|
|
|
|
normalized_control_type = _validate_control_upload(handler, request, control_reference, control_type)
|
|
if normalized_control_type is not None:
|
|
# Make the selected transfer mode visible while choosing the model's
|
|
# reference-video decode policy. The upload itself stays unpersisted
|
|
# until reference parsing succeeds, preserving failure cleanup.
|
|
_attach_control_upload(request, normalized_control_type)
|
|
|
|
supports_mixed_reference_inputs = handler.supports_mixed_reference_inputs
|
|
if input_reference is not None:
|
|
input_reference_bytes = await _read_upload_limited(
|
|
input_reference,
|
|
max_bytes=_minimax_h3_upload_limit(input_reference) if supports_mixed_reference_inputs else None,
|
|
)
|
|
if supports_mixed_reference_inputs:
|
|
input_reference_kind = _uploaded_media_kind(input_reference)
|
|
_validate_minimax_h3_image_payload(
|
|
input_reference_bytes,
|
|
filename=input_reference.filename,
|
|
allow_non_image=input_reference_kind != "image",
|
|
)
|
|
|
|
decode_spec = ReferenceVideoDecodeSpec()
|
|
if not input_references and (parsed_video_reference is not None or input_reference_bytes is not None):
|
|
stage_configs = (
|
|
handler.stage_configs
|
|
or app_stage_configs
|
|
or getattr(getattr(handler, "_engine_client", None), "stage_configs", None)
|
|
)
|
|
decode_spec = _reference_video_decode_spec(request, stage_configs)
|
|
reference_image = None
|
|
reference_video = None
|
|
reference_audio: ReferenceAudio | None = None
|
|
if input_references:
|
|
if not supports_mixed_reference_inputs:
|
|
video_paths = await _persist_uploaded_video_references(input_references)
|
|
reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths))
|
|
images, audio_paths = [], []
|
|
else:
|
|
images, video_paths, audio_paths = await _persist_uploaded_media_references(input_references)
|
|
if images:
|
|
reference_image = ReferenceImage(data=images if len(images) > 1 else images[0])
|
|
if video_paths:
|
|
reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths))
|
|
if audio_paths:
|
|
reference_audio = ReferenceAudio(path=audio_paths, cleanup_paths=tuple(audio_paths))
|
|
else:
|
|
video_paths: list[str] = []
|
|
try:
|
|
image_items = _reference_list(request.image_reference)
|
|
video_items = _reference_list(request.video_reference)
|
|
image_data = []
|
|
for item in image_items:
|
|
media_data = await decode_input_reference(item, None, None)
|
|
if not isinstance(media_data, Image.Image):
|
|
raise InvalidInputReferenceError("image_reference did not decode to an image")
|
|
image_data.append(media_data)
|
|
|
|
video_frames: list[Image.Image] | None = None
|
|
for item in video_items:
|
|
media_data = await decode_input_reference(
|
|
None,
|
|
item,
|
|
None,
|
|
max_video_frames=decode_spec.max_frames,
|
|
video_keep=decode_spec.keep,
|
|
)
|
|
if not isinstance(media_data, VideoFrames):
|
|
raise InvalidInputReferenceError("video_reference did not decode to a video")
|
|
if media_data.source_path is not None:
|
|
video_paths.append(media_data.source_path)
|
|
else:
|
|
if len(video_items) != 1:
|
|
raise InvalidInputReferenceError(
|
|
"multiple video URL references must be downloadable source videos"
|
|
)
|
|
video_frames = list(media_data)
|
|
|
|
if input_reference_bytes is not None:
|
|
media_data = await decode_input_reference(
|
|
None,
|
|
None,
|
|
input_reference_bytes,
|
|
max_video_frames=decode_spec.max_frames,
|
|
video_keep=decode_spec.keep,
|
|
)
|
|
if isinstance(media_data, Image.Image):
|
|
image_data.append(media_data)
|
|
elif isinstance(media_data, VideoFrames):
|
|
if media_data.source_path is not None:
|
|
video_paths.append(media_data.source_path)
|
|
else:
|
|
video_frames = list(media_data)
|
|
|
|
if image_data:
|
|
reference_image = ReferenceImage(data=image_data if len(image_data) > 1 else image_data[0])
|
|
if video_paths:
|
|
reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths))
|
|
elif video_frames is not None:
|
|
reference_video = ReferenceVideo(data=video_frames)
|
|
except InvalidInputReferenceError as exc:
|
|
for path in video_paths:
|
|
if os.path.exists(path):
|
|
os.unlink(path)
|
|
raise HTTPException(400, detail=str(exc) or "Invalid input reference.") from exc
|
|
|
|
audio_paths = [] if reference_audio is None else list(_reference_list(reference_audio.path))
|
|
if request.audio_reference is not None:
|
|
try:
|
|
for audio_reference in _reference_list(request.audio_reference):
|
|
audio_paths.append(await decode_audio_url(audio_reference.audio_url))
|
|
except InvalidInputReferenceError as exc:
|
|
_cleanup_video_references(reference_video, reference_audio)
|
|
cleanup_paths = set(() if reference_audio is None else reference_audio.cleanup_paths)
|
|
for path in audio_paths:
|
|
if path not in cleanup_paths and os.path.exists(path):
|
|
os.unlink(path)
|
|
raise HTTPException(400, detail=str(exc)) from exc
|
|
if audio_paths:
|
|
cleanup_paths = (
|
|
tuple(audio_paths)
|
|
if reference_audio is None
|
|
else reference_audio.cleanup_paths
|
|
+ tuple(path for path in audio_paths if path not in reference_audio.cleanup_paths)
|
|
)
|
|
reference_audio = ReferenceAudio(
|
|
path=audio_paths if len(audio_paths) > 1 else audio_paths[0],
|
|
cleanup_paths=cleanup_paths,
|
|
)
|
|
|
|
control_path: str | None = None
|
|
if control_reference is not None and normalized_control_type is not None:
|
|
try:
|
|
control_path = await _persist_uploaded_control_reference(
|
|
control_reference,
|
|
max_bytes=CONTROL_REFERENCE_MAX_BYTES,
|
|
)
|
|
_attach_control_upload(request, normalized_control_type, control_path)
|
|
except (asyncio.CancelledError, HTTPException, OSError, TypeError, ValueError):
|
|
_cleanup_video_references(reference_video, reference_audio, control_path)
|
|
raise
|
|
|
|
latent_edit_input: LatentEditInput | None = None
|
|
if has_latent_edit:
|
|
source_paths: list[str] = []
|
|
try:
|
|
source_video_path = None
|
|
source_audio_path = None
|
|
if source_video is not None:
|
|
source_video_path = await _persist_latent_edit_source(
|
|
source_video,
|
|
field_name="source_video",
|
|
)
|
|
source_paths.append(source_video_path)
|
|
if source_audio is not None:
|
|
source_audio_path = await _persist_latent_edit_source(
|
|
source_audio,
|
|
field_name="source_audio",
|
|
)
|
|
source_paths.append(source_audio_path)
|
|
latent_edit_input = LatentEditInput(
|
|
source_video=source_video_path,
|
|
source_audio=source_audio_path,
|
|
video_noise_mask=parsed_video_noise_mask,
|
|
audio_noise_mask=parsed_audio_noise_mask,
|
|
cleanup_paths=tuple(source_paths),
|
|
)
|
|
except (asyncio.CancelledError, Exception):
|
|
for path in source_paths:
|
|
with suppress(OSError):
|
|
os.unlink(path)
|
|
_cleanup_video_references(reference_video, reference_audio, control_path)
|
|
raise
|
|
|
|
return (
|
|
request,
|
|
handler,
|
|
effective_model_name,
|
|
reference_image,
|
|
reference_video,
|
|
reference_audio,
|
|
control_path,
|
|
latent_edit_input,
|
|
)
|