[Feature] Align MiniMax H3 official input matrix (#5752)

Signed-off-by: david6666666 <530634352@qq.com>
This commit is contained in:
WeiQing Chen authored and GitHub committed 2026-08-05 07:12:01 +00:00
1 parent 6a604be610
commit ae7bee0c43
21 files changed
+1887 -331

No files matched your search

+274 -26
View File
@@ -144,7 +144,11 @@ from vllm_omni.entrypoints.openai.stage_params import (
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER, FileStorageHandle
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE, VIDEO_TASKS
from vllm_omni.entrypoints.openai.utils import get_stage_type, parse_lora_request
from vllm_omni.entrypoints.openai.video_api_utils import decode_audio_url, decode_input_reference
from vllm_omni.entrypoints.openai.video_api_utils import (
VideoFrames,
decode_audio_url,
decode_input_reference,
)
from vllm_omni.entrypoints.openpi.serving import ServingRealtimeRobotOpenPI
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
@@ -155,6 +159,13 @@ logger = init_logger(__name__)
router = APIRouter()
MAX_UINT32_SEED = 2**32 - 1
MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
MINIMAX_H3_MAX_REFERENCE_VIDEO_BYTES = 50 * 1024 * 1024
MINIMAX_H3_MAX_REFERENCE_AUDIO_BYTES = 15 * 1024 * 1024
MINIMAX_H3_MAX_REFERENCE_COUNT = 12
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES = frozenset({".mp4", ".mov"})
MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES = frozenset({".wav", ".mp3"})
profiler_router = APIRouter()
@@ -2865,8 +2876,11 @@ def _cleanup_video_references(
for path in reference_video.cleanup_paths:
if os.path.exists(path):
os.unlink(path)
if reference_audio is not None and os.path.exists(reference_audio.path):
os.unlink(reference_audio.path)
if reference_audio is not None:
cleanup_paths = reference_audio.cleanup_paths or tuple(_reference_list(reference_audio.path))
for path in cleanup_paths:
if os.path.exists(path):
os.unlink(path)
async def _run_video_generation_job(
@@ -2975,6 +2989,147 @@ async def _persist_uploaded_video_references(uploads: list[UploadFile]) -> list[
return paths
def _reference_list(value: Any) -> list[Any]:
if value is None:
return []
return list(value) if isinstance(value, list) else [value]
def _uploaded_media_kind(upload: UploadFile) -> str:
content_type = (upload.content_type or "").lower()
if content_type.startswith("image/"):
return "image"
if content_type.startswith("audio/"):
return "audio"
suffix = Path(upload.filename or "").suffix.lower()
if suffix in {".png", ".jpg", ".jpeg", ".webp", ".heic", ".heif"}:
return "image"
if suffix in {".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg"}:
return "audio"
return "video"
def _minimax_h3_upload_limit(upload: UploadFile) -> int:
kind = _uploaded_media_kind(upload)
if kind == "image":
return MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES
if kind == "audio":
return MINIMAX_H3_MAX_REFERENCE_AUDIO_BYTES
return MINIMAX_H3_MAX_REFERENCE_VIDEO_BYTES
async def _read_upload_limited(upload: UploadFile, *, max_bytes: int | None = None) -> bytes:
"""Read an upload with an optional hard byte limit."""
if max_bytes is None:
return await upload.read()
declared_size = getattr(upload, "size", None)
if isinstance(declared_size, Integral) and int(declared_size) > max_bytes:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"Uploaded reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.",
)
chunks: list[bytes] = []
total = 0
while True:
chunk = await upload.read(min(1024 * 1024, max_bytes - total + 1))
if not chunk:
break
total += len(chunk)
if total > max_bytes:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"Uploaded reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.",
)
chunks.append(chunk)
return b"".join(chunks)
def _validate_minimax_h3_image_payload(
payload: bytes,
*,
filename: str | None,
allow_non_image: bool = False,
) -> None:
"""Validate a H3 image before converting it to a format-less PIL image."""
try:
with Image.open(io.BytesIO(payload)) as image:
image_format = str(image.format or "").lower()
except (OSError, ValueError) as exc:
if allow_non_image:
return
raise HTTPException(400, detail=f"Invalid uploaded image reference: {filename}") from exc
if image_format not in MINIMAX_H3_REFERENCE_IMAGE_FORMATS:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=(
"MiniMax H3 reference images must use JPG, JPEG, PNG, WEBP, HEIC, or HEIF; "
f"got {image_format or 'unknown'}."
),
)
async def _persist_uploaded_media_references(
uploads: list[UploadFile],
) -> tuple[list[Image.Image], list[str], list[str]]:
"""Persist a mixed MiniMax H3 multipart reference list.
Images are decoded in memory; videos and audio remain files because H3's
reference encoders need the original container streams (including video
soundtracks).
"""
images: list[Image.Image] = []
videos: list[str] = []
audios: list[str] = []
paths: list[str] = []
if len(uploads) > MINIMAX_H3_MAX_REFERENCE_COUNT:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"MiniMax H3 accepts at most {MINIMAX_H3_MAX_REFERENCE_COUNT} total references.",
)
try:
for upload in uploads:
kind = _uploaded_media_kind(upload)
payload = await _read_upload_limited(upload, max_bytes=_minimax_h3_upload_limit(upload))
if kind == "image":
try:
_validate_minimax_h3_image_payload(payload, filename=upload.filename)
with Image.open(io.BytesIO(payload)) as image:
images.append(image.convert("RGB"))
except (OSError, ValueError) as exc:
raise HTTPException(400, detail=f"Invalid uploaded image reference: {upload.filename}") from exc
continue
suffix = Path(upload.filename or "").suffix.lower()
if kind == "video" and suffix and suffix not in MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="MiniMax H3 reference videos must use an MP4 or MOV file.",
)
if kind == "audio" and suffix and suffix not in MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="MiniMax H3 reference audio must use a WAV or MP3 file.",
)
if not suffix or len(suffix) > 8:
suffix = ".mp3" if kind == "audio" else ".mp4"
fd, path = tempfile.mkstemp(prefix="vllm_omni_reference_", suffix=suffix)
paths.append(path)
with os.fdopen(fd, "wb") as output:
output.write(payload)
if kind == "audio":
audios.append(path)
else:
videos.append(path)
except Exception:
for path in paths:
if os.path.exists(path):
os.unlink(path)
raise
return images, videos, audios
async def _parse_video_form(
raw_request: Request,
prompt: str = Form(...),
@@ -2991,6 +3146,10 @@ async def _parse_video_form(
height: int | None = Form(default=None),
num_frames: int | None = Form(default=None),
fps: int | None = Form(default=None),
aspect_ratio: str | None = Form(default=None),
short_edge: int | None = Form(default=None, ge=1),
num_outputs_per_prompt: int = Form(default=1, ge=1, le=10),
start_time_seconds: float | None = Form(default=None, ge=0.0),
num_inference_steps: int | None = Form(default=None),
guidance_scale: float | None = Form(default=None),
guidance_scale_2: float | None = Form(default=None),
@@ -3021,25 +3180,25 @@ async def _parse_video_form(
Used by both ``POST /v1/videos`` (async) and ``POST /v1/videos/sync``.
"""
input_references = input_references or []
input_reference_bytes = await input_reference.read() if input_reference is not None else None
input_reference_bytes: bytes | None = None
parsed_image_reference = _parse_form_json(image_reference)
parsed_video_reference = _parse_form_json(video_reference)
parsed_audio_reference = _parse_form_json(audio_reference)
if input_references and any(
item is not None for item in (parsed_image_reference, parsed_video_reference, input_reference_bytes)
item is not None for item in (parsed_image_reference, parsed_video_reference, input_reference)
):
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Provide input_references alone, without input_reference, image_reference, or video_reference.",
)
provided_references = sum(
item is not None for item in (parsed_image_reference, parsed_video_reference, input_reference_bytes)
)
if provided_references > 1:
if input_reference is not None and (parsed_image_reference is not None or parsed_video_reference is not None):
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="Provide only one of input_reference, image_reference, or video_reference.",
detail=(
"Provide only one of input_reference, image_reference, or video_reference when using "
"input_reference; image_reference and video_reference may be combined."
),
)
request_data: dict[str, Any] = {
@@ -3055,6 +3214,10 @@ async def _parse_video_form(
"height": height,
"num_frames": num_frames,
"fps": fps,
"aspect_ratio": aspect_ratio,
"short_edge": short_edge,
"num_outputs_per_prompt": num_outputs_per_prompt,
"start_time_seconds": start_time_seconds,
"num_inference_steps": num_inference_steps,
"guidance_scale": guidance_scale,
"guidance_scale_2": guidance_scale_2,
@@ -3103,6 +3266,20 @@ async def _parse_video_form(
detail=f"Video generation setup failed: {str(e)}",
)
supports_mixed_reference_inputs = bool(getattr(handler, "supports_mixed_reference_inputs", False))
if input_reference is not None:
input_reference_bytes = await _read_upload_limited(
input_reference,
max_bytes=_minimax_h3_upload_limit(input_reference) if supports_mixed_reference_inputs else None,
)
if supports_mixed_reference_inputs:
input_reference_kind = _uploaded_media_kind(input_reference)
_validate_minimax_h3_image_payload(
input_reference_bytes,
filename=input_reference.filename,
allow_non_image=input_reference_kind != "image",
)
decode_spec = ReferenceVideoDecodeSpec()
if not input_references and (parsed_video_reference is not None or input_reference_bytes is not None):
stage_configs = (
@@ -3113,32 +3290,103 @@ async def _parse_video_form(
decode_spec = _reference_video_decode_spec(request, stage_configs)
reference_image = None
reference_video = None
reference_audio: ReferenceAudio | None = None
if input_references:
paths = await _persist_uploaded_video_references(input_references)
reference_video = ReferenceVideo(data=paths, cleanup_paths=tuple(paths))
if not supports_mixed_reference_inputs:
video_paths = await _persist_uploaded_video_references(input_references)
reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths))
images, audio_paths = [], []
else:
images, video_paths, audio_paths = await _persist_uploaded_media_references(input_references)
if images:
reference_image = ReferenceImage(data=images if len(images) > 1 else images[0])
if video_paths:
reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths))
if audio_paths:
reference_audio = ReferenceAudio(path=audio_paths, cleanup_paths=tuple(audio_paths))
else:
video_paths: list[str] = []
try:
media_data = await decode_input_reference(
request.image_reference,
request.video_reference,
input_reference_bytes,
max_video_frames=decode_spec.max_frames,
video_keep=decode_spec.keep,
)
image_items = _reference_list(request.image_reference)
video_items = _reference_list(request.video_reference)
image_data = []
for item in image_items:
media_data = await decode_input_reference(item, None, None)
if not isinstance(media_data, Image.Image):
raise InvalidInputReferenceError("image_reference did not decode to an image")
image_data.append(media_data)
video_frames: list[Image.Image] | None = None
for item in video_items:
media_data = await decode_input_reference(
None,
item,
None,
max_video_frames=decode_spec.max_frames,
video_keep=decode_spec.keep,
)
if not isinstance(media_data, VideoFrames):
raise InvalidInputReferenceError("video_reference did not decode to a video")
if media_data.source_path is not None:
video_paths.append(media_data.source_path)
else:
if len(video_items) != 1:
raise InvalidInputReferenceError(
"multiple video URL references must be downloadable source videos"
)
video_frames = list(media_data)
if input_reference_bytes is not None:
media_data = await decode_input_reference(
None,
None,
input_reference_bytes,
max_video_frames=decode_spec.max_frames,
video_keep=decode_spec.keep,
)
if isinstance(media_data, Image.Image):
image_data.append(media_data)
elif isinstance(media_data, VideoFrames):
if media_data.source_path is not None:
video_paths.append(media_data.source_path)
else:
video_frames = list(media_data)
if image_data:
reference_image = ReferenceImage(data=image_data if len(image_data) > 1 else image_data[0])
if video_paths:
reference_video = ReferenceVideo(data=video_paths, cleanup_paths=tuple(video_paths))
elif video_frames is not None:
reference_video = ReferenceVideo(data=video_frames)
except InvalidInputReferenceError as exc:
for path in video_paths:
if os.path.exists(path):
os.unlink(path)
raise HTTPException(400, detail=str(exc) or "Invalid input reference.") from exc
reference_image = ReferenceImage(data=media_data) if isinstance(media_data, Image.Image) else None
reference_video = ReferenceVideo(data=media_data) if isinstance(media_data, list) else None
reference_audio: ReferenceAudio | None = None
audio_paths = [] if reference_audio is None else list(_reference_list(reference_audio.path))
if request.audio_reference is not None:
try:
audio_path = await decode_audio_url(request.audio_reference.audio_url)
for audio_reference in _reference_list(request.audio_reference):
audio_paths.append(await decode_audio_url(audio_reference.audio_url))
except InvalidInputReferenceError as exc:
_cleanup_video_references(reference_video, None)
_cleanup_video_references(reference_video, reference_audio)
cleanup_paths = set(() if reference_audio is None else reference_audio.cleanup_paths)
for path in audio_paths:
if path not in cleanup_paths and os.path.exists(path):
os.unlink(path)
raise HTTPException(400, detail=str(exc)) from exc
reference_audio = ReferenceAudio(path=audio_path)
if audio_paths:
cleanup_paths = (
tuple(audio_paths)
if reference_audio is None
else reference_audio.cleanup_paths
+ tuple(path for path in audio_paths if path not in reference_audio.cleanup_paths)
)
reference_audio = ReferenceAudio(
path=audio_paths if len(audio_paths) > 1 else audio_paths[0],
cleanup_paths=cleanup_paths,
)
return request, handler, effective_model_name, reference_image, reference_video, reference_audio