feat(comfyui): add MiniMax H3 first and last frame inputs (#7449)
Signed-off-by: avraichur96 <avraichur96@gmail.com> Signed-off-by: Abhinav Raichur <avraichur96@gmail.com>
This commit is contained in:
1 parent
ae3880f0e3
commit
1b87115c9e
5 files changed
+476
-8
No files matched your search
@@ -134,11 +134,13 @@ You can configure per-stage sampling parameters for multi-stage models.
|
||||
</p>
|
||||
|
||||
> [!TIP]
|
||||
> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
|
||||
> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
|
||||
> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
|
||||
>
|
||||
> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
|
||||
>
|
||||
> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
|
||||
> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
|
||||
> `references`. Task routing is automatic from which inputs you connect.
|
||||
|
||||
For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
|
||||
3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
|
||||
|
||||
@@ -203,6 +203,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
},
|
||||
"optional": {
|
||||
"frame": ("IMAGE",),
|
||||
"first_frame": ("IMAGE",),
|
||||
"last_frame": ("IMAGE",),
|
||||
"references": ("VIDEO_REFERENCES",),
|
||||
"sampling_params": ("SAMPLING_PARAMS",),
|
||||
"lora": ("REMOTE_LORA",),
|
||||
@@ -216,12 +218,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
FUNCTION = "generate"
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
|
||||
def VALIDATE_INPUTS(
|
||||
cls,
|
||||
url,
|
||||
model,
|
||||
frame=None,
|
||||
first_frame=None,
|
||||
last_frame=None,
|
||||
references=None,
|
||||
fast_h3=None,
|
||||
**_kwargs,
|
||||
) -> str | Literal[True]:
|
||||
base = super().VALIDATE_INPUTS(url, model)
|
||||
if base is not True:
|
||||
return base
|
||||
if frame is not None and references is not None:
|
||||
return "Provide only one of frame or references, not both."
|
||||
if frame is not None and (first_frame is not None or last_frame is not None):
|
||||
return "Provide either frame or first_frame/last_frame, not both."
|
||||
if references is not None and (first_frame is not None or last_frame is not None):
|
||||
return "Provide either first_frame/last_frame or references, not both."
|
||||
if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
|
||||
return (
|
||||
"FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
|
||||
)
|
||||
return True
|
||||
|
||||
async def generate(
|
||||
@@ -235,6 +255,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
duration: float,
|
||||
negative_prompt: str | None = None,
|
||||
frame: torch.Tensor | None = None,
|
||||
first_frame: torch.Tensor | None = None,
|
||||
last_frame: torch.Tensor | None = None,
|
||||
references: dict | None = None,
|
||||
sampling_params: dict | list[dict] | None = None,
|
||||
model_params: dict | None = None,
|
||||
@@ -252,8 +274,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
spec_model = model
|
||||
|
||||
if fast_h3 is not None:
|
||||
if frame is not None or references is not None:
|
||||
raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.")
|
||||
if any(value is not None for value in (frame, first_frame, last_frame, references)):
|
||||
raise ValueError(
|
||||
"FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
|
||||
"last_frame, and references inputs."
|
||||
)
|
||||
if lora is not None:
|
||||
raise ValueError(
|
||||
"FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
|
||||
@@ -314,6 +339,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
spec_model=spec_model,
|
||||
prompt=prompt,
|
||||
frame=frame, # frame present => fl2va / Wan I2V
|
||||
first_frame=first_frame,
|
||||
last_frame=last_frame,
|
||||
references=references,
|
||||
width=width,
|
||||
height=height,
|
||||
|
||||
@@ -277,6 +277,8 @@ class VLLMOmniClient:
|
||||
fps: int,
|
||||
negative_prompt: str | None = None,
|
||||
frame: torch.Tensor | None = None,
|
||||
first_frame: torch.Tensor | None = None,
|
||||
last_frame: torch.Tensor | None = None,
|
||||
references: dict | None = None,
|
||||
sampling_params: dict | None = None,
|
||||
model_params: dict | None = None,
|
||||
@@ -292,6 +294,16 @@ class VLLMOmniClient:
|
||||
"""
|
||||
if frame is not None and references is not None:
|
||||
raise ValueError("Provide only one of frame or references, not both.")
|
||||
if frame is not None and (first_frame is not None or last_frame is not None):
|
||||
raise ValueError("Provide either frame or first_frame/last_frame, not both.")
|
||||
if references is not None and (first_frame is not None or last_frame is not None):
|
||||
raise ValueError("Provide either first_frame/last_frame or references, not both.")
|
||||
|
||||
spec, matched_pattern = lookup_model_spec(spec_model or model)
|
||||
if (first_frame is not None or last_frame is not None) and (
|
||||
matched_pattern is None or "MiniMax-H3" not in matched_pattern
|
||||
):
|
||||
raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
|
||||
|
||||
# === regular payload fields ===
|
||||
form = aiohttp.FormData()
|
||||
@@ -311,11 +323,24 @@ class VLLMOmniClient:
|
||||
|
||||
# === multimodal inputs (first-last-frames, references, etc.) ===
|
||||
input_reference_image: torch.Tensor | None = None
|
||||
keyframe_images: list[tuple[str, torch.Tensor]] = []
|
||||
video_task: str | None = None
|
||||
|
||||
if frame is not None:
|
||||
input_reference_image = frame
|
||||
video_task = "fl2va"
|
||||
elif first_frame is not None or last_frame is not None:
|
||||
frame_indices: list[int] = []
|
||||
if first_frame is not None:
|
||||
keyframe_images.append(("first_frame.png", first_frame))
|
||||
frame_indices.append(0)
|
||||
if last_frame is not None:
|
||||
keyframe_images.append(("last_frame.png", last_frame))
|
||||
frame_indices.append(-1)
|
||||
if len(keyframe_images) == 1:
|
||||
input_reference_image = keyframe_images[0][1]
|
||||
extra_params["frame_indices"] = frame_indices
|
||||
video_task = "fl2va"
|
||||
elif references is not None:
|
||||
reference_formats = (
|
||||
("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
|
||||
@@ -353,7 +378,7 @@ class VLLMOmniClient:
|
||||
video_task = "t2va"
|
||||
|
||||
if input_reference_image is not None:
|
||||
image_filename = "image.png" # Required for multipart form
|
||||
image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
|
||||
form.add_field(
|
||||
"input_reference",
|
||||
image_tensor_to_png_bytes(input_reference_image, image_filename),
|
||||
@@ -361,12 +386,20 @@ class VLLMOmniClient:
|
||||
content_type="image/png",
|
||||
)
|
||||
|
||||
if len(keyframe_images) == 2:
|
||||
for image_filename, image in keyframe_images:
|
||||
form.add_field(
|
||||
"input_references",
|
||||
image_tensor_to_png_bytes(image, image_filename),
|
||||
filename=image_filename,
|
||||
content_type="image/png",
|
||||
)
|
||||
|
||||
# === model specific params. Either use a specialized builder, or add flattened fields as-is ===
|
||||
if model_params is not None:
|
||||
model_params = dict(model_params)
|
||||
model_params.pop("type", None)
|
||||
|
||||
spec, _ = lookup_model_spec(spec_model or model)
|
||||
params_builder = spec.get("params_builder") if spec else None
|
||||
if params_builder is not None:
|
||||
form_fields = params_builder(
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
{
|
||||
"id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
|
||||
"revision": 0,
|
||||
"last_node_id": 9,
|
||||
"last_link_id": 4,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "LoadImage",
|
||||
"pos": [40, 20],
|
||||
"size": [270, 314],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
|
||||
{"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
|
||||
{"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
|
||||
],
|
||||
"title": "First Frame",
|
||||
"properties": {"Node name for S&R": "LoadImage"},
|
||||
"widgets_values": ["first_frame.png", "image"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "LoadImage",
|
||||
"pos": [40, 390],
|
||||
"size": [270, 314],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
|
||||
{"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
|
||||
{"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
|
||||
],
|
||||
"title": "Last Frame (optional)",
|
||||
"properties": {"Node name for S&R": "LoadImage"},
|
||||
"widgets_values": ["last_frame.png", "image"]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [350, 20],
|
||||
"size": [390, 240],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "MiniMax H3 I2V modes",
|
||||
"properties": {},
|
||||
"widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
|
||||
"color": "#432",
|
||||
"bgcolor": "#000"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "VLLMOmniDiffusionSampling",
|
||||
"pos": [350, 300],
|
||||
"size": [300, 226],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
|
||||
{"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
|
||||
{"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
|
||||
{"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
|
||||
{"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
|
||||
{"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
|
||||
{"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
|
||||
],
|
||||
"title": "Base H3 Sampling (50 steps)",
|
||||
"properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
|
||||
"widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "VLLMOmniMiniMaxH3Params",
|
||||
"pos": [350, 570],
|
||||
"size": [300, 82],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
|
||||
{"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
|
||||
],
|
||||
"title": "Base H3 Flow Shifts",
|
||||
"properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
|
||||
"widgets_values": [3, 12]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "VLLMOmniRemoteLoRA",
|
||||
"pos": [700, 550],
|
||||
"size": [360, 150],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
|
||||
{"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
|
||||
{"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
|
||||
{"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
|
||||
],
|
||||
"title": "Optional H3 Turbo Remote LoRA",
|
||||
"properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
|
||||
"widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [700, 300],
|
||||
"size": [420, 210],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Optional Turbo settings",
|
||||
"properties": {},
|
||||
"widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
|
||||
"color": "#432",
|
||||
"bgcolor": "#000"
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "VLLMOmniGenerateVideo",
|
||||
"pos": [1160, 80],
|
||||
"size": [440, 610],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
|
||||
{"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
|
||||
{"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
|
||||
{"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
|
||||
{"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
|
||||
{"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
|
||||
{"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
|
||||
{"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
|
||||
{"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
|
||||
{"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
|
||||
{"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
|
||||
{"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
|
||||
{"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
|
||||
{"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
|
||||
{"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
|
||||
{"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
|
||||
],
|
||||
"properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
|
||||
"widgets_values": [
|
||||
"http://localhost:8000/v1",
|
||||
"MiniMaxAI/MiniMax-H3",
|
||||
"A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
|
||||
"",
|
||||
1344,
|
||||
768,
|
||||
24,
|
||||
5.167
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "SaveVideo",
|
||||
"pos": [1650, 170],
|
||||
"size": [374, 314],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
|
||||
{"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
|
||||
{"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
|
||||
{"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
|
||||
],
|
||||
"outputs": [
|
||||
{"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
|
||||
],
|
||||
"properties": {},
|
||||
"widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 8, 1, "IMAGE"],
|
||||
[2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
|
||||
[3, 5, 0, 8, 6, "VIDEO_PARAMS"],
|
||||
[4, 8, 0, 9, 0, "VIDEO"]
|
||||
],
|
||||
"groups": [
|
||||
{"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
|
||||
{"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
|
||||
{"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
|
||||
],
|
||||
"config": {},
|
||||
"extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -84,6 +84,7 @@ class SamplingKind(str, Enum):
|
||||
MUSIC_NONE = "music_none"
|
||||
VIDEO_NONE = "video_none"
|
||||
VIDEO_DIFFUSION_SINGLE = "video_diffusion_single"
|
||||
VIDEO_FL2VA = "video_fl2va"
|
||||
VIDEO_FASTH3 = "video_fasth3"
|
||||
VIDEO_REF2VA_IMAGE_AUDIO = "video_ref2va_image_audio"
|
||||
VIDEO_REF2VA_MULTI_VIDEO = "video_ref2va_multi_video"
|
||||
@@ -150,7 +151,7 @@ AR_LIST_SAMPLING_PARAMS = [
|
||||
|
||||
VIDEO_MODEL_PARAMS = WanModelSpecificParams({"guidance_scale_2": 5.0, "boundary_ratio": 0.98, "flow_shift": 12.0})
|
||||
|
||||
# Matches MiniMax-H3 recipe Ref2VA defaults (task is auto-routed by the client).
|
||||
# Matches MiniMax-H3 recipe defaults (task is auto-routed by the client).
|
||||
H3_MODEL_PARAMS = MiniMaxH3ModelSpecificParams(
|
||||
{
|
||||
"type": "minimax_h3",
|
||||
@@ -417,6 +418,34 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam
|
||||
LORA_PARAMS,
|
||||
)
|
||||
_assert_model_param_values(received_sampling_params_list[0], VIDEO_MODEL_PARAMS)
|
||||
elif sampling_case.kind is SamplingKind.VIDEO_FL2VA:
|
||||
assert len(received_sampling_params_list) == 1
|
||||
assert isinstance(prompt, dict)
|
||||
multi_modal_data = prompt.get("multi_modal_data")
|
||||
assert isinstance(multi_modal_data, dict)
|
||||
input_images = multi_modal_data.get("image")
|
||||
|
||||
if isinstance(input_images, list):
|
||||
assert len(input_images) == 2
|
||||
assert input_images[0].getpixel((0, 0)) == (0, 0, 0)
|
||||
assert input_images[1].getpixel((0, 0)) == (255, 255, 255)
|
||||
expected_frame_indices = [0, -1]
|
||||
else:
|
||||
assert isinstance(input_images, Image.Image)
|
||||
pixel = input_images.getpixel((0, 0))
|
||||
assert pixel in {(0, 0, 0), (255, 255, 255)}
|
||||
is_first_frame = pixel == (0, 0, 0)
|
||||
expected_frame_indices = [0] if is_first_frame else [-1]
|
||||
|
||||
_assert_model_param_values(
|
||||
received_sampling_params_list[0],
|
||||
{
|
||||
"flow_shift": 12.0,
|
||||
"task": "fl2va",
|
||||
"audio_flow_shift": 3.0,
|
||||
"frame_indices": expected_frame_indices,
|
||||
},
|
||||
)
|
||||
elif sampling_case.kind is SamplingKind.VIDEO_FASTH3:
|
||||
assert len(received_sampling_params_list) == 1
|
||||
received = received_sampling_params_list[0]
|
||||
@@ -937,6 +966,164 @@ async def test_video_generation_node(api_server: str, model: str, image_input: b
|
||||
assert isinstance(result[0], VideoInput)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server_case",
|
||||
[
|
||||
pytest.param(
|
||||
ServerCase(
|
||||
served_model="MiniMaxAI/MiniMax-H3",
|
||||
stage_list=["diffusion"],
|
||||
stage_configs=[H3_STAGE_CONFIG],
|
||||
outputs=[_build_diffusion_video_output()],
|
||||
),
|
||||
id="minimax-h3",
|
||||
),
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"sampling_case",
|
||||
[pytest.param(SamplingCase(kind=SamplingKind.VIDEO_FL2VA, sampling_params=None), id="fl2va")],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"frame_selection",
|
||||
[
|
||||
pytest.param("first", id="first-frame"),
|
||||
pytest.param("last", id="last-frame"),
|
||||
pytest.param("both", id="first-and-last-frame"),
|
||||
],
|
||||
)
|
||||
async def test_video_generation_node_minimax_h3_fl2va(
|
||||
api_server: str,
|
||||
sampling_case: SamplingCase,
|
||||
frame_selection: str,
|
||||
):
|
||||
node = VLLMOmniGenerateVideo()
|
||||
first_frame = torch.zeros((1, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
|
||||
last_frame = torch.ones((1, VIDEO_HEIGHT, VIDEO_WIDTH, 3), dtype=torch.float32)
|
||||
frame_inputs = {
|
||||
"first": {"first_frame": first_frame},
|
||||
"last": {"last_frame": last_frame},
|
||||
"both": {"first_frame": first_frame, "last_frame": last_frame},
|
||||
}[frame_selection]
|
||||
|
||||
result = await node.generate(
|
||||
url=api_server,
|
||||
model="MiniMaxAI/MiniMax-H3",
|
||||
prompt="A cinematic transition between the supplied keyframes.",
|
||||
negative_prompt="",
|
||||
width=VIDEO_WIDTH,
|
||||
height=VIDEO_HEIGHT,
|
||||
fps=VIDEO_FPS,
|
||||
duration=VIDEO_DURATION,
|
||||
model_params=H3_MODEL_PARAMS,
|
||||
**frame_inputs,
|
||||
)
|
||||
|
||||
assert isinstance(result, tuple)
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0], VideoInput)
|
||||
|
||||
|
||||
def test_video_generation_node_exposes_explicit_keyframe_inputs():
|
||||
node_inputs = VLLMOmniGenerateVideo.INPUT_TYPES()
|
||||
optional_inputs = node_inputs["optional"]
|
||||
|
||||
assert optional_inputs["first_frame"] == ("IMAGE",)
|
||||
assert optional_inputs["last_frame"] == ("IMAGE",)
|
||||
assert node_inputs["required"]["duration"][0] == "FLOAT"
|
||||
assert "num_frames" not in node_inputs["required"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "message"),
|
||||
[
|
||||
({"frame": object(), "first_frame": object()}, "frame or first_frame/last_frame"),
|
||||
({"frame": object(), "last_frame": object()}, "frame or first_frame/last_frame"),
|
||||
({"last_frame": object(), "references": {}}, "first_frame/last_frame or references"),
|
||||
],
|
||||
)
|
||||
async def test_video_generation_node_rejects_conflicting_frame_inputs(kwargs: dict, message: str):
|
||||
node = VLLMOmniGenerateVideo()
|
||||
validation_result = node.VALIDATE_INPUTS(
|
||||
url="http://localhost:8000/v1",
|
||||
model="MiniMaxAI/MiniMax-H3",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
assert isinstance(validation_result, str)
|
||||
assert message in validation_result
|
||||
|
||||
with pytest.raises(ValueError, match=message):
|
||||
await node.generate(
|
||||
url="http://localhost:8000/v1",
|
||||
model="MiniMaxAI/MiniMax-H3",
|
||||
prompt="test",
|
||||
width=VIDEO_WIDTH,
|
||||
height=VIDEO_HEIGHT,
|
||||
fps=VIDEO_FPS,
|
||||
duration=VIDEO_DURATION,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("keyframe_name", ["first_frame", "last_frame"])
|
||||
async def test_video_generation_node_rejects_keyframes_with_fast_h3(keyframe_name: str):
|
||||
node = VLLMOmniGenerateVideo()
|
||||
fast_h3 = {"url": "http://localhost:8000/v1", "model": "MiniMaxAI/MiniMax-H3"}
|
||||
keyframe = {keyframe_name: object()}
|
||||
|
||||
validation_result = node.VALIDATE_INPUTS(
|
||||
url="http://ignored.invalid/v1",
|
||||
model="ignored-model",
|
||||
fast_h3=fast_h3,
|
||||
**keyframe,
|
||||
)
|
||||
|
||||
assert isinstance(validation_result, str)
|
||||
assert "FastH3 Preview supports T2VA only" in validation_result
|
||||
|
||||
with pytest.raises(ValueError, match="FastH3 Preview supports T2VA only"):
|
||||
await node.generate(
|
||||
url="http://ignored.invalid/v1",
|
||||
model="ignored-model",
|
||||
prompt="test",
|
||||
width=VIDEO_WIDTH,
|
||||
height=VIDEO_HEIGHT,
|
||||
fps=VIDEO_FPS,
|
||||
duration=VIDEO_DURATION,
|
||||
fast_h3=fast_h3,
|
||||
**keyframe,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"keyframes",
|
||||
[
|
||||
{"first_frame": object()},
|
||||
{"last_frame": object()},
|
||||
{"first_frame": object(), "last_frame": object()},
|
||||
],
|
||||
)
|
||||
async def test_video_generation_node_rejects_h3_keyframes_for_other_models(keyframes: dict):
|
||||
with pytest.raises(ValueError, match="supported only for MiniMax-H3; use frame"):
|
||||
await VLLMOmniGenerateVideo().generate(
|
||||
url="http://localhost:8000/v1",
|
||||
model="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
prompt="test",
|
||||
width=VIDEO_WIDTH,
|
||||
height=VIDEO_HEIGHT,
|
||||
fps=VIDEO_FPS,
|
||||
duration=VIDEO_DURATION,
|
||||
**keyframes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"server_case,deployment_model",
|
||||
|
||||
Reference in new issue
Block a user