feat(comfyui): add MiniMax H3 first and last frame inputs (#7449)

Signed-off-by: avraichur96 <avraichur96@gmail.com>
Signed-off-by: Abhinav Raichur  <avraichur96@gmail.com>
This commit is contained in:
Abhinav Raichur authored and GitHub committed 2026-09-21 10:24:25 +08:00
1 parent ae3880f0e3
commit 1b87115c9e
5 files changed
+476 -8

No files matched your search

+4 -2
View File
@@ -134,11 +134,13 @@ You can configure per-stage sampling parameters for multi-stage models.
</p>
> [!TIP]
> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
> Connect **frame** for backward-compatible first-frame / image-to-video behavior. For MiniMax-H3 FL2VA,
> connect **first_frame**, **last_frame**, or both to condition the start frame, end frame, or both.
>
> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
>
> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
> Do not combine `frame` with `first_frame` or `last_frame`, and do not combine any frame input with
> `references`. Task routing is automatic from which inputs you connect.
For MiniMax-H3 Ref2VA, **Video References** accepts up to 9 images (`image_1`–`image_9`),
3 videos (`video_1`–`video_3`), and 3 audio clips (`audio_1`–`audio_3`), with at most
@@ -203,6 +203,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
},
"optional": {
"frame": ("IMAGE",),
"first_frame": ("IMAGE",),
"last_frame": ("IMAGE",),
"references": ("VIDEO_REFERENCES",),
"sampling_params": ("SAMPLING_PARAMS",),
"lora": ("REMOTE_LORA",),
@@ -216,12 +218,30 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
FUNCTION = "generate"
@classmethod
def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
def VALIDATE_INPUTS(
cls,
url,
model,
frame=None,
first_frame=None,
last_frame=None,
references=None,
fast_h3=None,
**_kwargs,
) -> str | Literal[True]:
base = super().VALIDATE_INPUTS(url, model)
if base is not True:
return base
if frame is not None and references is not None:
return "Provide only one of frame or references, not both."
if frame is not None and (first_frame is not None or last_frame is not None):
return "Provide either frame or first_frame/last_frame, not both."
if references is not None and (first_frame is not None or last_frame is not None):
return "Provide either first_frame/last_frame or references, not both."
if fast_h3 is not None and any(value is not None for value in (frame, first_frame, last_frame, references)):
return (
"FastH3 Preview supports T2VA only; disconnect frame, first_frame, last_frame, and references inputs."
)
return True
async def generate(
@@ -235,6 +255,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
duration: float,
negative_prompt: str | None = None,
frame: torch.Tensor | None = None,
first_frame: torch.Tensor | None = None,
last_frame: torch.Tensor | None = None,
references: dict | None = None,
sampling_params: dict | list[dict] | None = None,
model_params: dict | None = None,
@@ -252,8 +274,11 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
spec_model = model
if fast_h3 is not None:
if frame is not None or references is not None:
raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.")
if any(value is not None for value in (frame, first_frame, last_frame, references)):
raise ValueError(
"FastH3 Preview supports T2VA only; disconnect frame, first_frame, "
"last_frame, and references inputs."
)
if lora is not None:
raise ValueError(
"FastH3 is already fused into the selected server; disconnect the request-level LoRA input."
@@ -314,6 +339,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
spec_model=spec_model,
prompt=prompt,
frame=frame, # frame present => fl2va / Wan I2V
first_frame=first_frame,
last_frame=last_frame,
references=references,
width=width,
height=height,
@@ -277,6 +277,8 @@ class VLLMOmniClient:
fps: int,
negative_prompt: str | None = None,
frame: torch.Tensor | None = None,
first_frame: torch.Tensor | None = None,
last_frame: torch.Tensor | None = None,
references: dict | None = None,
sampling_params: dict | None = None,
model_params: dict | None = None,
@@ -292,6 +294,16 @@ class VLLMOmniClient:
"""
if frame is not None and references is not None:
raise ValueError("Provide only one of frame or references, not both.")
if frame is not None and (first_frame is not None or last_frame is not None):
raise ValueError("Provide either frame or first_frame/last_frame, not both.")
if references is not None and (first_frame is not None or last_frame is not None):
raise ValueError("Provide either first_frame/last_frame or references, not both.")
spec, matched_pattern = lookup_model_spec(spec_model or model)
if (first_frame is not None or last_frame is not None) and (
matched_pattern is None or "MiniMax-H3" not in matched_pattern
):
raise ValueError("first_frame and last_frame are supported only for MiniMax-H3; use frame for this model.")
# === regular payload fields ===
form = aiohttp.FormData()
@@ -311,11 +323,24 @@ class VLLMOmniClient:
# === multimodal inputs (first-last-frames, references, etc.) ===
input_reference_image: torch.Tensor | None = None
keyframe_images: list[tuple[str, torch.Tensor]] = []
video_task: str | None = None
if frame is not None:
input_reference_image = frame
video_task = "fl2va"
elif first_frame is not None or last_frame is not None:
frame_indices: list[int] = []
if first_frame is not None:
keyframe_images.append(("first_frame.png", first_frame))
frame_indices.append(0)
if last_frame is not None:
keyframe_images.append(("last_frame.png", last_frame))
frame_indices.append(-1)
if len(keyframe_images) == 1:
input_reference_image = keyframe_images[0][1]
extra_params["frame_indices"] = frame_indices
video_task = "fl2va"
elif references is not None:
reference_formats = (
("image", MAX_REFERENCE_IMAGES, "png", "image/png", image_tensor_to_png_bytes),
@@ -353,7 +378,7 @@ class VLLMOmniClient:
video_task = "t2va"
if input_reference_image is not None:
image_filename = "image.png" # Required for multipart form
image_filename = keyframe_images[0][0] if keyframe_images else "image.png"
form.add_field(
"input_reference",
image_tensor_to_png_bytes(input_reference_image, image_filename),
@@ -361,12 +386,20 @@ class VLLMOmniClient:
content_type="image/png",
)
if len(keyframe_images) == 2:
for image_filename, image in keyframe_images:
form.add_field(
"input_references",
image_tensor_to_png_bytes(image, image_filename),
filename=image_filename,
content_type="image/png",
)
# === model specific params. Either use a specialized builder, or add flattened fields as-is ===
if model_params is not None:
model_params = dict(model_params)
model_params.pop("type", None)
spec, _ = lookup_model_spec(spec_model or model)
params_builder = spec.get("params_builder") if spec else None
if params_builder is not None:
form_fields = params_builder(
@@ -0,0 +1,219 @@
{
"id": "6676f19a-7f41-4c80-8cb7-f80adcb4ac44",
"revision": 0,
"last_node_id": 9,
"last_link_id": 4,
"nodes": [
{
"id": 1,
"type": "LoadImage",
"pos": [40, 20],
"size": [270, 314],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [
{"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
{"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
],
"outputs": [
{"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": [1]},
{"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
],
"title": "First Frame",
"properties": {"Node name for S&R": "LoadImage"},
"widgets_values": ["first_frame.png", "image"]
},
{
"id": 2,
"type": "LoadImage",
"pos": [40, 390],
"size": [270, 314],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{"localized_name": "image", "name": "image", "type": "COMBO", "widget": {"name": "image"}, "link": null},
{"localized_name": "choose file to upload", "name": "upload", "type": "IMAGEUPLOAD", "widget": {"name": "upload"}, "link": null}
],
"outputs": [
{"localized_name": "IMAGE", "name": "IMAGE", "type": "IMAGE", "links": null},
{"localized_name": "MASK", "name": "MASK", "type": "MASK", "links": null}
],
"title": "Last Frame (optional)",
"properties": {"Node name for S&R": "LoadImage"},
"widgets_values": ["last_frame.png", "image"]
},
{
"id": 3,
"type": "MarkdownNote",
"pos": [350, 20],
"size": [390, 240],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "MiniMax H3 I2V modes",
"properties": {},
"widgets_values": ["This workflow uses the remote vLLM-Omni Generate Video node.\n\n- First-frame only (default): keep First Frame connected.\n- Last-frame only: disconnect First Frame and connect Last Frame to last_frame.\n- First+last-frame: connect both image loaders.\n\nLeave the legacy frame and references inputs disconnected. Select images from ComfyUI's input folder. H3 runs at 24 FPS; the 5.167-second duration resolves to 124 frames, satisfying 17k+5."],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 4,
"type": "VLLMOmniDiffusionSampling",
"pos": [350, 300],
"size": [300, 226],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{"localized_name": "n", "name": "n", "type": "INT", "widget": {"name": "n"}, "link": null},
{"localized_name": "num_inference_steps", "name": "num_inference_steps", "type": "INT", "widget": {"name": "num_inference_steps"}, "link": null},
{"localized_name": "guidance_scale", "name": "guidance_scale", "type": "FLOAT", "widget": {"name": "guidance_scale"}, "link": null},
{"localized_name": "true_cfg_scale", "name": "true_cfg_scale", "type": "FLOAT", "widget": {"name": "true_cfg_scale"}, "link": null},
{"localized_name": "vae_use_slicing", "name": "vae_use_slicing", "type": "BOOLEAN", "widget": {"name": "vae_use_slicing"}, "link": null},
{"localized_name": "vae_use_tiling", "name": "vae_use_tiling", "type": "BOOLEAN", "widget": {"name": "vae_use_tiling"}, "link": null},
{"localized_name": "seed", "name": "seed", "type": "INT", "widget": {"name": "seed"}, "link": null}
],
"outputs": [
{"localized_name": "diffusion sampling params", "name": "diffusion sampling params", "type": "SAMPLING_PARAMS", "links": [2]}
],
"title": "Base H3 Sampling (50 steps)",
"properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"},
"widgets_values": [1, 50, 1, 1, false, false, 757358688076805, "fixed"]
},
{
"id": 5,
"type": "VLLMOmniMiniMaxH3Params",
"pos": [350, 570],
"size": [300, 82],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{"localized_name": "audio_flow_shift", "name": "audio_flow_shift", "type": "FLOAT", "widget": {"name": "audio_flow_shift"}, "link": null},
{"localized_name": "flow_shift", "name": "flow_shift", "type": "FLOAT", "widget": {"name": "flow_shift"}, "link": null}
],
"outputs": [
{"localized_name": "MiniMax-H3 video params", "name": "MiniMax-H3 video params", "type": "VIDEO_PARAMS", "links": [3]}
],
"title": "Base H3 Flow Shifts",
"properties": {"Node name for S&R": "VLLMOmniMiniMaxH3Params"},
"widgets_values": [3, 12]
},
{
"id": 6,
"type": "VLLMOmniRemoteLoRA",
"pos": [700, 550],
"size": [360, 150],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{"localized_name": "local_path", "name": "local_path", "type": "STRING", "widget": {"name": "local_path"}, "link": null},
{"localized_name": "name", "name": "name", "type": "STRING", "widget": {"name": "name"}, "link": null},
{"localized_name": "scale", "name": "scale", "type": "FLOAT", "widget": {"name": "scale"}, "link": null},
{"localized_name": "int_id", "name": "int_id", "type": "INT", "widget": {"name": "int_id"}, "link": null}
],
"outputs": [
{"localized_name": "lora", "name": "lora", "type": "REMOTE_LORA", "links": null}
],
"title": "Optional H3 Turbo Remote LoRA",
"properties": {"Node name for S&R": "VLLMOmniRemoteLoRA"},
"widgets_values": ["minimax_h3_fl2v_turbo_8step_v1.0_768p_bf16.safetensors", "h3-turbo-8step", 1, 0]
},
{
"id": 7,
"type": "MarkdownNote",
"pos": [700, 300],
"size": [420, 210],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "Optional Turbo settings",
"properties": {},
"widgets_values": ["Base H3 is configured by default: 50 steps, flow_shift 12, audio_flow_shift 3.\n\nFor the shown 8-step Diffusers Turbo artifact, set its full local_path on the server, start the server with the matching preloaded LoRA, connect this Remote LoRA to Generate Video, set num_inference_steps to 9, and set flow_shift to 6. Keep audio_flow_shift at 3 and scale at 1.\n\nDo not use the _comfyui_ repack or FastH3 here."],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 8,
"type": "VLLMOmniGenerateVideo",
"pos": [1160, 80],
"size": [440, 610],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{"localized_name": "frame", "name": "frame", "shape": 7, "type": "IMAGE", "link": null},
{"localized_name": "first_frame", "name": "first_frame", "shape": 7, "type": "IMAGE", "link": 1},
{"localized_name": "last_frame", "name": "last_frame", "shape": 7, "type": "IMAGE", "link": null},
{"localized_name": "references", "name": "references", "shape": 7, "type": "VIDEO_REFERENCES", "link": null},
{"localized_name": "sampling_params", "name": "sampling_params", "shape": 7, "type": "SAMPLING_PARAMS", "link": 2},
{"localized_name": "lora", "name": "lora", "shape": 7, "type": "REMOTE_LORA", "link": null},
{"localized_name": "model_params", "name": "model_params", "shape": 7, "type": "VIDEO_PARAMS", "link": 3},
{"localized_name": "fast_h3", "name": "fast_h3", "shape": 7, "type": "FASTH3_DEPLOYMENT", "link": null},
{"localized_name": "url", "name": "url", "type": "STRING", "widget": {"name": "url"}, "link": null},
{"localized_name": "model", "name": "model", "type": "STRING", "widget": {"name": "model"}, "link": null},
{"localized_name": "prompt", "name": "prompt", "type": "STRING", "widget": {"name": "prompt"}, "link": null},
{"localized_name": "negative_prompt", "name": "negative_prompt", "type": "STRING", "widget": {"name": "negative_prompt"}, "link": null},
{"localized_name": "width", "name": "width", "type": "INT", "widget": {"name": "width"}, "link": null},
{"localized_name": "height", "name": "height", "type": "INT", "widget": {"name": "height"}, "link": null},
{"localized_name": "fps", "name": "fps", "type": "INT", "widget": {"name": "fps"}, "link": null},
{"localized_name": "duration", "name": "duration", "type": "FLOAT", "widget": {"name": "duration"}, "link": null}
],
"outputs": [
{"localized_name": "video", "name": "video", "type": "VIDEO", "links": [4]}
],
"properties": {"Node name for S&R": "VLLMOmniGenerateVideo"},
"widgets_values": [
"http://localhost:8000/v1",
"MiniMaxAI/MiniMax-H3",
"A cinematic product shot begins exactly from the supplied keyframe. The subject moves naturally through a dark studio while blue and amber rim lights shift across reflective surfaces. The camera makes a slow controlled orbit. Audio: subtle room tone, precise mechanical details, and a restrained electronic score.",
"",
1344,
768,
24,
5.167
]
},
{
"id": 9,
"type": "SaveVideo",
"pos": [1650, 170],
"size": [374, 314],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{"localized_name": "video", "name": "video", "type": "VIDEO", "link": 4},
{"localized_name": "filename_prefix", "name": "filename_prefix", "type": "STRING", "widget": {"name": "filename_prefix"}, "link": null},
{"localized_name": "format", "name": "format", "type": "COMBO", "widget": {"name": "format"}, "link": null},
{"localized_name": "codec", "name": "codec", "type": "COMFY_DYNAMICCOMBO_V3", "widget": {"name": "codec"}, "link": null}
],
"outputs": [
{"localized_name": "video", "name": "video", "type": "VIDEO", "links": null}
],
"properties": {},
"widgets_values": ["video/MiniMax_H3_I2V", "auto", "auto"]
}
],
"links": [
[1, 1, 0, 8, 1, "IMAGE"],
[2, 4, 0, 8, 4, "SAMPLING_PARAMS"],
[3, 5, 0, 8, 6, "VIDEO_PARAMS"],
[4, 8, 0, 9, 0, "VIDEO"]
],
"groups": [
{"id": 1, "title": "Portable first/last-frame inputs", "bounding": [10, -30, 760, 770], "color": "#3f789e", "flags": {}},
{"id": 2, "title": "H3 sampling and optional Turbo", "bounding": [320, 270, 830, 460], "color": "#8a6d3b", "flags": {}},
{"id": 3, "title": "Remote generation and audio-preserving save", "bounding": [1130, 30, 930, 700], "color": "#3f789e", "flags": {}}
],
"config": {},
"extra": {"workflowRendererVersion": "LG", "ds": {"scale": 0.75, "offset": [180, 120]}},
"version": 0.4
}