anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Frontend] MiniMax-H3: Add latent-mask editing to the ComfyUI extension (#7575)

Browse Source 
Signed-off-by: chen hongwei <1792043268@qq.com>
Signed-off-by: Wang Zhipeng <wangzhipeng628@gmail.com>
Signed-off-by: princepride <wangzhipeng628@gmail.com>
Co-authored-by: Wang Zhipeng <wangzhipeng628@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
chen hongweiWang ZhipengClaude Opus 5.5
 authored and GitHub committed 2026-10-06 16:32:40 +00:00
1 parent 7be49aa0ce
commit bad88bca25
15 files changed
+325 -1878
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/ComfyUI-vLLM-Omni/README.md
+2 -2
View File
@@ -271,10 +271,10 @@ The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20
Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
- `source_video` / `source_audio` — the media to edit.
- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. The client area-downsamples it by 16 to keep the upload small, and the server resizes it to the video latent grid.
- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
The node requires at least one mask; the server then enforces the cross-field rules (a source requires a matching mask, and a non-trivial mask requires its source). This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
### TTS (e.g., Qwen TTS series)
apps/ComfyUI-vLLM-Omni/__init__.py
+2 -2
View File
@@ -36,7 +36,6 @@ from .comfyui_vllm_omni.nodes import (
# A dictionary that contains all nodes you want to export with their names
NODE_CLASS_MAPPINGS = {
"VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
# === Generation ===
"VLLMOmniGenerateImage": VLLMOmniGenerateImage,
"VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
@@ -46,6 +45,7 @@ NODE_CLASS_MAPPINGS = {
"VLLMOmniVoiceClone": VLLMOmniVoiceClone,
"VLLMOmniVideoReferences": VLLMOmniVideoReferences,
"VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
"VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
# === Params ===
"VLLMOmniARSampling": VLLMOmniARSampling,
"VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -59,7 +59,6 @@ NODE_CLASS_MAPPINGS = {
# A dictionary that contains the friendly/humanly readable titles for the nodes
NODE_DISPLAY_NAME_MAPPINGS = {
"VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
# === Generation ===
"VLLMOmniGenerateImage": "Generate Image",
"VLLMOmniGenerateVideo": "Generate Video",
@@ -69,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"VLLMOmniVoiceClone": "TTS Voice Cloning",
"VLLMOmniVideoReferences": "Video References",
"VLLMOmniLatentMaskEditing": "Latent Mask Editing",
"VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
# === Params ===
"VLLMOmniARSampling": "AR Sampling Params",
"VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py
+7 -7
View File
@@ -8,7 +8,7 @@ import torch
from comfy_api.input import AudioInput, VideoInput
from .utils.api_client import VLLMOmniClient
from .utils.latent_mask import _align_frame_count, _video_latent_t
from .utils.latent_mask import _align_frame_count
from .utils.logger import get_logger
from .utils.models import lookup_model_spec
from .utils.types import (
@@ -1133,13 +1133,13 @@ class VLLMOmniMiniMaxH3TemporalMask:
else:
raise ValueError(f"Unknown temporal mask mode: {mode}")
available = min(frames, math.floor(boundary * 24 + 1e-8))
# Snap the preserved prefix to whole VAE clips (5 + 17n frames) so every
# latent it maps to is fully preserved.
prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
preserved = _video_latent_t(prefix) if prefix else 0
total = _video_latent_t(frames)
mask = torch.ones(total, 1, 1)
mask[:preserved] = 0
# One slice per output frame; the server pools frames to the latent grid.
mask = torch.ones(frames, 1, 1)
mask[:prefix] = 0
indices = torch.arange(frames, device=images.device)
source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
preview_images = images.index_select(0, source_indices)
preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
return mask, 24.0, preview_images, preview_mask
return mask, 24.0, preview_images, mask
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py
+14 -19
View File
@@ -30,7 +30,7 @@ from .format import (
video_to_base64,
video_to_bytes,
)
from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
from .latent_mask import scalar_mask_to_json, video_mask_to_json
from .logger import get_logger, pretty_printer
from .models import lookup_model_spec
from .types import (
@@ -388,6 +388,15 @@ class VLLMOmniClient:
content_type="image/png",
)
if len(keyframe_images) == 2:
for image_filename, image in keyframe_images:
form.add_field(
"input_references",
image_tensor_to_png_bytes(image, image_filename),
filename=image_filename,
content_type="image/png",
)
# === latent-mask editing (MiniMax H3) ===
if latent_edit is not None:
source_video = latent_edit.get("source_video")
@@ -398,21 +407,16 @@ class VLLMOmniClient:
if video_mask is None and audio_mask is None:
raise ValueError("Latent-mask editing requires at least one mask.")
video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
audio_mask_trivial = audio_mask is None or audio_mask == 1.0
if not video_mask_trivial and source_video is None:
raise ValueError("A non-trivial video mask requires a source video.")
if not audio_mask_trivial and source_audio is None and source_video is None:
raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
if source_video is not None:
form.add_field(
"source_video",
video_to_bytes(source_video, "source.mp4"),
video_to_bytes(source_video),
filename="source.mp4",
content_type="video/mp4",
)
if source_audio is not None:
# The filename extension selects the audio codec (mp3), so it is
# meaningful even though aiohttp uses the explicit filename below.
form.add_field(
"source_audio",
audio_to_bytes(source_audio, "source_audio.mp3"),
@@ -420,7 +424,7 @@ class VLLMOmniClient:
content_type="audio/mpeg",
)
if video_mask is not None:
mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
mask_json = video_mask_to_json(video_mask)
form.add_field(
"video_noise_mask",
mask_json.encode("utf-8"),
@@ -435,15 +439,6 @@ class VLLMOmniClient:
content_type="application/json",
)
if len(keyframe_images) == 2:
for image_filename, image in keyframe_images:
form.add_field(
"input_references",
image_tensor_to_png_bytes(image, image_filename),
filename=image_filename,
content_type="image/png",
)
# === model specific params. Either use a specialized builder, or add flattened fields as-is ===
if model_params is not None:
model_params = dict(model_params)
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py
+35 -29
View File
@@ -1,11 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Client-side serialization of MiniMax-H3 latent-edit masks.
A video mask is sent in frame space — a 2D spatial ``[H, W]`` (applied to every
frame) or a 3D ``[T, H, W]`` (one slice per frame) — and the server resolves it
to the latent grid. An audio mask is a scalar applied to all time steps.
"""
import json
import math
import torch
import torch.nn.functional as F
# The server rejects mask JSON above 8 MiB, so the spatial axes are
# area-downsampled by the VAE spatial stride before upload. This depends only on
# the stride, not on the H3 shape lattice, which the server still owns.
_VAE_SPATIAL_STRIDE = 16
_MASK_DECIMALS = 4
def scalar_mask_to_json(value: float) -> str:
if not 0.0 <= value <= 1.0:
@@ -13,6 +27,27 @@ def scalar_mask_to_json(value: float) -> str:
return str(value)
def video_mask_to_json(mask: torch.Tensor) -> str:
"""Serialize a frame-space video mask to JSON for the ``video_noise_mask`` field.
``mask`` is a 2D spatial mask or a 3D frame-space mask. Its spatial axes are
area-downsampled by the VAE stride; the server resizes the result to the
latent grid, so the client neither floors the canvas nor maps frames to
latents.
"""
if mask.ndim not in (2, 3):
raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
frames = mask.unsqueeze(0) if mask.ndim == 2 else mask
size = tuple(max(1, math.ceil(dim / _VAE_SPATIAL_STRIDE)) for dim in frames.shape[1:])
frames = F.interpolate(frames.unsqueeze(1).float(), size=size, mode="area").squeeze(1)
if mask.ndim == 2:
frames = frames.squeeze(0)
# Round in float64 so tolist() emits short decimals rather than float32 noise.
return json.dumps(frames.double().round(decimals=_MASK_DECIMALS).tolist(), separators=(",", ":"))
# The temporal-mask node snaps its preserve boundary to the H3 frame lattice;
# this mirrors the server shape planner.
def _align_frame_count(frame_count: int) -> int:
if frame_count <= 0:
return 1
@@ -20,32 +55,3 @@ def _align_frame_count(frame_count: int) -> int:
while current % 17 != 5:
current += 1
return current
def _video_latent_t(frame_count: int) -> int:
if frame_count <= 5:
return 2
return ((int(frame_count) - 5) // 17) * 5 + 2
def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
tv = _video_latent_t(_align_frame_count(num_frames))
height = int(height) // 32 * 32
width = int(width) // 32 * 32
gh, gw = height // 16, width // 16
if mask.ndim == 2:
mask = mask.unsqueeze(0)
elif mask.ndim != 3:
raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
if grid.shape[0] == 1:
grid = grid.expand(tv, gh, gw)
elif grid.shape[0] != tv:
grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
return grid
def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())
apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md
+1 -1
View File
@@ -12,6 +12,6 @@ Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflo
Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
1. Upload your source video in Load Video.
2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position, or replace that chain with Load Image (as Mask) for an arbitrary-shape mask. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.
apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json
-499
View File
@@ -1,499 +0,0 @@
{
"id": "latent-mask-editing-7380",
"revision": 0,
"last_node_id": 7,
"last_link_id": 5,
"nodes": [
{
"id": 1,
"type": "MarkdownNote",
"pos": [
40,
20
],
"size": [
420,
210
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "Latent-mask editing server prerequisite",
"properties": {},
"widgets_values": [
"Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 2,
"type": "LoadVideo",
"pos": [
40,
280
],
"size": [
420,
120
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"localized_name": "file",
"name": "file",
"type": "COMBO",
"widget": {
"name": "file"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": [
1
]
}
],
"properties": {
"Node name for S&R": "LoadVideo"
},
"widgets_values": [
""
]
},
{
"id": 3,
"type": "LoadImageMask",
"pos": [
40,
460
],
"size": [
420,
160
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "image",
"type": "STRING",
"widget": {
"name": "image"
},
"link": null
},
{
"name": "channel",
"type": "STRING",
"widget": {
"name": "channel"
},
"link": null
}
],
"outputs": [
{
"localized_name": "mask",
"name": "mask",
"type": "MASK",
"links": [
2
]
}
],
"properties": {
"Node name for S&R": "LoadImageMask"
},
"widgets_values": [
"",
"red"
]
},
{
"id": 4,
"type": "VLLMOmniLatentMaskEditing",
"pos": [
560,
260
],
"size": [
420,
300
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"localized_name": "source_video",
"name": "source_video",
"type": "VIDEO",
"link": 1,
"shape": 7
},
{
"localized_name": "source_audio",
"name": "source_audio",
"type": "AUDIO",
"link": null
},
{
"localized_name": "video_mask",
"name": "video_mask",
"type": "MASK",
"link": 2,
"shape": 7
},
{
"localized_name": "audio_mask",
"name": "audio_mask",
"type": "FLOAT",
"widget": {
"name": "audio_mask"
},
"link": null
}
],
"outputs": [
{
"localized_name": "latent_edit",
"name": "latent_edit",
"type": "LATENT_MASK_EDITING",
"links": [
3
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniLatentMaskEditing"
},
"widgets_values": [
0.5
]
},
{
"id": 5,
"type": "VLLMOmniMiniMaxH3Params",
"pos": [
40,
680
],
"size": [
420,
140
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"localized_name": "audio_flow_shift",
"name": "audio_flow_shift",
"type": "FLOAT",
"widget": {
"name": "audio_flow_shift"
},
"link": null
},
{
"localized_name": "flow_shift",
"name": "flow_shift",
"type": "FLOAT",
"widget": {
"name": "flow_shift"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video params",
"name": "video params",
"type": "VIDEO_PARAMS",
"links": [
4
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniMiniMaxH3Params"
},
"widgets_values": [
3.0,
12.0
]
},
{
"id": 6,
"type": "VLLMOmniGenerateVideo",
"pos": [
1080,
200
],
"size": [
540,
680
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"localized_name": "frame",
"name": "frame",
"type": "IMAGE",
"link": null,
"shape": 7
},
{
"localized_name": "references",
"name": "references",
"type": "VIDEO_REFERENCES",
"link": null,
"shape": 7
},
{
"localized_name": "sampling_params",
"name": "sampling_params",
"type": "SAMPLING_PARAMS",
"link": null
},
{
"localized_name": "lora",
"name": "lora",
"type": "REMOTE_LORA",
"link": null
},
{
"localized_name": "model_params",
"name": "model_params",
"type": "VIDEO_PARAMS",
"link": 4,
"shape": 7
},
{
"localized_name": "fast_h3",
"name": "fast_h3",
"type": "FASTH3_DEPLOYMENT",
"link": null
},
{
"localized_name": "latent_edit",
"name": "latent_edit",
"type": "LATENT_MASK_EDITING",
"link": 3,
"shape": 7
},
{
"localized_name": "url",
"name": "url",
"type": "STRING",
"widget": {
"name": "url"
},
"link": null
},
{
"localized_name": "model",
"name": "model",
"type": "STRING",
"widget": {
"name": "model"
},
"link": null
},
{
"localized_name": "prompt",
"name": "prompt",
"type": "STRING",
"widget": {
"name": "prompt"
},
"link": null
},
{
"localized_name": "negative_prompt",
"name": "negative_prompt",
"type": "STRING",
"widget": {
"name": "negative_prompt"
},
"link": null
},
{
"localized_name": "width",
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"localized_name": "height",
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
},
{
"localized_name": "fps",
"name": "fps",
"type": "INT",
"widget": {
"name": "fps"
},
"link": null
},
{
"localized_name": "duration",
"name": "duration",
"type": "FLOAT",
"widget": {
"name": "duration"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": [
5
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"Partially restyle the clip and soundtrack",
"",
160,
120,
24,
4.458
]
},
{
"id": 7,
"type": "SaveVideo",
"pos": [
1700,
300
],
"size": [
420,
160
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"link": 5,
"shape": 7
},
{
"localized_name": "filename_prefix",
"name": "filename_prefix",
"type": "STRING",
"widget": {
"name": "filename_prefix"
},
"link": null
},
{
"localized_name": "format",
"name": "format",
"type": "COMFY_DYNAMICCOMBO_V3",
"widget": {
"name": "format"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": []
}
],
"properties": {
"Node name for S&R": "SaveVideo"
},
"widgets_values": [
"vllm-omni/latent-edit",
"mp4"
]
}
],
"links": [
[
1,
2,
0,
4,
0,
"VIDEO"
],
[
2,
3,
0,
4,
2,
"MASK"
],
[
3,
4,
0,
6,
6,
"LATENT_MASK_EDITING"
],
[
4,
5,
0,
6,
4,
"VIDEO_PARAMS"
],
[
5,
6,
0,
7,
0,
"VIDEO"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json
-560
View File
@@ -1,560 +0,0 @@
{
"last_node_id": 9,
"last_link_id": 7,
"nodes": [
{
"id": 2,
"type": "LoadVideo",
"pos": [
40,
100
],
"size": [
420,
120
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"name": "file",
"type": "STRING",
"widget": {
"name": "file"
},
"link": null
}
],
"outputs": [
{
"name": "video",
"type": "VIDEO",
"links": [
1
]
}
],
"properties": {
"Node name for S&R": "LoadVideo"
},
"widgets_values": [
""
]
},
{
"id": 3,
"type": "SolidMask",
"pos": [
40,
300
],
"size": [
420,
160
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"name": "value",
"type": "FLOAT",
"widget": {
"name": "value"
},
"link": null
},
{
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
}
],
"outputs": [
{
"name": "mask",
"type": "MASK",
"links": [
2
]
}
],
"properties": {
"Node name for S&R": "SolidMask"
},
"widgets_values": [
0.0,
160,
120
]
},
{
"id": 4,
"type": "SolidMask",
"pos": [
40,
500
],
"size": [
420,
160
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"name": "value",
"type": "FLOAT",
"widget": {
"name": "value"
},
"link": null
},
{
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
}
],
"outputs": [
{
"name": "mask",
"type": "MASK",
"links": [
3
]
}
],
"properties": {
"Node name for S&R": "SolidMask"
},
"widgets_values": [
1.0,
160,
120
]
},
{
"id": 5,
"type": "BatchMasksNode",
"pos": [
250,
380
],
"size": [
200,
100
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"name": "masks.mask0",
"type": "MASK",
"link": 2,
"slot_index": 0
},
{
"name": "masks.mask1",
"type": "MASK",
"link": 3,
"slot_index": 1
}
],
"outputs": [
{
"name": "MASK",
"type": "MASK",
"links": [
4
]
}
],
"properties": {
"Node name for S&R": "BatchMasksNode"
},
"widgets_values": []
},
{
"id": 6,
"type": "VLLMOmniLatentMaskEditing",
"pos": [
450,
180
],
"size": [
400,
120
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"name": "source_video",
"type": "VIDEO",
"link": 1
},
{
"name": "source_audio",
"type": "AUDIO",
"link": null
},
{
"name": "video_mask",
"type": "MASK",
"link": 4
},
{
"name": "audio_mask",
"type": "FLOAT",
"widget": {
"name": "audio_mask"
},
"link": null
}
],
"outputs": [
{
"name": "latent_edit",
"type": "LATENT_MASK_EDITING",
"links": [
5
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniLatentMaskEditing"
},
"widgets_values": [
-1.0
]
},
{
"id": 7,
"type": "VLLMOmniMiniMaxH3Params",
"pos": [
40,
700
],
"size": [
420,
160
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"name": "audio_flow_shift",
"type": "FLOAT",
"widget": {
"name": "audio_flow_shift"
},
"link": null
},
{
"name": "flow_shift",
"type": "FLOAT",
"widget": {
"name": "flow_shift"
},
"link": null
}
],
"outputs": [
{
"name": "video params",
"type": "VIDEO_PARAMS",
"links": [
6
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniMiniMaxH3Params"
},
"widgets_values": [
3.0,
12.0
]
},
{
"id": 8,
"type": "VLLMOmniGenerateVideo",
"pos": [
700,
180
],
"size": [
450,
380
],
"flags": {},
"order": 7,
"mode": 0,
"inputs": [
{
"name": "frame",
"type": "IMAGE",
"link": null
},
{
"name": "references",
"type": "VIDEO_REFERENCES",
"link": null
},
{
"name": "sampling_params",
"type": "SAMPLING_PARAMS",
"link": null
},
{
"name": "lora",
"type": "LORA",
"link": null
},
{
"name": "model_params",
"type": "VIDEO_PARAMS",
"link": 6
},
{
"name": "fast_h3",
"type": "FAST_H3",
"link": null
},
{
"name": "latent_edit",
"type": "LATENT_MASK_EDITING",
"link": 5
},
{
"name": "url",
"type": "STRING",
"widget": {
"name": "url"
},
"link": null
},
{
"name": "model",
"type": "STRING",
"widget": {
"name": "model"
},
"link": null
},
{
"name": "prompt",
"type": "STRING",
"widget": {
"name": "prompt"
},
"link": null
},
{
"name": "negative_prompt",
"type": "STRING",
"widget": {
"name": "negative_prompt"
},
"link": null
},
{
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
},
{
"name": "fps",
"type": "INT",
"widget": {
"name": "fps"
},
"link": null
},
{
"name": "duration",
"type": "FLOAT",
"widget": {
"name": "duration"
},
"link": null
}
],
"outputs": [
{
"name": "video",
"type": "VIDEO",
"links": [
7
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"temporal mask: regenerate the ending",
"",
160,
120,
24,
4.458
]
},
{
"id": 9,
"type": "SaveVideo",
"pos": [
900,
180
],
"size": [
420,
200
],
"flags": {},
"order": 8,
"mode": 0,
"inputs": [
{
"name": "video",
"type": "VIDEO",
"link": 7
},
{
"name": "filename_prefix",
"type": "STRING",
"widget": {
"name": "filename_prefix"
},
"link": null
},
{
"name": "format",
"type": "STRING",
"widget": {
"name": "format"
},
"link": null
}
],
"outputs": [
{
"name": "video",
"type": "VIDEO",
"links": []
}
],
"properties": {
"Node name for S&R": "SaveVideo"
},
"widgets_values": [
"temporal_mask",
"mp4"
]
}
],
"links": [
[
1,
2,
0,
6,
0,
"VIDEO"
],
[
2,
3,
0,
5,
0,
"MASK"
],
[
3,
4,
0,
5,
1,
"MASK"
],
[
4,
5,
0,
6,
2,
"MASK"
],
[
5,
6,
0,
8,
6,
"LATENT_MASK_EDITING"
],
[
6,
7,
0,
8,
4,
"VIDEO_PARAMS"
],
[
7,
8,
0,
9,
0,
"VIDEO"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json
-511
View File
@@ -1,511 +0,0 @@
{
"id": "latent-mask-editing-7380",
"revision": 0,
"last_node_id": 7,
"last_link_id": 5,
"nodes": [
{
"id": 1,
"type": "MarkdownNote",
"pos": [
40,
20
],
"size": [
420,
210
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "Latent-mask editing server prerequisite",
"properties": {},
"widgets_values": [
"Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 2,
"type": "LoadVideo",
"pos": [
40,
280
],
"size": [
420,
120
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"localized_name": "file",
"name": "file",
"type": "COMBO",
"widget": {
"name": "file"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": [
1
]
}
],
"properties": {
"Node name for S&R": "LoadVideo"
},
"widgets_values": [
""
]
},
{
"id": 3,
"type": "SolidMask",
"pos": [
40,
460
],
"size": [
420,
160
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"localized_name": "value",
"name": "value",
"type": "FLOAT",
"widget": {
"name": "value"
},
"link": null
},
{
"localized_name": "width",
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"localized_name": "height",
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
}
],
"outputs": [
{
"localized_name": "mask",
"name": "mask",
"type": "MASK",
"links": [
2
]
}
],
"properties": {
"Node name for S&R": "SolidMask"
},
"widgets_values": [
1.0,
160,
120
]
},
{
"id": 4,
"type": "VLLMOmniLatentMaskEditing",
"pos": [
560,
260
],
"size": [
420,
300
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"localized_name": "source_video",
"name": "source_video",
"type": "VIDEO",
"link": 1,
"shape": 7
},
{
"localized_name": "source_audio",
"name": "source_audio",
"type": "AUDIO",
"link": null
},
{
"localized_name": "video_mask",
"name": "video_mask",
"type": "MASK",
"link": 2,
"shape": 7
},
{
"localized_name": "audio_mask",
"name": "audio_mask",
"type": "FLOAT",
"widget": {
"name": "audio_mask"
},
"link": null
}
],
"outputs": [
{
"localized_name": "latent_edit",
"name": "latent_edit",
"type": "LATENT_MASK_EDITING",
"links": [
3
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniLatentMaskEditing"
},
"widgets_values": [
0.5
]
},
{
"id": 5,
"type": "VLLMOmniMiniMaxH3Params",
"pos": [
40,
680
],
"size": [
420,
140
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"localized_name": "audio_flow_shift",
"name": "audio_flow_shift",
"type": "FLOAT",
"widget": {
"name": "audio_flow_shift"
},
"link": null
},
{
"localized_name": "flow_shift",
"name": "flow_shift",
"type": "FLOAT",
"widget": {
"name": "flow_shift"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video params",
"name": "video params",
"type": "VIDEO_PARAMS",
"links": [
4
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniMiniMaxH3Params"
},
"widgets_values": [
3.0,
12.0
]
},
{
"id": 6,
"type": "VLLMOmniGenerateVideo",
"pos": [
1080,
200
],
"size": [
540,
680
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"localized_name": "frame",
"name": "frame",
"type": "IMAGE",
"link": null,
"shape": 7
},
{
"localized_name": "references",
"name": "references",
"type": "VIDEO_REFERENCES",
"link": null,
"shape": 7
},
{
"localized_name": "sampling_params",
"name": "sampling_params",
"type": "SAMPLING_PARAMS",
"link": null
},
{
"localized_name": "lora",
"name": "lora",
"type": "REMOTE_LORA",
"link": null
},
{
"localized_name": "model_params",
"name": "model_params",
"type": "VIDEO_PARAMS",
"link": 4,
"shape": 7
},
{
"localized_name": "fast_h3",
"name": "fast_h3",
"type": "FASTH3_DEPLOYMENT",
"link": null
},
{
"localized_name": "latent_edit",
"name": "latent_edit",
"type": "LATENT_MASK_EDITING",
"link": 3,
"shape": 7
},
{
"localized_name": "url",
"name": "url",
"type": "STRING",
"widget": {
"name": "url"
},
"link": null
},
{
"localized_name": "model",
"name": "model",
"type": "STRING",
"widget": {
"name": "model"
},
"link": null
},
{
"localized_name": "prompt",
"name": "prompt",
"type": "STRING",
"widget": {
"name": "prompt"
},
"link": null
},
{
"localized_name": "negative_prompt",
"name": "negative_prompt",
"type": "STRING",
"widget": {
"name": "negative_prompt"
},
"link": null
},
{
"localized_name": "width",
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"localized_name": "height",
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
},
{
"localized_name": "fps",
"name": "fps",
"type": "INT",
"widget": {
"name": "fps"
},
"link": null
},
{
"localized_name": "duration",
"name": "duration",
"type": "FLOAT",
"widget": {
"name": "duration"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": [
5
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"Partially restyle the clip and soundtrack",
"",
160,
120,
24,
4.458
]
},
{
"id": 7,
"type": "SaveVideo",
"pos": [
1700,
300
],
"size": [
420,
160
],
"flags": {},
"order": 6,
"mode": 0,
"inputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"link": 5,
"shape": 7
},
{
"localized_name": "filename_prefix",
"name": "filename_prefix",
"type": "STRING",
"widget": {
"name": "filename_prefix"
},
"link": null
},
{
"localized_name": "format",
"name": "format",
"type": "COMFY_DYNAMICCOMBO_V3",
"widget": {
"name": "format"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": []
}
],
"properties": {
"Node name for S&R": "SaveVideo"
},
"widgets_values": [
"vllm-omni/latent-edit",
"mp4"
]
}
],
"links": [
[
1,
2,
0,
4,
0,
"VIDEO"
],
[
2,
3,
0,
4,
2,
"MASK"
],
[
3,
4,
0,
6,
6,
"LATENT_MASK_EDITING"
],
[
4,
5,
0,
6,
4,
"VIDEO_PARAMS"
],
[
5,
6,
0,
7,
0,
"VIDEO"
]
],
"groups": [],
"config": {},
"extra": {},
"version": 0.4
}
apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json
+10 -10
View File
@@ -2293,10 +2293,10 @@
"title": "Usage Guide",
"properties": {},
"widgets_values": [
"Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
"Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. For an arbitrary-shape spatial mask, replace a case's Mask Rectangle chain with Load Image (as Mask). Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
],
"widgets_values_named": {
"text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
"text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. For an arbitrary-shape spatial mask, replace a case's Mask Rectangle chain with Load Image (as Mask). Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference."
},
"color": "#432",
"bgcolor": "#000"
@@ -2823,7 +2823,7 @@
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8001/v1",
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ",
"",
@@ -2833,7 +2833,7 @@
5
],
"widgets_values_named": {
"url": "http://127.0.0.1:8001/v1",
"url": "http://127.0.0.1:8000/v1",
"model": "MiniMaxAI/MiniMax-H3",
"prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ",
"negative_prompt": "",
@@ -3677,7 +3677,7 @@
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8001/v1",
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ",
"",
@@ -3687,7 +3687,7 @@
5
],
"widgets_values_named": {
"url": "http://127.0.0.1:8001/v1",
"url": "http://127.0.0.1:8000/v1",
"model": "MiniMaxAI/MiniMax-H3",
"prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ",
"negative_prompt": "",
@@ -3916,7 +3916,7 @@
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8001/v1",
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.",
"",
@@ -3926,7 +3926,7 @@
5
],
"widgets_values_named": {
"url": "http://127.0.0.1:8001/v1",
"url": "http://127.0.0.1:8000/v1",
"model": "MiniMaxAI/MiniMax-H3",
"prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.",
"negative_prompt": "",
@@ -4363,7 +4363,7 @@
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://127.0.0.1:8001/v1",
"http://127.0.0.1:8000/v1",
"MiniMaxAI/MiniMax-H3",
"integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A",
"",
@@ -4373,7 +4373,7 @@
10
],
"widgets_values_named": {
"url": "http://127.0.0.1:8001/v1",
"url": "http://127.0.0.1:8000/v1",
"model": "MiniMaxAI/MiniMax-H3",
"prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A",
"negative_prompt": "",