diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index f33e26b40..c26b2a3de 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C - **Generate Image** for text-to-image and image-to-image tasks - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video +- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks) - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks @@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model | | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts | | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded | -| Teal | Video References | Reference media | +| Teal | Video References, Latent Mask Editing | Reference media | Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept. @@ -263,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then: The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server. +#### Latent-mask editing (MiniMax-H3) + +The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations. + +Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts: + +- `source_video` / `source_audio` — the media to edit. +- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid. +- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend. + +A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3). + ### TTS (e.g., Qwen TTS series) (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**) diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 8c1b03d87..999f85ab2 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import ( VLLMOmniGenerateImage, VLLMOmniGenerateMusic, VLLMOmniGenerateVideo, + VLLMOmniLatentMaskEditing, VLLMOmniMiniMaxH3Params, + VLLMOmniMiniMaxH3TemporalMask, VLLMOmniQwenTTSParams, VLLMOmniRemoteLoRA, VLLMOmniSamplingParamsList, @@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import ( # A dictionary that contains all nodes you want to export with their names NODE_CLASS_MAPPINGS = { + "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask, # === Generation === "VLLMOmniGenerateImage": VLLMOmniGenerateImage, "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, @@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = { "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic, "VLLMOmniVoiceClone": VLLMOmniVoiceClone, "VLLMOmniVideoReferences": VLLMOmniVideoReferences, + "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing, # === Params === "VLLMOmniARSampling": VLLMOmniARSampling, "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling, @@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { + "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask", # === Generation === "VLLMOmniGenerateImage": "Generate Image", "VLLMOmniGenerateVideo": "Generate Video", @@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "VLLMOmniGenerateMusic": "Generate Music", "VLLMOmniVoiceClone": "TTS Voice Cloning", "VLLMOmniVideoReferences": "Video References", + "VLLMOmniLatentMaskEditing": "Latent Mask Editing", # === Params === "VLLMOmniARSampling": "AR Sampling Params", "VLLMOmniDiffusionSampling": "Diffusion Sampling Params", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index 6d5006a11..5bddb9345 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +import math from typing import Literal import torch from comfy_api.input import AudioInput, VideoInput from .utils.api_client import VLLMOmniClient +from .utils.latent_mask import _align_frame_count, _video_latent_t from .utils.logger import get_logger from .utils.models import lookup_model_spec from .utils.types import ( @@ -17,6 +19,7 @@ from .utils.types import ( AutoregressionSamplingParams, DiffusionSamplingParams, FastH3Deployment, + LatentMaskEditing, MiniMaxH3ModelSpecificParams, QwenTTSModelSpecificParams, VideoReferences, @@ -210,6 +213,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): "lora": ("REMOTE_LORA",), "model_params": ("VIDEO_PARAMS",), "fast_h3": ("FASTH3_DEPLOYMENT",), + "latent_edit": ("LATENT_MASK_EDITING",), }, } @@ -262,6 +266,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): model_params: dict | None = None, lora: dict | None = None, fast_h3: dict | None = None, + latent_edit: dict | None = None, **kwargs, ): if kwargs: @@ -350,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): sampling_params=sampling_params, lora=lora, model_params=model_params, + latent_edit=latent_edit, ) return (output,) @@ -1051,3 +1057,89 @@ class VLLMOmniVideoReferences: if value is not None: refs[f"{kind}_{index}"] = value return (refs,) + + +class VLLMOmniLatentMaskEditing: + @classmethod + def INPUT_TYPES(cls): + return { + "required": {}, + "optional": { + "source_video": ("VIDEO",), + "source_audio": ("AUDIO",), + "video_mask": ("MASK",), + "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}), + }, + } + + RETURN_TYPES = ("LATENT_MASK_EDITING",) + RETURN_NAMES = ("latent_edit",) + FUNCTION = "get_latent_edit" + CATEGORY = "vLLM-Omni" + + def get_latent_edit( + self, + source_video: VideoInput | None = None, + source_audio: AudioInput | None = None, + video_mask: torch.Tensor | None = None, + audio_mask: float = -1.0, + **kwargs, + ): + if kwargs: + logger.info("Uncaught kwargs: %s", kwargs) + edit = LatentMaskEditing() + if source_video is not None: + edit["source_video"] = source_video + if source_audio is not None: + edit["source_audio"] = source_audio + if video_mask is not None: + edit["video_mask"] = video_mask + if audio_mask >= 0.0: + edit["audio_mask"] = audio_mask + return (edit,) + + +class VLLMOmniMiniMaxH3TemporalMask: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}), + "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}), + "mode": (["continuation", "extension"],), + "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}), + } + } + + RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK") + RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask") + FUNCTION = "build" + CATEGORY = "vLLM-Omni" + + def build(self, images, source_fps, duration, mode, preserve_fraction=0.5): + if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0: + raise ValueError("Source must contain frames and have a positive finite FPS.") + if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1: + raise ValueError("Invalid duration or preserve_fraction.") + frames = _align_frame_count(max(1, round(duration * 24))) + source_seconds = images.shape[0] / source_fps + if mode == "extension": + if frames / 24 <= source_seconds: + raise ValueError("Extension output must be longer than the source; increase duration.") + boundary = source_seconds + elif mode == "continuation": + boundary = min(source_seconds, frames / 24) * preserve_fraction + else: + raise ValueError(f"Unknown temporal mask mode: {mode}") + available = min(frames, math.floor(boundary * 24 + 1e-8)) + prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17) + preserved = _video_latent_t(prefix) if prefix else 0 + total = _video_latent_t(frames) + mask = torch.ones(total, 1, 1) + mask[:preserved] = 0 + indices = torch.arange(frames, device=images.device) + source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1) + preview_images = images.index_select(0, source_indices) + preview_mask = mask.index_select(0, torch.arange(frames) * total // frames) + return mask, 24.0, preview_images, preview_mask diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index 21935414b..69e1af899 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -30,6 +30,7 @@ from .format import ( video_to_base64, video_to_bytes, ) +from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json from .logger import get_logger, pretty_printer from .models import lookup_model_spec from .types import ( @@ -283,6 +284,7 @@ class VLLMOmniClient: sampling_params: dict | None = None, model_params: dict | None = None, lora: dict | None = None, + latent_edit: dict | None = None, spec_model: str | None = None, **extra_params, ) -> VideoInput: @@ -386,6 +388,53 @@ class VLLMOmniClient: content_type="image/png", ) + # === latent-mask editing (MiniMax H3) === + if latent_edit is not None: + source_video = latent_edit.get("source_video") + source_audio = latent_edit.get("source_audio") + video_mask = latent_edit.get("video_mask") + audio_mask = latent_edit.get("audio_mask") + + if video_mask is None and audio_mask is None: + raise ValueError("Latent-mask editing requires at least one mask.") + + video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item()) + audio_mask_trivial = audio_mask is None or audio_mask == 1.0 + if not video_mask_trivial and source_video is None: + raise ValueError("A non-trivial video mask requires a source video.") + if not audio_mask_trivial and source_audio is None and source_video is None: + raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.") + + if source_video is not None: + form.add_field( + "source_video", + video_to_bytes(source_video, "source.mp4"), + filename="source.mp4", + content_type="video/mp4", + ) + if source_audio is not None: + form.add_field( + "source_audio", + audio_to_bytes(source_audio, "source_audio.mp3"), + filename="source_audio.mp3", + content_type="audio/mpeg", + ) + if video_mask is not None: + mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames) + form.add_field( + "video_noise_mask", + mask_json.encode("utf-8"), + filename="video-mask.json", + content_type="application/json", + ) + if audio_mask is not None: + form.add_field( + "audio_noise_mask", + scalar_mask_to_json(audio_mask).encode("utf-8"), + filename="audio-mask.json", + content_type="application/json", + ) + if len(keyframe_images) == 2: for image_filename, image in keyframe_images: form.add_field( diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py new file mode 100644 index 000000000..532cf8184 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import json + +import torch +import torch.nn.functional as F + + +def scalar_mask_to_json(value: float) -> str: + if not 0.0 <= value <= 1.0: + raise ValueError(f"mask value must be in [0, 1], got {value}") + return str(value) + + +def _align_frame_count(frame_count: int) -> int: + if frame_count <= 0: + return 1 + current = int(frame_count) + while current % 17 != 5: + current += 1 + return current + + +def _video_latent_t(frame_count: int) -> int: + if frame_count <= 5: + return 2 + return ((int(frame_count) - 5) // 17) * 5 + 2 + + +def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor: + tv = _video_latent_t(_align_frame_count(num_frames)) + height = int(height) // 32 * 32 + width = int(width) // 32 * 32 + gh, gw = height // 16, width // 16 + + if mask.ndim == 2: + mask = mask.unsqueeze(0) + elif mask.ndim != 3: + raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D") + + grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1) + if grid.shape[0] == 1: + grid = grid.expand(tv, gh, gw) + elif grid.shape[0] != tv: + grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0) + return grid + + +def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str: + return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist()) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py index 6d12a98c2..6194613de 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py @@ -40,6 +40,10 @@ class VideoReferences(dict): pass +class LatentMaskEditing(dict): + pass + + class FastH3Deployment(dict): """Descriptor for a server that fused FastH3 at startup.""" diff --git a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md new file mode 100644 index 000000000..9ed10273f --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md @@ -0,0 +1,17 @@ +# WF-05: MiniMax-H3 latent-mask editing + +Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case. + +## Requirements + +- ComfyUI with ComfyUI-vLLM-Omni installed. +- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup. + +## Use + +Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run. + +1. Upload your source video in Load Video. +2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate. +3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip). +4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases. diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json new file mode 100644 index 000000000..17be731bf --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json @@ -0,0 +1,499 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "LoadImageMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "STRING", + "widget": { + "name": "image" + }, + "link": null + }, + { + "name": "channel", + "type": "STRING", + "widget": { + "name": "channel" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImageMask" + }, + "widgets_values": [ + "", + "red" + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json new file mode 100644 index 000000000..b2b3a6980 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json @@ -0,0 +1,560 @@ +{ + "last_node_id": 9, + "last_link_id": 7, + "nodes": [ + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 100 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "file", + "type": "STRING", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "SolidMask", + "pos": [ + 40, + 500 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 5, + "type": "BatchMasksNode", + "pos": [ + 250, + 380 + ], + "size": [ + 200, + 100 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "masks.mask0", + "type": "MASK", + "link": 2, + "slot_index": 0 + }, + { + "name": "masks.mask1", + "type": "MASK", + "link": 3, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "BatchMasksNode" + }, + "widgets_values": [] + }, + { + "id": 6, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 450, + 180 + ], + "size": [ + 400, + 120 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "source_video", + "type": "VIDEO", + "link": 1 + }, + { + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "name": "video_mask", + "type": "MASK", + "link": 4 + }, + { + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + -1.0 + ] + }, + { + "id": 7, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 700 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 8, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 700, + 180 + ], + "size": [ + 450, + 380 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "frame", + "type": "IMAGE", + "link": null + }, + { + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "name": "lora", + "type": "LORA", + "link": null + }, + { + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 6 + }, + { + "name": "fast_h3", + "type": "FAST_H3", + "link": null + }, + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 5 + }, + { + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "temporal mask: regenerate the ending", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 9, + "type": "SaveVideo", + "pos": [ + 900, + 180 + ], + "size": [ + 420, + 200 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 7 + }, + { + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "name": "format", + "type": "STRING", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "temporal_mask", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 6, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 5, + 0, + "MASK" + ], + [ + 3, + 4, + 0, + 5, + 1, + "MASK" + ], + [ + 4, + 5, + 0, + 6, + 2, + "MASK" + ], + [ + 5, + 6, + 0, + 8, + 6, + "LATENT_MASK_EDITING" + ], + [ + 6, + 7, + 0, + 8, + 4, + "VIDEO_PARAMS" + ], + [ + 7, + 8, + 0, + 9, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json new file mode 100644 index 000000000..4192ef4c5 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json @@ -0,0 +1,511 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json new file mode 100644 index 000000000..fc4341730 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json @@ -0,0 +1,5241 @@ +{ + "id": "eebf673c-c66a-4f20-9e87-b728a82f6e5f", + "revision": 0, + "last_node_id": 99, + "last_link_id": 162, + "nodes": [ + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 2160 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 11, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 1870 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 25, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 3760 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 29, + "type": "MarkdownNote", + "pos": [ + 1050, + 3470 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Inpainting guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." + ], + "widgets_values_named": { + "text": "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 30, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 3470 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 34, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 5360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 38 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 45, + "type": "MarkdownNote", + "pos": [ + 1050, + 5070 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Continuation guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." + ], + "widgets_values_named": { + "text": "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 46, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 5070 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 48 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 49, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 6960 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 51 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 59, + "type": "VLLMOmniMiniMaxH3TemporalMask", + "pos": [ + 580, + 6040 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 123 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 124 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 60 + ] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 146 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 153, + 156 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 155 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" + }, + "widgets_values": [ + 24, + 10, + "extension", + 0.5 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 10, + "mode": "extension", + "preserve_fraction": 0.5 + } + }, + { + "id": 60, + "type": "MarkdownNote", + "pos": [ + 1050, + 6670 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Extension guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." + ], + "widgets_values_named": { + "text": "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + -268.4516045720269, + 1190 + ], + "size": [ + 420, + 314 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + }, + { + "localized_name": "choose file to upload", + "name": "upload", + "type": "IMAGEUPLOAD", + "widget": { + "name": "upload" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 1, + 17, + 28, + 36, + 49 + ] + } + ], + "title": "Shared Source Video | All Cases", + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "", + "image" + ], + "widgets_values_named": { + "file": "", + "upload": "image" + } + }, + { + "id": 62, + "type": "MarkdownNote", + "pos": [ + 1050, + 1870 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Object Removal guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." + ], + "widgets_values_named": { + "text": "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 61, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 6670 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 61 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 65, + "type": "CreateVideo", + "pos": [ + 1990, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 130 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 131 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 77 + ] + } + ], + "title": "Object Removal | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 69, + "type": "CreateVideo", + "pos": [ + 1990, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 137 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 138 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 91 + ] + } + ], + "title": "Inpainting | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 73, + "type": "CreateVideo", + "pos": [ + 1990, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 152 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 142 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 105 + ] + } + ], + "title": "Continuation | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 77, + "type": "CreateVideo", + "pos": [ + 1990, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 158 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 146 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 119 + ] + } + ], + "title": "Extension | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 79, + "type": "EmptyImage", + "pos": [ + 1090, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 126 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 80, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 125 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 126 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 127 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 129 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 81, + "type": "ImageBlend", + "pos": [ + 1690, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 128 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 129 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 130 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 82, + "type": "EmptyImage", + "pos": [ + 1090, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 133 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 83, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 132 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 133 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 134 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 136 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 84, + "type": "ImageBlend", + "pos": [ + 1690, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 135 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 136 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 137 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 89, + "type": "EmptyImage", + "pos": [ + 1090, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 148 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 90, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 147 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 148 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 149 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 151 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 91, + "type": "ImageBlend", + "pos": [ + 1690, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 150 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 151 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 152 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 92, + "type": "EmptyImage", + "pos": [ + 1090, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 154 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 93, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 153 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 154 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 155 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 157 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 94, + "type": "ImageBlend", + "pos": [ + 1690, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 156 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 157 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 158 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + -268.4516045720269, + 2138.3724768826028 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Usage Guide", + "properties": {}, + "widgets_values": [ + "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." + ], + "widgets_values_named": { + "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." + }, + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 15, + "type": "GetVideoComponents", + "pos": [ + -268.4516045720269, + 2438.3724768826028 + ], + "size": [ + 193.530859375, + 106 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 120, + 123, + 125, + 128, + 132, + 135 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 121, + 124, + 131, + 138 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "COMBO", + "links": [] + }, + { + "localized_name": "color_space", + "name": "color_space", + "type": "COMBO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents" + } + }, + { + "id": 95, + "type": "SolidMask", + "pos": [ + 580, + 1240 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 159 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 97, + "type": "SolidMask", + "pos": [ + 580, + 2840 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 161 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 66, + "type": "SaveVideo", + "pos": [ + 2530, + 1240 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 77 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Object Removal)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-object-removal-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 78, + "type": "SaveVideo", + "pos": [ + 2530, + 6040 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 119 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Extension)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-extension-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 1870 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 10 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 4 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 3 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "title": "Generate Video(Object Removal)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 96, + "type": "SolidMask", + "pos": [ + 580, + 1360 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 160 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 9, + "type": "MaskComposite", + "pos": [ + 580, + 1540 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 159 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 160 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 63, + 127 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 98, + "type": "SolidMask", + "pos": [ + 580, + 2960 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 162 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 99, + "type": "MaskComposite", + "pos": [ + 580, + 3140 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 161 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 162 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 35, + 134 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 70, + "type": "SaveVideo", + "pos": [ + 2530, + 2840 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 91 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Inpainting)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-inpainting-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 1870 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 1 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 63 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 2530, + 1870 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Object Removal)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-object-removal", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 27, + "type": "SaveVideo", + "pos": [ + 2530, + 3470 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 31 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Inpainting)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-inpainting", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 24, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 3470 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 28 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 35 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 29 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 26, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 3470 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 33 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 30 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 29 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 31 + ] + } + ], + "title": "Generate Video(Inpainting)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 33, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 5070 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 36 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 47 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 37 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.8 + ], + "widgets_values_named": { + "audio_mask": 0.8 + } + }, + { + "id": 35, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 5070 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 48 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 38 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 37 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 39 + ] + } + ], + "title": "Generate Video(Continuation)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 44, + "type": "VLLMOmniMiniMaxH3TemporalMask", + "pos": [ + 580, + 4440 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 120 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 121 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 47 + ] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 142 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 147, + 150 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 149 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" + }, + "widgets_values": [ + 24, + 5, + "continuation", + 0.2 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 5, + "mode": "continuation", + "preserve_fraction": 0.2 + } + }, + { + "id": 36, + "type": "SaveVideo", + "pos": [ + 2530, + 5070 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 39 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Continuation)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-continuation", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 74, + "type": "SaveVideo", + "pos": [ + 2530, + 4440 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 105 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Continuation)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-continuation-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 50, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 6670 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 61 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 51 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 50 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 52 + ] + } + ], + "title": "Generate Video(Extension)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "", + 1344, + 768, + 24, + 10 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 10 + } + }, + { + "id": 48, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 6670 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 49 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 60 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 50 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.9 + ], + "widgets_values_named": { + "audio_mask": 0.9 + } + }, + { + "id": 51, + "type": "SaveVideo", + "pos": [ + 2530, + 6670 + ], + "size": [ + 420, + 338 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 52 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Extension)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-extension", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ], + [ + 10, + 11, + 0, + 6, + 2, + "SAMPLING_PARAMS" + ], + [ + 17, + 2, + 0, + 15, + 0, + "VIDEO" + ], + [ + 28, + 2, + 0, + 24, + 0, + "VIDEO" + ], + [ + 29, + 24, + 0, + 26, + 6, + "LATENT_MASK_EDITING" + ], + [ + 30, + 25, + 0, + 26, + 4, + "VIDEO_PARAMS" + ], + [ + 31, + 26, + 0, + 27, + 0, + "VIDEO" + ], + [ + 33, + 30, + 0, + 26, + 2, + "SAMPLING_PARAMS" + ], + [ + 35, + 99, + 0, + 24, + 2, + "MASK" + ], + [ + 36, + 2, + 0, + 33, + 0, + "VIDEO" + ], + [ + 37, + 33, + 0, + 35, + 6, + "LATENT_MASK_EDITING" + ], + [ + 38, + 34, + 0, + 35, + 4, + "VIDEO_PARAMS" + ], + [ + 39, + 35, + 0, + 36, + 0, + "VIDEO" + ], + [ + 47, + 44, + 0, + 33, + 2, + "MASK" + ], + [ + 48, + 46, + 0, + 35, + 2, + "SAMPLING_PARAMS" + ], + [ + 49, + 2, + 0, + 48, + 0, + "VIDEO" + ], + [ + 50, + 48, + 0, + 50, + 6, + "LATENT_MASK_EDITING" + ], + [ + 51, + 49, + 0, + 50, + 4, + "VIDEO_PARAMS" + ], + [ + 52, + 50, + 0, + 51, + 0, + "VIDEO" + ], + [ + 60, + 59, + 0, + 48, + 2, + "MASK" + ], + [ + 61, + 61, + 0, + 50, + 2, + "SAMPLING_PARAMS" + ], + [ + 63, + 9, + 0, + 4, + 2, + "MASK" + ], + [ + 77, + 65, + 0, + 66, + 0, + "VIDEO" + ], + [ + 91, + 69, + 0, + 70, + 0, + "VIDEO" + ], + [ + 105, + 73, + 0, + 74, + 0, + "VIDEO" + ], + [ + 119, + 77, + 0, + 78, + 0, + "VIDEO" + ], + [ + 120, + 15, + 0, + 44, + 0, + "IMAGE" + ], + [ + 121, + 15, + 2, + 44, + 1, + "FLOAT" + ], + [ + 123, + 15, + 0, + 59, + 0, + "IMAGE" + ], + [ + 124, + 15, + 2, + 59, + 1, + "FLOAT" + ], + [ + 125, + 15, + 0, + 80, + 0, + "IMAGE" + ], + [ + 126, + 79, + 0, + 80, + 1, + "IMAGE" + ], + [ + 127, + 9, + 0, + 80, + 2, + "MASK" + ], + [ + 128, + 15, + 0, + 81, + 0, + "IMAGE" + ], + [ + 129, + 80, + 0, + 81, + 1, + "IMAGE" + ], + [ + 130, + 81, + 0, + 65, + 0, + "IMAGE" + ], + [ + 131, + 15, + 2, + 65, + 2, + "FLOAT" + ], + [ + 132, + 15, + 0, + 83, + 0, + "IMAGE" + ], + [ + 133, + 82, + 0, + 83, + 1, + "IMAGE" + ], + [ + 134, + 99, + 0, + 83, + 2, + "MASK" + ], + [ + 135, + 15, + 0, + 84, + 0, + "IMAGE" + ], + [ + 136, + 83, + 0, + 84, + 1, + "IMAGE" + ], + [ + 137, + 84, + 0, + 69, + 0, + "IMAGE" + ], + [ + 138, + 15, + 2, + 69, + 2, + "FLOAT" + ], + [ + 142, + 44, + 1, + 73, + 2, + "FLOAT" + ], + [ + 146, + 59, + 1, + 77, + 2, + "FLOAT" + ], + [ + 147, + 44, + 2, + 90, + 0, + "IMAGE" + ], + [ + 148, + 89, + 0, + 90, + 1, + "IMAGE" + ], + [ + 149, + 44, + 3, + 90, + 2, + "MASK" + ], + [ + 150, + 44, + 2, + 91, + 0, + "IMAGE" + ], + [ + 151, + 90, + 0, + 91, + 1, + "IMAGE" + ], + [ + 152, + 91, + 0, + 73, + 0, + "IMAGE" + ], + [ + 153, + 59, + 2, + 93, + 0, + "IMAGE" + ], + [ + 154, + 92, + 0, + 93, + 1, + "IMAGE" + ], + [ + 155, + 59, + 3, + 93, + 2, + "MASK" + ], + [ + 156, + 59, + 2, + 94, + 0, + "IMAGE" + ], + [ + 157, + 93, + 0, + 94, + 1, + "IMAGE" + ], + [ + 158, + 94, + 0, + 77, + 0, + "IMAGE" + ], + [ + 159, + 95, + 0, + 9, + 0, + "MASK" + ], + [ + 160, + 96, + 0, + 9, + 1, + "MASK" + ], + [ + 161, + 97, + 0, + 99, + 0, + "MASK" + ], + [ + 162, + 98, + 0, + 99, + 1, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "Shared Input", + "bounding": [ + -308.451604572027, + 1080, + 473.7665553206702, + 1508.3724768826028 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 2, + "title": "Output Hub", + "bounding": [ + 2470, + 1080, + 540, + 6400 + ], + "color": "#444", + "flags": {} + }, + { + "id": 10, + "title": "Case 01 | Object Removal", + "bounding": [ + 540, + 1080, + 1880, + 1540 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 20, + "title": "Case 01 | Object Removal Output", + "bounding": [ + 2490, + 1130, + 500, + 1490 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 11, + "title": "Case 02 | Inpainting", + "bounding": [ + 540, + 2680, + 1880, + 1540 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 21, + "title": "Case 02 | Inpainting Output", + "bounding": [ + 2490, + 2730, + 500, + 1490 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 12, + "title": "Case 03 | Continuation", + "bounding": [ + 540, + 4280, + 1880, + 1540 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 22, + "title": "Case 03 | Continuation Output", + "bounding": [ + 2490, + 4330, + 500, + 1490 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 13, + "title": "Case 04 | Extension", + "bounding": [ + 540, + 5880, + 1880, + 1540 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 23, + "title": "Case 04 | Extension Output", + "bounding": [ + 2490, + 5930, + 500, + 1490 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 24, + "title": "Object Removal | Mask Generation", + "bounding": [ + 560, + 1130, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 25, + "title": "Inpainting | Mask Generation", + "bounding": [ + 560, + 2730, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 26, + "title": "Continuation | Mask Generation", + "bounding": [ + 560, + 4330, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 27, + "title": "Extension | Mask Generation", + "bounding": [ + 560, + 5930, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9837545257996714, + "offset": [ + -1807.8247536090078, + -6349.252392678693 + ] + } + }, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/web/main.js b/apps/ComfyUI-vLLM-Omni/web/main.js index 9aba6260e..f8c633e11 100644 --- a/apps/ComfyUI-vLLM-Omni/web/main.js +++ b/apps/ComfyUI-vLLM-Omni/web/main.js @@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = { REMOTE_LORA: "deployment", FASTH3_DEPLOYMENT: "deployment", VIDEO_REFERENCES: "media", + LATENT_MASK_EDITING: "media", }; function familyOf(nodeData) { diff --git a/tests/e2e/features/comfyui/mock_videos_server.py b/tests/e2e/features/comfyui/mock_videos_server.py new file mode 100644 index 000000000..c2ddfbb5e --- /dev/null +++ b/tests/e2e/features/comfyui/mock_videos_server.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests. + +Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test +can assert exactly what the client serialized, and returns a synthetic MP4. +""" + +import asyncio +import json +import os +import subprocess +import uuid + +from fastapi import FastAPI, Request +from fastapi.responses import Response +from starlette.datastructures import UploadFile + +app = FastAPI() +_JOBS: dict[str, dict] = {} +_STATE_FILE = os.environ.get("MOCK_STATE_FILE") + + +def _record(fields: dict) -> None: + if _STATE_FILE: + with open(_STATE_FILE, "w") as f: + json.dump(fields, f) + + +def _synthetic_mp4() -> bytes: + return subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=24:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=32000:duration=1", + "-map", + "0:v", + "-map", + "1:a", + "-shortest", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-ar", + "32000", + "-ac", + "2", + "-movflags", + "frag_keyframe+empty_moov", + "-f", + "mp4", + "pipe:1", + ], + check=True, + capture_output=True, + ).stdout + + +@app.post("/v1/videos") +async def create_video(request: Request): + form = await request.form() + fields = {} + for key, value in form.items(): + if isinstance(value, UploadFile): + data = await value.read() + fields[key] = { + "file": True, + "size": len(data), + "filename": value.filename, + "content_type": value.content_type, + } + if value.content_type == "application/json": + fields[key]["json"] = json.loads(data) + else: + fields[key] = str(value) + _record(fields) + + job_id = str(uuid.uuid4()) + _JOBS[job_id] = {"status": "queued"} + return {"id": job_id, "status": "queued"} + + +@app.get("/v1/videos/{job_id}") +async def get_video(job_id: str): + if job_id not in _JOBS: + return {"status": "failed", "error": "unknown job"} + return {"id": job_id, "status": "completed"} + + +@app.get("/v1/videos/{job_id}/content") +async def get_video_content(job_id: str): + if job_id not in _JOBS: + return Response(status_code=404) + mp4 = await asyncio.to_thread(_synthetic_mp4) + return Response(content=mp4, media_type="video/mp4") + + +@app.delete("/v1/videos/{job_id}") +async def delete_video(job_id: str): + _JOBS.pop(job_id, None) + return {"deleted": True} diff --git a/tests/e2e/features/comfyui/test_latent_mask.py b/tests/e2e/features/comfyui/test_latent_mask.py new file mode 100644 index 000000000..e208fd5b4 --- /dev/null +++ b/tests/e2e/features/comfyui/test_latent_mask.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Unit tests for latent-mask serialization helpers.""" + +import json + +import pytest +import torch +from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_scalar_mask_to_json(): + assert scalar_mask_to_json(0.5) == "0.5" + assert scalar_mask_to_json(1.0) == "1.0" + with pytest.raises(ValueError): + scalar_mask_to_json(1.5) + with pytest.raises(ValueError): + scalar_mask_to_json(-0.1) + + +def test_video_mask_grid_shape(): + mask = torch.zeros(256, 448) + grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107)) + assert len(grid) == 2 + 5 * ((107 - 5) // 17) + assert len(grid[0]) == 256 // 16 + assert len(grid[0][0]) == 448 // 16 + + +def test_video_mask_grid_shape_non_aligned_frames(): + # 96 snaps up to 107 (17n+5), so Tv == 32, matching the server. + grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96)) + assert len(grid) == 32 + + +def test_video_mask_grid_shape_short_clip(): + # num_frames <= 5 -> Tv == 2. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1)) + assert len(grid) == 2 + + +def test_video_mask_grid_shape_unaligned_width(): + # width=500 floors to 480 (multiple of 32) -> gw == 30, not 31. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22)) + assert len(grid[0][0]) == 30 + + +def test_video_mask_3d_input(): + grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107)) + assert grid[0][0][0] == 1.0 + + +def test_video_mask_temporal(): + # 3D mask: first half 0 (preserve), second half 1 (regenerate). + mask = torch.zeros(7, 64, 64) + mask[4:] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + assert all(v == 0.0 for row in grid[0] for v in row) + assert all(v == 1.0 for row in grid[-1] for v in row) + + +def test_video_mask_temporal_resize(): + # 2 temporal slices resize to tv == 7. + mask = torch.zeros(2, 64, 64) + mask[1] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + + +def test_video_mask_uniform_preserved(): + grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107)) + assert all(v == 0.5 for row in grid for cell in row for v in cell) + + +def test_video_mask_values_in_range(): + grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22)) + assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell) diff --git a/tests/e2e/features/comfyui/test_latent_mask_editing.py b/tests/e2e/features/comfyui/test_latent_mask_editing.py new file mode 100644 index 000000000..9162974b6 --- /dev/null +++ b/tests/e2e/features/comfyui/test_latent_mask_editing.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""End-to-end test for latent-mask editing serialization. + +Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload +against a mock ``/v1/videos`` server and asserts the multipart fields the +client produced. Runs on CPU: the ComfyUI ``comfy_api`` / ``comfy_extras`` +modules are mocked by this directory's ``conftest.py``. +""" + +import asyncio +import json +import os +import socket +import subprocess +import sys +import time + +import pytest +import torch +from comfy_api.input import VideoInput +from comfyui_vllm_omni.utils.api_client import VLLMOmniClient + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture +def mock_server(tmp_path): + port = _free_port() + state_file = tmp_path / "state.json" + env = dict(os.environ, MOCK_STATE_FILE=str(state_file)) + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "mock_videos_server:app", "--host", "127.0.0.1", "--port", str(port)], + cwd=_TESTS_DIR, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.time() + 30 + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + break + except OSError: + time.sleep(0.2) + yield f"http://127.0.0.1:{port}/v1", state_file + finally: + proc.terminate() + proc.wait(timeout=10) + + +def test_latent_edit_serialization(mock_server): + base_url, state_file = mock_server + + # A non-trivial video mask requires a source video. The mocked VideoInput + # only needs to provide ``save_to`` for the client's multipart upload. + source_video = VideoInput(b"mock_source_video") + mask = torch.zeros(1, 120, 160) + latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5} + + async def run(): + client = VLLMOmniClient(base_url) + return await client.generate_video( + model="MiniMaxAI/MiniMax-H3", + prompt="restyle the clip", + width=160, + height=120, + num_frames=22, + fps=24, + latent_edit=latent_edit, + ) + + out = asyncio.run(run()) + assert out is not None + + with open(state_file) as f: + fields = json.load(f) + + assert fields["source_video"]["file"] is True + assert fields["source_video"]["content_type"] == "video/mp4" + for name in ("video_noise_mask", "audio_noise_mask"): + assert fields[name]["file"] is True + assert fields[name]["content_type"] == "application/json" + assert fields["audio_noise_mask"]["json"] == 0.5 + + video_mask = fields["video_noise_mask"]["json"] + assert len(video_mask) == 7 # num_frames=22 aligns to 22 (17n+5) -> Tv = 7 + assert len(video_mask[0]) == 6 # height 120 floors to 96 (multiple of 32) -> gh 6 + assert len(video_mask[0][0]) == 10 # width 160 -> gw 10