[Frontend] Add MiniMax-H3 latent editing workflows (WF-05) (#7898)
Signed-off-by: chen hongwei <1792043268@qq.com> Signed-off-by: Weiming Liao <liaowm5@gmail.com> Co-authored-by: chen hongwei <1792043268@qq.com> Co-authored-by: chen hongwei <54873389+avicii-forever@users.noreply.github.com>
This commit is contained in:
15 files changed
+7339
-1
No files matched your search
@@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C
|
||||
|
||||
- **Generate Image** for text-to-image and image-to-image tasks
|
||||
- **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
|
||||
- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks)
|
||||
- **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup
|
||||
- **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
|
||||
- **TTS** and **TTS Voice Clone** for TTS tasks
|
||||
@@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it
|
||||
| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model |
|
||||
| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts |
|
||||
| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded |
|
||||
| Teal | Video References | Reference media |
|
||||
| Teal | Video References, Latent Mask Editing | Reference media |
|
||||
|
||||
Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept.
|
||||
|
||||
@@ -263,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then:
|
||||
|
||||
The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server.
|
||||
|
||||
#### Latent-mask editing (MiniMax-H3)
|
||||
|
||||
The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations.
|
||||
|
||||
Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts:
|
||||
|
||||
- `source_video` / `source_audio` — the media to edit.
|
||||
- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid.
|
||||
- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend.
|
||||
|
||||
A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3).
|
||||
|
||||
### TTS (e.g., Qwen TTS series)
|
||||
|
||||
(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)
|
||||
|
||||
@@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import (
|
||||
VLLMOmniGenerateImage,
|
||||
VLLMOmniGenerateMusic,
|
||||
VLLMOmniGenerateVideo,
|
||||
VLLMOmniLatentMaskEditing,
|
||||
VLLMOmniMiniMaxH3Params,
|
||||
VLLMOmniMiniMaxH3TemporalMask,
|
||||
VLLMOmniQwenTTSParams,
|
||||
VLLMOmniRemoteLoRA,
|
||||
VLLMOmniSamplingParamsList,
|
||||
@@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import (
|
||||
|
||||
# A dictionary that contains all nodes you want to export with their names
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask,
|
||||
# === Generation ===
|
||||
"VLLMOmniGenerateImage": VLLMOmniGenerateImage,
|
||||
"VLLMOmniGenerateVideo": VLLMOmniGenerateVideo,
|
||||
@@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
"VLLMOmniGenerateMusic": VLLMOmniGenerateMusic,
|
||||
"VLLMOmniVoiceClone": VLLMOmniVoiceClone,
|
||||
"VLLMOmniVideoReferences": VLLMOmniVideoReferences,
|
||||
"VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing,
|
||||
# === Params ===
|
||||
"VLLMOmniARSampling": VLLMOmniARSampling,
|
||||
"VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
|
||||
@@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
|
||||
# A dictionary that contains the friendly/humanly readable titles for the nodes
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask",
|
||||
# === Generation ===
|
||||
"VLLMOmniGenerateImage": "Generate Image",
|
||||
"VLLMOmniGenerateVideo": "Generate Video",
|
||||
@@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"VLLMOmniGenerateMusic": "Generate Music",
|
||||
"VLLMOmniVoiceClone": "TTS Voice Cloning",
|
||||
"VLLMOmniVideoReferences": "Video References",
|
||||
"VLLMOmniLatentMaskEditing": "Latent Mask Editing",
|
||||
# === Params ===
|
||||
"VLLMOmniARSampling": "AR Sampling Params",
|
||||
"VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import math
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
from comfy_api.input import AudioInput, VideoInput
|
||||
|
||||
from .utils.api_client import VLLMOmniClient
|
||||
from .utils.latent_mask import _align_frame_count, _video_latent_t
|
||||
from .utils.logger import get_logger
|
||||
from .utils.models import lookup_model_spec
|
||||
from .utils.types import (
|
||||
@@ -17,6 +19,7 @@ from .utils.types import (
|
||||
AutoregressionSamplingParams,
|
||||
DiffusionSamplingParams,
|
||||
FastH3Deployment,
|
||||
LatentMaskEditing,
|
||||
MiniMaxH3ModelSpecificParams,
|
||||
QwenTTSModelSpecificParams,
|
||||
VideoReferences,
|
||||
@@ -210,6 +213,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
"lora": ("REMOTE_LORA",),
|
||||
"model_params": ("VIDEO_PARAMS",),
|
||||
"fast_h3": ("FASTH3_DEPLOYMENT",),
|
||||
"latent_edit": ("LATENT_MASK_EDITING",),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -262,6 +266,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
model_params: dict | None = None,
|
||||
lora: dict | None = None,
|
||||
fast_h3: dict | None = None,
|
||||
latent_edit: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if kwargs:
|
||||
@@ -350,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
|
||||
sampling_params=sampling_params,
|
||||
lora=lora,
|
||||
model_params=model_params,
|
||||
latent_edit=latent_edit,
|
||||
)
|
||||
return (output,)
|
||||
|
||||
@@ -1051,3 +1057,89 @@ class VLLMOmniVideoReferences:
|
||||
if value is not None:
|
||||
refs[f"{kind}_{index}"] = value
|
||||
return (refs,)
|
||||
|
||||
|
||||
class VLLMOmniLatentMaskEditing:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {},
|
||||
"optional": {
|
||||
"source_video": ("VIDEO",),
|
||||
"source_audio": ("AUDIO",),
|
||||
"video_mask": ("MASK",),
|
||||
"audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}),
|
||||
},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("LATENT_MASK_EDITING",)
|
||||
RETURN_NAMES = ("latent_edit",)
|
||||
FUNCTION = "get_latent_edit"
|
||||
CATEGORY = "vLLM-Omni"
|
||||
|
||||
def get_latent_edit(
|
||||
self,
|
||||
source_video: VideoInput | None = None,
|
||||
source_audio: AudioInput | None = None,
|
||||
video_mask: torch.Tensor | None = None,
|
||||
audio_mask: float = -1.0,
|
||||
**kwargs,
|
||||
):
|
||||
if kwargs:
|
||||
logger.info("Uncaught kwargs: %s", kwargs)
|
||||
edit = LatentMaskEditing()
|
||||
if source_video is not None:
|
||||
edit["source_video"] = source_video
|
||||
if source_audio is not None:
|
||||
edit["source_audio"] = source_audio
|
||||
if video_mask is not None:
|
||||
edit["video_mask"] = video_mask
|
||||
if audio_mask >= 0.0:
|
||||
edit["audio_mask"] = audio_mask
|
||||
return (edit,)
|
||||
|
||||
|
||||
class VLLMOmniMiniMaxH3TemporalMask:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"images": ("IMAGE",),
|
||||
"source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}),
|
||||
"duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}),
|
||||
"mode": (["continuation", "extension"],),
|
||||
"preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK")
|
||||
RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask")
|
||||
FUNCTION = "build"
|
||||
CATEGORY = "vLLM-Omni"
|
||||
|
||||
def build(self, images, source_fps, duration, mode, preserve_fraction=0.5):
|
||||
if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0:
|
||||
raise ValueError("Source must contain frames and have a positive finite FPS.")
|
||||
if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1:
|
||||
raise ValueError("Invalid duration or preserve_fraction.")
|
||||
frames = _align_frame_count(max(1, round(duration * 24)))
|
||||
source_seconds = images.shape[0] / source_fps
|
||||
if mode == "extension":
|
||||
if frames / 24 <= source_seconds:
|
||||
raise ValueError("Extension output must be longer than the source; increase duration.")
|
||||
boundary = source_seconds
|
||||
elif mode == "continuation":
|
||||
boundary = min(source_seconds, frames / 24) * preserve_fraction
|
||||
else:
|
||||
raise ValueError(f"Unknown temporal mask mode: {mode}")
|
||||
available = min(frames, math.floor(boundary * 24 + 1e-8))
|
||||
prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17)
|
||||
preserved = _video_latent_t(prefix) if prefix else 0
|
||||
total = _video_latent_t(frames)
|
||||
mask = torch.ones(total, 1, 1)
|
||||
mask[:preserved] = 0
|
||||
indices = torch.arange(frames, device=images.device)
|
||||
source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1)
|
||||
preview_images = images.index_select(0, source_indices)
|
||||
preview_mask = mask.index_select(0, torch.arange(frames) * total // frames)
|
||||
return mask, 24.0, preview_images, preview_mask
|
||||
@@ -30,6 +30,7 @@ from .format import (
|
||||
video_to_base64,
|
||||
video_to_bytes,
|
||||
)
|
||||
from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json
|
||||
from .logger import get_logger, pretty_printer
|
||||
from .models import lookup_model_spec
|
||||
from .types import (
|
||||
@@ -283,6 +284,7 @@ class VLLMOmniClient:
|
||||
sampling_params: dict | None = None,
|
||||
model_params: dict | None = None,
|
||||
lora: dict | None = None,
|
||||
latent_edit: dict | None = None,
|
||||
spec_model: str | None = None,
|
||||
**extra_params,
|
||||
) -> VideoInput:
|
||||
@@ -386,6 +388,53 @@ class VLLMOmniClient:
|
||||
content_type="image/png",
|
||||
)
|
||||
|
||||
# === latent-mask editing (MiniMax H3) ===
|
||||
if latent_edit is not None:
|
||||
source_video = latent_edit.get("source_video")
|
||||
source_audio = latent_edit.get("source_audio")
|
||||
video_mask = latent_edit.get("video_mask")
|
||||
audio_mask = latent_edit.get("audio_mask")
|
||||
|
||||
if video_mask is None and audio_mask is None:
|
||||
raise ValueError("Latent-mask editing requires at least one mask.")
|
||||
|
||||
video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item())
|
||||
audio_mask_trivial = audio_mask is None or audio_mask == 1.0
|
||||
if not video_mask_trivial and source_video is None:
|
||||
raise ValueError("A non-trivial video mask requires a source video.")
|
||||
if not audio_mask_trivial and source_audio is None and source_video is None:
|
||||
raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.")
|
||||
|
||||
if source_video is not None:
|
||||
form.add_field(
|
||||
"source_video",
|
||||
video_to_bytes(source_video, "source.mp4"),
|
||||
filename="source.mp4",
|
||||
content_type="video/mp4",
|
||||
)
|
||||
if source_audio is not None:
|
||||
form.add_field(
|
||||
"source_audio",
|
||||
audio_to_bytes(source_audio, "source_audio.mp3"),
|
||||
filename="source_audio.mp3",
|
||||
content_type="audio/mpeg",
|
||||
)
|
||||
if video_mask is not None:
|
||||
mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames)
|
||||
form.add_field(
|
||||
"video_noise_mask",
|
||||
mask_json.encode("utf-8"),
|
||||
filename="video-mask.json",
|
||||
content_type="application/json",
|
||||
)
|
||||
if audio_mask is not None:
|
||||
form.add_field(
|
||||
"audio_noise_mask",
|
||||
scalar_mask_to_json(audio_mask).encode("utf-8"),
|
||||
filename="audio-mask.json",
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
if len(keyframe_images) == 2:
|
||||
for image_filename, image in keyframe_images:
|
||||
form.add_field(
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import json
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def scalar_mask_to_json(value: float) -> str:
|
||||
if not 0.0 <= value <= 1.0:
|
||||
raise ValueError(f"mask value must be in [0, 1], got {value}")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _align_frame_count(frame_count: int) -> int:
|
||||
if frame_count <= 0:
|
||||
return 1
|
||||
current = int(frame_count)
|
||||
while current % 17 != 5:
|
||||
current += 1
|
||||
return current
|
||||
|
||||
|
||||
def _video_latent_t(frame_count: int) -> int:
|
||||
if frame_count <= 5:
|
||||
return 2
|
||||
return ((int(frame_count) - 5) // 17) * 5 + 2
|
||||
|
||||
|
||||
def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor:
|
||||
tv = _video_latent_t(_align_frame_count(num_frames))
|
||||
height = int(height) // 32 * 32
|
||||
width = int(width) // 32 * 32
|
||||
gh, gw = height // 16, width // 16
|
||||
|
||||
if mask.ndim == 2:
|
||||
mask = mask.unsqueeze(0)
|
||||
elif mask.ndim != 3:
|
||||
raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D")
|
||||
|
||||
grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1)
|
||||
if grid.shape[0] == 1:
|
||||
grid = grid.expand(tv, gh, gw)
|
||||
elif grid.shape[0] != tv:
|
||||
grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0)
|
||||
return grid
|
||||
|
||||
|
||||
def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str:
|
||||
return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist())
|
||||
@@ -40,6 +40,10 @@ class VideoReferences(dict):
|
||||
pass
|
||||
|
||||
|
||||
class LatentMaskEditing(dict):
|
||||
pass
|
||||
|
||||
|
||||
class FastH3Deployment(dict):
|
||||
"""Descriptor for a server that fused FastH3 at startup."""
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# WF-05: MiniMax-H3 latent-mask editing
|
||||
|
||||
Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case.
|
||||
|
||||
## Requirements
|
||||
|
||||
- ComfyUI with ComfyUI-vLLM-Omni installed.
|
||||
- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup.
|
||||
|
||||
## Use
|
||||
|
||||
Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run.
|
||||
|
||||
1. Upload your source video in Load Video.
|
||||
2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate.
|
||||
3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip).
|
||||
4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases.
|
||||
+499
@@ -0,0 +1,499 @@
|
||||
{
|
||||
"id": "latent-mask-editing-7380",
|
||||
"revision": 0,
|
||||
"last_node_id": 7,
|
||||
"last_link_id": 5,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
40,
|
||||
20
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
210
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Latent-mask editing server prerequisite",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#000"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "LoadVideo",
|
||||
"pos": [
|
||||
40,
|
||||
280
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
120
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "file",
|
||||
"name": "file",
|
||||
"type": "COMBO",
|
||||
"widget": {
|
||||
"name": "file"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "LoadImageMask",
|
||||
"pos": [
|
||||
40,
|
||||
460
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "image",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "image"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "channel",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "channel"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "mask",
|
||||
"name": "mask",
|
||||
"type": "MASK",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadImageMask"
|
||||
},
|
||||
"widgets_values": [
|
||||
"",
|
||||
"red"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "VLLMOmniLatentMaskEditing",
|
||||
"pos": [
|
||||
560,
|
||||
260
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
300
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "source_video",
|
||||
"name": "source_video",
|
||||
"type": "VIDEO",
|
||||
"link": 1,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "source_audio",
|
||||
"name": "source_audio",
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "video_mask",
|
||||
"name": "video_mask",
|
||||
"type": "MASK",
|
||||
"link": 2,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "audio_mask",
|
||||
"name": "audio_mask",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "audio_mask"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "latent_edit",
|
||||
"name": "latent_edit",
|
||||
"type": "LATENT_MASK_EDITING",
|
||||
"links": [
|
||||
3
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniLatentMaskEditing"
|
||||
},
|
||||
"widgets_values": [
|
||||
0.5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "VLLMOmniMiniMaxH3Params",
|
||||
"pos": [
|
||||
40,
|
||||
680
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
140
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "audio_flow_shift",
|
||||
"name": "audio_flow_shift",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "audio_flow_shift"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "flow_shift",
|
||||
"name": "flow_shift",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "flow_shift"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video params",
|
||||
"name": "video params",
|
||||
"type": "VIDEO_PARAMS",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniMiniMaxH3Params"
|
||||
},
|
||||
"widgets_values": [
|
||||
3.0,
|
||||
12.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "VLLMOmniGenerateVideo",
|
||||
"pos": [
|
||||
1080,
|
||||
200
|
||||
],
|
||||
"size": [
|
||||
540,
|
||||
680
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "frame",
|
||||
"name": "frame",
|
||||
"type": "IMAGE",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "references",
|
||||
"name": "references",
|
||||
"type": "VIDEO_REFERENCES",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "sampling_params",
|
||||
"name": "sampling_params",
|
||||
"type": "SAMPLING_PARAMS",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "lora",
|
||||
"name": "lora",
|
||||
"type": "REMOTE_LORA",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "model_params",
|
||||
"name": "model_params",
|
||||
"type": "VIDEO_PARAMS",
|
||||
"link": 4,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "fast_h3",
|
||||
"name": "fast_h3",
|
||||
"type": "FASTH3_DEPLOYMENT",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "latent_edit",
|
||||
"name": "latent_edit",
|
||||
"type": "LATENT_MASK_EDITING",
|
||||
"link": 3,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "url",
|
||||
"name": "url",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "url"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "model",
|
||||
"name": "model",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "model"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "prompt",
|
||||
"name": "prompt",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "prompt"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "negative_prompt",
|
||||
"name": "negative_prompt",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "negative_prompt"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "width",
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "width"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "height",
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "height"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "fps",
|
||||
"name": "fps",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "fps"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "duration",
|
||||
"name": "duration",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "duration"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniGenerateVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
"http://127.0.0.1:8000/v1",
|
||||
"MiniMaxAI/MiniMax-H3",
|
||||
"Partially restyle the clip and soundtrack",
|
||||
"",
|
||||
160,
|
||||
120,
|
||||
24,
|
||||
4.458
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "SaveVideo",
|
||||
"pos": [
|
||||
1700,
|
||||
300
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"link": 5,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "filename_prefix",
|
||||
"name": "filename_prefix",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "filename_prefix"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "format",
|
||||
"name": "format",
|
||||
"type": "COMFY_DYNAMICCOMBO_V3",
|
||||
"widget": {
|
||||
"name": "format"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": []
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SaveVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
"vllm-omni/latent-edit",
|
||||
"mp4"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
4,
|
||||
0,
|
||||
"VIDEO"
|
||||
],
|
||||
[
|
||||
2,
|
||||
3,
|
||||
0,
|
||||
4,
|
||||
2,
|
||||
"MASK"
|
||||
],
|
||||
[
|
||||
3,
|
||||
4,
|
||||
0,
|
||||
6,
|
||||
6,
|
||||
"LATENT_MASK_EDITING"
|
||||
],
|
||||
[
|
||||
4,
|
||||
5,
|
||||
0,
|
||||
6,
|
||||
4,
|
||||
"VIDEO_PARAMS"
|
||||
],
|
||||
[
|
||||
5,
|
||||
6,
|
||||
0,
|
||||
7,
|
||||
0,
|
||||
"VIDEO"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
+560
@@ -0,0 +1,560 @@
|
||||
{
|
||||
"last_node_id": 9,
|
||||
"last_link_id": 7,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 2,
|
||||
"type": "LoadVideo",
|
||||
"pos": [
|
||||
40,
|
||||
100
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
120
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "file",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "file"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "SolidMask",
|
||||
"pos": [
|
||||
40,
|
||||
300
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "value"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "width"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "height"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "mask",
|
||||
"type": "MASK",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SolidMask"
|
||||
},
|
||||
"widgets_values": [
|
||||
0.0,
|
||||
160,
|
||||
120
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "SolidMask",
|
||||
"pos": [
|
||||
40,
|
||||
500
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "value"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "width"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "height"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "mask",
|
||||
"type": "MASK",
|
||||
"links": [
|
||||
3
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SolidMask"
|
||||
},
|
||||
"widgets_values": [
|
||||
1.0,
|
||||
160,
|
||||
120
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "BatchMasksNode",
|
||||
"pos": [
|
||||
250,
|
||||
380
|
||||
],
|
||||
"size": [
|
||||
200,
|
||||
100
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "masks.mask0",
|
||||
"type": "MASK",
|
||||
"link": 2,
|
||||
"slot_index": 0
|
||||
},
|
||||
{
|
||||
"name": "masks.mask1",
|
||||
"type": "MASK",
|
||||
"link": 3,
|
||||
"slot_index": 1
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "MASK",
|
||||
"type": "MASK",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "BatchMasksNode"
|
||||
},
|
||||
"widgets_values": []
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "VLLMOmniLatentMaskEditing",
|
||||
"pos": [
|
||||
450,
|
||||
180
|
||||
],
|
||||
"size": [
|
||||
400,
|
||||
120
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source_video",
|
||||
"type": "VIDEO",
|
||||
"link": 1
|
||||
},
|
||||
{
|
||||
"name": "source_audio",
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "video_mask",
|
||||
"type": "MASK",
|
||||
"link": 4
|
||||
},
|
||||
{
|
||||
"name": "audio_mask",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "audio_mask"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "latent_edit",
|
||||
"type": "LATENT_MASK_EDITING",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniLatentMaskEditing"
|
||||
},
|
||||
"widgets_values": [
|
||||
-1.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "VLLMOmniMiniMaxH3Params",
|
||||
"pos": [
|
||||
40,
|
||||
700
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "audio_flow_shift",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "audio_flow_shift"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "flow_shift",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "flow_shift"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "video params",
|
||||
"type": "VIDEO_PARAMS",
|
||||
"links": [
|
||||
6
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniMiniMaxH3Params"
|
||||
},
|
||||
"widgets_values": [
|
||||
3.0,
|
||||
12.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "VLLMOmniGenerateVideo",
|
||||
"pos": [
|
||||
700,
|
||||
180
|
||||
],
|
||||
"size": [
|
||||
450,
|
||||
380
|
||||
],
|
||||
"flags": {},
|
||||
"order": 7,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "frame",
|
||||
"type": "IMAGE",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "references",
|
||||
"type": "VIDEO_REFERENCES",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "sampling_params",
|
||||
"type": "SAMPLING_PARAMS",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "lora",
|
||||
"type": "LORA",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "model_params",
|
||||
"type": "VIDEO_PARAMS",
|
||||
"link": 6
|
||||
},
|
||||
{
|
||||
"name": "fast_h3",
|
||||
"type": "FAST_H3",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "latent_edit",
|
||||
"type": "LATENT_MASK_EDITING",
|
||||
"link": 5
|
||||
},
|
||||
{
|
||||
"name": "url",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "url"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "model",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "model"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "prompt",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "prompt"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "negative_prompt",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "negative_prompt"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "width"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "height"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "fps",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "fps"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "duration",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "duration"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
7
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniGenerateVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
"http://127.0.0.1:8000/v1",
|
||||
"MiniMaxAI/MiniMax-H3",
|
||||
"temporal mask: regenerate the ending",
|
||||
"",
|
||||
160,
|
||||
120,
|
||||
24,
|
||||
4.458
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "SaveVideo",
|
||||
"pos": [
|
||||
900,
|
||||
180
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
200
|
||||
],
|
||||
"flags": {},
|
||||
"order": 8,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"link": 7
|
||||
},
|
||||
{
|
||||
"name": "filename_prefix",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "filename_prefix"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"name": "format",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "format"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": []
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SaveVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
"temporal_mask",
|
||||
"mp4"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
6,
|
||||
0,
|
||||
"VIDEO"
|
||||
],
|
||||
[
|
||||
2,
|
||||
3,
|
||||
0,
|
||||
5,
|
||||
0,
|
||||
"MASK"
|
||||
],
|
||||
[
|
||||
3,
|
||||
4,
|
||||
0,
|
||||
5,
|
||||
1,
|
||||
"MASK"
|
||||
],
|
||||
[
|
||||
4,
|
||||
5,
|
||||
0,
|
||||
6,
|
||||
2,
|
||||
"MASK"
|
||||
],
|
||||
[
|
||||
5,
|
||||
6,
|
||||
0,
|
||||
8,
|
||||
6,
|
||||
"LATENT_MASK_EDITING"
|
||||
],
|
||||
[
|
||||
6,
|
||||
7,
|
||||
0,
|
||||
8,
|
||||
4,
|
||||
"VIDEO_PARAMS"
|
||||
],
|
||||
[
|
||||
7,
|
||||
8,
|
||||
0,
|
||||
9,
|
||||
0,
|
||||
"VIDEO"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
{
|
||||
"id": "latent-mask-editing-7380",
|
||||
"revision": 0,
|
||||
"last_node_id": 7,
|
||||
"last_link_id": 5,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "MarkdownNote",
|
||||
"pos": [
|
||||
40,
|
||||
20
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
210
|
||||
],
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [],
|
||||
"title": "Latent-mask editing server prerequisite",
|
||||
"properties": {},
|
||||
"widgets_values": [
|
||||
"Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends."
|
||||
],
|
||||
"color": "#432",
|
||||
"bgcolor": "#000"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "LoadVideo",
|
||||
"pos": [
|
||||
40,
|
||||
280
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
120
|
||||
],
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "file",
|
||||
"name": "file",
|
||||
"type": "COMBO",
|
||||
"widget": {
|
||||
"name": "file"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "LoadVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
""
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "SolidMask",
|
||||
"pos": [
|
||||
40,
|
||||
460
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "value",
|
||||
"name": "value",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "value"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "width",
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "width"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "height",
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "height"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "mask",
|
||||
"name": "mask",
|
||||
"type": "MASK",
|
||||
"links": [
|
||||
2
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SolidMask"
|
||||
},
|
||||
"widgets_values": [
|
||||
1.0,
|
||||
160,
|
||||
120
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "VLLMOmniLatentMaskEditing",
|
||||
"pos": [
|
||||
560,
|
||||
260
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
300
|
||||
],
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "source_video",
|
||||
"name": "source_video",
|
||||
"type": "VIDEO",
|
||||
"link": 1,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "source_audio",
|
||||
"name": "source_audio",
|
||||
"type": "AUDIO",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "video_mask",
|
||||
"name": "video_mask",
|
||||
"type": "MASK",
|
||||
"link": 2,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "audio_mask",
|
||||
"name": "audio_mask",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "audio_mask"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "latent_edit",
|
||||
"name": "latent_edit",
|
||||
"type": "LATENT_MASK_EDITING",
|
||||
"links": [
|
||||
3
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniLatentMaskEditing"
|
||||
},
|
||||
"widgets_values": [
|
||||
0.5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "VLLMOmniMiniMaxH3Params",
|
||||
"pos": [
|
||||
40,
|
||||
680
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
140
|
||||
],
|
||||
"flags": {},
|
||||
"order": 4,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "audio_flow_shift",
|
||||
"name": "audio_flow_shift",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "audio_flow_shift"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "flow_shift",
|
||||
"name": "flow_shift",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "flow_shift"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video params",
|
||||
"name": "video params",
|
||||
"type": "VIDEO_PARAMS",
|
||||
"links": [
|
||||
4
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniMiniMaxH3Params"
|
||||
},
|
||||
"widgets_values": [
|
||||
3.0,
|
||||
12.0
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "VLLMOmniGenerateVideo",
|
||||
"pos": [
|
||||
1080,
|
||||
200
|
||||
],
|
||||
"size": [
|
||||
540,
|
||||
680
|
||||
],
|
||||
"flags": {},
|
||||
"order": 5,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "frame",
|
||||
"name": "frame",
|
||||
"type": "IMAGE",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "references",
|
||||
"name": "references",
|
||||
"type": "VIDEO_REFERENCES",
|
||||
"link": null,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "sampling_params",
|
||||
"name": "sampling_params",
|
||||
"type": "SAMPLING_PARAMS",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "lora",
|
||||
"name": "lora",
|
||||
"type": "REMOTE_LORA",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "model_params",
|
||||
"name": "model_params",
|
||||
"type": "VIDEO_PARAMS",
|
||||
"link": 4,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "fast_h3",
|
||||
"name": "fast_h3",
|
||||
"type": "FASTH3_DEPLOYMENT",
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "latent_edit",
|
||||
"name": "latent_edit",
|
||||
"type": "LATENT_MASK_EDITING",
|
||||
"link": 3,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "url",
|
||||
"name": "url",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "url"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "model",
|
||||
"name": "model",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "model"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "prompt",
|
||||
"name": "prompt",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "prompt"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "negative_prompt",
|
||||
"name": "negative_prompt",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "negative_prompt"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "width",
|
||||
"name": "width",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "width"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "height",
|
||||
"name": "height",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "height"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "fps",
|
||||
"name": "fps",
|
||||
"type": "INT",
|
||||
"widget": {
|
||||
"name": "fps"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "duration",
|
||||
"name": "duration",
|
||||
"type": "FLOAT",
|
||||
"widget": {
|
||||
"name": "duration"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": [
|
||||
5
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VLLMOmniGenerateVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
"http://127.0.0.1:8000/v1",
|
||||
"MiniMaxAI/MiniMax-H3",
|
||||
"Partially restyle the clip and soundtrack",
|
||||
"",
|
||||
160,
|
||||
120,
|
||||
24,
|
||||
4.458
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "SaveVideo",
|
||||
"pos": [
|
||||
1700,
|
||||
300
|
||||
],
|
||||
"size": [
|
||||
420,
|
||||
160
|
||||
],
|
||||
"flags": {},
|
||||
"order": 6,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"link": 5,
|
||||
"shape": 7
|
||||
},
|
||||
{
|
||||
"localized_name": "filename_prefix",
|
||||
"name": "filename_prefix",
|
||||
"type": "STRING",
|
||||
"widget": {
|
||||
"name": "filename_prefix"
|
||||
},
|
||||
"link": null
|
||||
},
|
||||
{
|
||||
"localized_name": "format",
|
||||
"name": "format",
|
||||
"type": "COMFY_DYNAMICCOMBO_V3",
|
||||
"widget": {
|
||||
"name": "format"
|
||||
},
|
||||
"link": null
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"localized_name": "video",
|
||||
"name": "video",
|
||||
"type": "VIDEO",
|
||||
"links": []
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SaveVideo"
|
||||
},
|
||||
"widgets_values": [
|
||||
"vllm-omni/latent-edit",
|
||||
"mp4"
|
||||
]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
4,
|
||||
0,
|
||||
"VIDEO"
|
||||
],
|
||||
[
|
||||
2,
|
||||
3,
|
||||
0,
|
||||
4,
|
||||
2,
|
||||
"MASK"
|
||||
],
|
||||
[
|
||||
3,
|
||||
4,
|
||||
0,
|
||||
6,
|
||||
6,
|
||||
"LATENT_MASK_EDITING"
|
||||
],
|
||||
[
|
||||
4,
|
||||
5,
|
||||
0,
|
||||
6,
|
||||
4,
|
||||
"VIDEO_PARAMS"
|
||||
],
|
||||
[
|
||||
5,
|
||||
6,
|
||||
0,
|
||||
7,
|
||||
0,
|
||||
"VIDEO"
|
||||
]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
+5241
File diff suppressed because it is too large.
Load diff
@@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = {
|
||||
REMOTE_LORA: "deployment",
|
||||
FASTH3_DEPLOYMENT: "deployment",
|
||||
VIDEO_REFERENCES: "media",
|
||||
LATENT_MASK_EDITING: "media",
|
||||
};
|
||||
|
||||
function familyOf(nodeData) {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests.
|
||||
|
||||
Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test
|
||||
can assert exactly what the client serialized, and returns a synthetic MP4.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import uuid
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import Response
|
||||
from starlette.datastructures import UploadFile
|
||||
|
||||
app = FastAPI()
|
||||
_JOBS: dict[str, dict] = {}
|
||||
_STATE_FILE = os.environ.get("MOCK_STATE_FILE")
|
||||
|
||||
|
||||
def _record(fields: dict) -> None:
|
||||
if _STATE_FILE:
|
||||
with open(_STATE_FILE, "w") as f:
|
||||
json.dump(fields, f)
|
||||
|
||||
|
||||
def _synthetic_mp4() -> bytes:
|
||||
return subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=size=320x240:rate=24:duration=1",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:sample_rate=32000:duration=1",
|
||||
"-map",
|
||||
"0:v",
|
||||
"-map",
|
||||
"1:a",
|
||||
"-shortest",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-ar",
|
||||
"32000",
|
||||
"-ac",
|
||||
"2",
|
||||
"-movflags",
|
||||
"frag_keyframe+empty_moov",
|
||||
"-f",
|
||||
"mp4",
|
||||
"pipe:1",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
).stdout
|
||||
|
||||
|
||||
@app.post("/v1/videos")
|
||||
async def create_video(request: Request):
|
||||
form = await request.form()
|
||||
fields = {}
|
||||
for key, value in form.items():
|
||||
if isinstance(value, UploadFile):
|
||||
data = await value.read()
|
||||
fields[key] = {
|
||||
"file": True,
|
||||
"size": len(data),
|
||||
"filename": value.filename,
|
||||
"content_type": value.content_type,
|
||||
}
|
||||
if value.content_type == "application/json":
|
||||
fields[key]["json"] = json.loads(data)
|
||||
else:
|
||||
fields[key] = str(value)
|
||||
_record(fields)
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
_JOBS[job_id] = {"status": "queued"}
|
||||
return {"id": job_id, "status": "queued"}
|
||||
|
||||
|
||||
@app.get("/v1/videos/{job_id}")
|
||||
async def get_video(job_id: str):
|
||||
if job_id not in _JOBS:
|
||||
return {"status": "failed", "error": "unknown job"}
|
||||
return {"id": job_id, "status": "completed"}
|
||||
|
||||
|
||||
@app.get("/v1/videos/{job_id}/content")
|
||||
async def get_video_content(job_id: str):
|
||||
if job_id not in _JOBS:
|
||||
return Response(status_code=404)
|
||||
mp4 = await asyncio.to_thread(_synthetic_mp4)
|
||||
return Response(content=mp4, media_type="video/mp4")
|
||||
|
||||
|
||||
@app.delete("/v1/videos/{job_id}")
|
||||
async def delete_video(job_id: str):
|
||||
_JOBS.pop(job_id, None)
|
||||
return {"deleted": True}
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Unit tests for latent-mask serialization helpers."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_scalar_mask_to_json():
|
||||
assert scalar_mask_to_json(0.5) == "0.5"
|
||||
assert scalar_mask_to_json(1.0) == "1.0"
|
||||
with pytest.raises(ValueError):
|
||||
scalar_mask_to_json(1.5)
|
||||
with pytest.raises(ValueError):
|
||||
scalar_mask_to_json(-0.1)
|
||||
|
||||
|
||||
def test_video_mask_grid_shape():
|
||||
mask = torch.zeros(256, 448)
|
||||
grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107))
|
||||
assert len(grid) == 2 + 5 * ((107 - 5) // 17)
|
||||
assert len(grid[0]) == 256 // 16
|
||||
assert len(grid[0][0]) == 448 // 16
|
||||
|
||||
|
||||
def test_video_mask_grid_shape_non_aligned_frames():
|
||||
# 96 snaps up to 107 (17n+5), so Tv == 32, matching the server.
|
||||
grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96))
|
||||
assert len(grid) == 32
|
||||
|
||||
|
||||
def test_video_mask_grid_shape_short_clip():
|
||||
# num_frames <= 5 -> Tv == 2.
|
||||
grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1))
|
||||
assert len(grid) == 2
|
||||
|
||||
|
||||
def test_video_mask_grid_shape_unaligned_width():
|
||||
# width=500 floors to 480 (multiple of 32) -> gw == 30, not 31.
|
||||
grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22))
|
||||
assert len(grid[0][0]) == 30
|
||||
|
||||
|
||||
def test_video_mask_3d_input():
|
||||
grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107))
|
||||
assert grid[0][0][0] == 1.0
|
||||
|
||||
|
||||
def test_video_mask_temporal():
|
||||
# 3D mask: first half 0 (preserve), second half 1 (regenerate).
|
||||
mask = torch.zeros(7, 64, 64)
|
||||
mask[4:] = 1.0
|
||||
grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22))
|
||||
assert len(grid) == 7
|
||||
assert all(v == 0.0 for row in grid[0] for v in row)
|
||||
assert all(v == 1.0 for row in grid[-1] for v in row)
|
||||
|
||||
|
||||
def test_video_mask_temporal_resize():
|
||||
# 2 temporal slices resize to tv == 7.
|
||||
mask = torch.zeros(2, 64, 64)
|
||||
mask[1] = 1.0
|
||||
grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22))
|
||||
assert len(grid) == 7
|
||||
|
||||
|
||||
def test_video_mask_uniform_preserved():
|
||||
grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107))
|
||||
assert all(v == 0.5 for row in grid for cell in row for v in cell)
|
||||
|
||||
|
||||
def test_video_mask_values_in_range():
|
||||
grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22))
|
||||
assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell)
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""End-to-end test for latent-mask editing serialization.
|
||||
|
||||
Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload
|
||||
against a mock ``/v1/videos`` server and asserts the multipart fields the
|
||||
client produced. Runs on CPU: the ComfyUI ``comfy_api`` / ``comfy_extras``
|
||||
modules are mocked by this directory's ``conftest.py``.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from comfy_api.input import VideoInput
|
||||
from comfyui_vllm_omni.utils.api_client import VLLMOmniClient
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_server(tmp_path):
|
||||
port = _free_port()
|
||||
state_file = tmp_path / "state.json"
|
||||
env = dict(os.environ, MOCK_STATE_FILE=str(state_file))
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "uvicorn", "mock_videos_server:app", "--host", "127.0.0.1", "--port", str(port)],
|
||||
cwd=_TESTS_DIR,
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
deadline = time.time() + 30
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=1):
|
||||
break
|
||||
except OSError:
|
||||
time.sleep(0.2)
|
||||
yield f"http://127.0.0.1:{port}/v1", state_file
|
||||
finally:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=10)
|
||||
|
||||
|
||||
def test_latent_edit_serialization(mock_server):
|
||||
base_url, state_file = mock_server
|
||||
|
||||
# A non-trivial video mask requires a source video. The mocked VideoInput
|
||||
# only needs to provide ``save_to`` for the client's multipart upload.
|
||||
source_video = VideoInput(b"mock_source_video")
|
||||
mask = torch.zeros(1, 120, 160)
|
||||
latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5}
|
||||
|
||||
async def run():
|
||||
client = VLLMOmniClient(base_url)
|
||||
return await client.generate_video(
|
||||
model="MiniMaxAI/MiniMax-H3",
|
||||
prompt="restyle the clip",
|
||||
width=160,
|
||||
height=120,
|
||||
num_frames=22,
|
||||
fps=24,
|
||||
latent_edit=latent_edit,
|
||||
)
|
||||
|
||||
out = asyncio.run(run())
|
||||
assert out is not None
|
||||
|
||||
with open(state_file) as f:
|
||||
fields = json.load(f)
|
||||
|
||||
assert fields["source_video"]["file"] is True
|
||||
assert fields["source_video"]["content_type"] == "video/mp4"
|
||||
for name in ("video_noise_mask", "audio_noise_mask"):
|
||||
assert fields[name]["file"] is True
|
||||
assert fields[name]["content_type"] == "application/json"
|
||||
assert fields["audio_noise_mask"]["json"] == 0.5
|
||||
|
||||
video_mask = fields["video_noise_mask"]["json"]
|
||||
assert len(video_mask) == 7 # num_frames=22 aligns to 22 (17n+5) -> Tv = 7
|
||||
assert len(video_mask[0]) == 6 # height 120 floors to 96 (multiple of 32) -> gh 6
|
||||
assert len(video_mask[0][0]) == 10 # width 160 -> gw 10
|
||||
Reference in new issue
Block a user