anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Frontend] Add ComfyUI support for r2v (MiniMax H3 as example) (#5756)

Browse Source 
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Zeyu Huang | 黃澤宇andCopilot Autofix powered by AI authored and GitHub committed 2026-08-04 11:51:12 +00:00
1 parent b9a51592f0
commit b6728db486
9 files changed
+413 -539
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/ComfyUI-vLLM-Omni/README.md
+7 -3
View File
@@ -47,7 +47,7 @@ If no, check your shell running the ComfyUI process. There may be some error mes
This extension offers the following nodes based on the output modalities (at **ComfyUI sidebar -> Node Library**):
- **Generate Image** for text-to-image and image-to-image tasks
- **Generate Video** for text-to-video and image-to-video tasks
- **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video
- **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
- **TTS** and **TTS Voice Clone** for TTS tasks
@@ -105,7 +105,7 @@ You can configure per-stage sampling parameters for multi-stage models.
> [!TIP]
> The node automatically choose text-to-image or image-to-image API endpoints depending on whether you connect an image input or not.
### Text-to-video and image-to-video generation (e.g., Wan)
### Text-to-video and image-to-video generation (e.g., Wan, MiniMax-H3)
(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Video Generation**)
@@ -117,7 +117,11 @@ You can configure per-stage sampling parameters for multi-stage models.
</p>
> [!TIP]
> The node automatically choose text-to-video or image-to-video API endpoints depending on whether you connect an image input or not.
> Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).
>
> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.
>
> Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect.
### TTS (e.g., Qwen TTS series)
apps/ComfyUI-vLLM-Omni/__init__.py
+6
View File
@@ -15,11 +15,13 @@ from .comfyui_vllm_omni.nodes import (
VLLMOmniDiffusionSampling,
VLLMOmniGenerateImage,
VLLMOmniGenerateVideo,
VLLMOmniMiniMaxH3Params,
VLLMOmniQwenTTSParams,
VLLMOmniRemoteLoRA,
VLLMOmniSamplingParamsList,
VLLMOmniTTS,
VLLMOmniUnderstanding,
VLLMOmniVideoReferences,
VLLMOmniVoiceClone,
VLLMOmniWanParams,
)
@@ -32,6 +34,7 @@ NODE_CLASS_MAPPINGS = {
"VLLMOmniUnderstanding": VLLMOmniUnderstanding,
"VLLMOmniTTS": VLLMOmniTTS,
"VLLMOmniVoiceClone": VLLMOmniVoiceClone,
"VLLMOmniVideoReferences": VLLMOmniVideoReferences,
# === Params ===
"VLLMOmniARSampling": VLLMOmniARSampling,
"VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
@@ -39,6 +42,7 @@ NODE_CLASS_MAPPINGS = {
"VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA,
"VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
"VLLMOmniWanParams": VLLMOmniWanParams,
"VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params,
}
# A dictionary that contains the friendly/humanly readable titles for the nodes
@@ -49,6 +53,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"VLLMOmniUnderstanding": "Multimodality Understanding",
"VLLMOmniTTS": "TTS (Text to Speech)",
"VLLMOmniVoiceClone": "TTS Voice Cloning",
"VLLMOmniVideoReferences": "Video References",
# === Params ===
"VLLMOmniARSampling": "AR Sampling Params",
"VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
@@ -56,6 +61,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"VLLMOmniRemoteLoRA": "LoRA",
"VLLMOmniQwenTTSParams": "Qwen TTS Params",
"VLLMOmniWanParams": "Wan Video Params",
"VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params",
}
WEB_DIRECTORY = "./web"
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py
+95 -5
View File
@@ -10,7 +10,9 @@ from .utils.types import (
AudioFormat,
AutoregressionSamplingParams,
DiffusionSamplingParams,
MiniMaxH3ModelSpecificParams,
QwenTTSModelSpecificParams,
VideoReferences,
WanModelSpecificParams,
)
from .utils.validators import (
@@ -163,7 +165,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
"num_frames": ("INT", {"default": 41, "min": 1}),
},
"optional": {
"image": ("IMAGE",),
"frame": ("IMAGE",),
"references": ("VIDEO_REFERENCES",),
"sampling_params": ("SAMPLING_PARAMS",),
"lora": ("REMOTE_LORA",),
"model_params": ("VIDEO_PARAMS",),
@@ -174,6 +177,15 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
RETURN_NAMES = ("video",)
FUNCTION = "generate"
@classmethod
def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
base = super().VALIDATE_INPUTS(url, model)
if base is not True:
return base
if frame is not None and references is not None:
return "Provide only one of frame or references, not both."
return True
async def generate(
self,
url: str,
@@ -184,7 +196,8 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
fps: int,
num_frames: int,
negative_prompt: str | None = None,
image: torch.Tensor | None = None,
frame: torch.Tensor | None = None,
references: dict | None = None,
sampling_params: dict | list[dict] | None = None,
model_params: dict | None = None,
lora: dict | None = None,
@@ -207,14 +220,16 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
if sampling_params is not None:
sampling_params.pop("type", None) # internal fields
if model_params is not None:
model_params.pop("type", None) # internal fields
# model_params["type"] is also an internal field, but is used in api_client
# if model_params is not None:
# model_params.pop("type", None) # internal fields
client = VLLMOmniClient(url)
output = await client.generate_video(
model=model,
prompt=prompt,
image=image, # image present => i2v, absent => t2v
frame=frame, # frame present => fl2va / Wan I2V
references=references,
width=width,
height=height,
num_frames=num_frames,
@@ -734,3 +749,78 @@ class VLLMOmniWanParams:
def get_params(self, **kwargs):
return (WanModelSpecificParams(kwargs),)
class VLLMOmniMiniMaxH3Params:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"audio_flow_shift": (
"FLOAT",
{"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1},
),
"flow_shift": (
"FLOAT",
{"default": 12.0, "min": 0.0, "max": 100.0, "step": 0.1},
),
}
}
RETURN_TYPES = ("VIDEO_PARAMS",)
RETURN_NAMES = ("MiniMax-H3 video params",)
FUNCTION = "get_params"
CATEGORY = "vLLM-Omni/Video Params"
def get_params(self, **kwargs):
params = MiniMaxH3ModelSpecificParams(kwargs)
params["type"] = "minimax_h3"
return (params,)
class VLLMOmniVideoReferences:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {},
"optional": {
"image_1": ("IMAGE",),
"image_2": ("IMAGE",),
"audio_1": ("AUDIO",),
"audio_2": ("AUDIO",),
"video_1": ("VIDEO",),
"video_2": ("VIDEO",),
},
}
RETURN_TYPES = ("VIDEO_REFERENCES",)
RETURN_NAMES = ("references",)
FUNCTION = "get_references"
CATEGORY = "vLLM-Omni"
def get_references(
self,
image_1: torch.Tensor | None = None,
image_2: torch.Tensor | None = None,
audio_1: AudioInput | None = None,
audio_2: AudioInput | None = None,
video_1: VideoInput | None = None,
video_2: VideoInput | None = None,
**kwargs,
):
if kwargs:
logger.info("Uncaught kwargs: %s", kwargs)
refs = VideoReferences()
if image_1 is not None:
refs["image_1"] = image_1
if image_2 is not None:
refs["image_2"] = image_2
if audio_1 is not None:
refs["audio_1"] = audio_1
if audio_2 is not None:
refs["audio_2"] = audio_2
if video_1 is not None:
refs["video_1"] = video_1
if video_2 is not None:
refs["video_2"] = video_2
return (refs,)
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py
+88 -10
View File
@@ -24,6 +24,7 @@ from .format import (
image_tensor_to_base64,
image_tensor_to_png_bytes,
video_to_base64,
video_to_bytes,
)
from .logger import get_logger, pretty_printer
from .models import lookup_model_spec
@@ -63,7 +64,11 @@ async def url_bytes(session: aiohttp.ClientSession, url: str, verb: str = "get",
class VLLMOmniClient:
def __init__(
self, base_url: str, timeout: float | None = None, poll_interval: float = 5.0, max_poll_duration: float = 60 * 5
self,
base_url: str,
timeout: float | None = None,
poll_interval: float = 5.0,
max_poll_duration: float = 60 * 30,
):
self.base_url = base_url
self.timeout = aiohttp.ClientTimeout(total=timeout)
@@ -261,12 +266,17 @@ class VLLMOmniClient:
num_frames: int,
fps: int,
negative_prompt: str | None = None,
image: torch.Tensor | None = None,
frame: torch.Tensor | None = None,
references: dict | None = None,
sampling_params: dict | None = None,
model_params: dict | None = None,
lora: dict | None = None,
**extra_body,
**extra_params,
) -> VideoInput:
if frame is not None and references is not None:
raise ValueError("Provide only one of frame or references, not both.")
# === regular payload fields ===
form = aiohttp.FormData()
form.add_field("model", model)
form.add_field("prompt", prompt)
@@ -279,23 +289,91 @@ class VLLMOmniClient:
if sampling_params is not None:
for k, v in sampling_params.items():
form.add_field(k, str(v))
if model_params is not None:
for k, v in model_params.items():
form.add_field(k, str(v))
if lora is not None:
form.add_field("lora", json.dumps(lora, ensure_ascii=False))
if extra_body:
form.add_field("extra_body", json.dumps(extra_body, ensure_ascii=False))
if image is not None:
# === multimodal inputs (first-last-frames, references, etc.) ===
input_reference_image: torch.Tensor | None = None
audio_reference: AudioInput | None = None
reference_videos: list[VideoInput] = []
video_task: str | None = None
if frame is not None:
input_reference_image = frame
video_task = "fl2va"
elif references is not None:
images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]
if not images and not audios and not videos:
raise ValueError("references is empty; connect at least one image/audio/video.")
if videos:
if images or audios:
raise ValueError(
"Video references cannot be combined with image or audio references. "
"Connect only video_1/video_2 for multi-video Ref2VA."
)
reference_videos = videos
video_task = "ref2va"
elif len(images) == 1 and len(audios) == 1:
input_reference_image = images[0]
audio_reference = audios[0]
video_task = "ref2va"
else:
raise ValueError(
"Invalid references combination. Supported modes: "
"(1) one or more videos only, or (2) exactly one image and one audio."
)
else:
video_task = "t2va"
if input_reference_image is not None:
image_filename = "image.png" # Required for multipart form
form.add_field(
"input_reference",
image_tensor_to_png_bytes(image, image_filename),
image_tensor_to_png_bytes(input_reference_image, image_filename),
filename=image_filename,
content_type="image/png",
)
if audio_reference is not None:
form.add_field(
"audio_reference",
json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
)
for idx, video in enumerate(reference_videos, start=1):
video_filename = f"reference_{idx}.mp4"
form.add_field(
"input_references",
video_to_bytes(video, video_filename),
filename=video_filename,
content_type="video/mp4",
)
# === model specific params. Either use a specialized builder, or add flattened fields as-is ===
if model_params is not None:
model_params = dict(model_params)
model_params.pop("type", None)
spec, _ = lookup_model_spec(model)
params_builder = spec.get("params_builder") if spec else None
if params_builder is not None:
form_fields = params_builder(
model_params or {},
extra_params={**extra_params, "task": video_task},
)
for k, v in form_fields.items():
form.add_field(k, v if isinstance(v, str) else str(v))
else:
if model_params is not None:
for k, v in model_params.items():
form.add_field(k, str(v))
if extra_params:
form.add_field("extra_params", json.dumps(extra_params, ensure_ascii=False))
async with aiohttp.ClientSession(timeout=self.timeout) as session:
# Start the video generation job
url = f"{self.base_url}/videos"
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py
+57 -6
View File
@@ -1,7 +1,12 @@
import json
import re
from typing import Any
from .logger import get_logger
from .types import Modality, ModelMode, Spec
logger = get_logger(__name__)
def _bagel_payload_preprocessor(payload: dict) -> dict:
try:
@@ -29,6 +34,30 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict:
return payload
def _minimaxh3_params_builder(
model_params: dict[str, Any],
*,
extra_params: dict[str, Any],
) -> dict[str, Any]:
"""Build multipart form fields for MiniMax-H3 from model_params + routed task."""
params = dict(model_params)
params.pop("type", None)
form_fields: dict[str, Any] = {}
merged_extra_params: dict[str, Any] = dict(extra_params or {})
if "flow_shift" in params:
form_fields["flow_shift"] = params.pop("flow_shift")
for key in ("audio_flow_shift",):
if key in params:
merged_extra_params[key] = params.pop(key)
if params:
logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params))
if merged_extra_params:
form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)
return form_fields
_MODEL_PIPELINE_SPECS: dict[str, Spec] = {
r"BAGEL-7B-MoT": {
"stages": [
@@ -71,6 +100,21 @@ _MODEL_PIPELINE_SPECS: dict[str, Spec] = {
}
],
},
r"MiniMax-H3(/FL2VA|/Ref2VA)?": {
"stages": ["diffusion"],
"modes": [
{
"mode": ModelMode.VIDEO_GENERATION,
"input_modalities": [
Modality.TEXT,
Modality.IMAGE,
Modality.VIDEO,
Modality.AUDIO,
],
}
],
"params_builder": _minimaxh3_params_builder,
},
}
# Convert dict keys to regex patterns
MODEL_PIPELINE_SPECS: dict[re.Pattern, Spec] = {}
@@ -80,12 +124,10 @@ del _MODEL_PIPELINE_SPECS
def lookup_model_spec(model: str) -> tuple[Spec | None, str | None]:
try:
last_component = model.rstrip("/").rsplit("/", 1)[-1]
except IndexError:
last_component = model
normalized = model.rstrip("/").lstrip("/")
lookup_key = normalized.split("/", 1)[-1]
for pattern, spec in MODEL_PIPELINE_SPECS.items():
if pattern.search(last_component):
if pattern.search(lookup_key):
return spec, pattern.pattern
return None, None
@@ -101,6 +143,9 @@ if __name__ == "__main__":
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
"Custom/Path/UnknownModel-Instruct",
"Not/Matching/Anything",
"/path/to/MiniMax-H3/FL2VA",
"/path/to/MiniMax-H3/Ref2VA",
"MiniMaxAI/MiniMax-H3",
]
test_payload = {"messages": [{"role": "user", "content": "prompt"}]}
@@ -112,7 +157,13 @@ if __name__ == "__main__":
if preprocessor := spec.get("payload_preprocessor"):
result = preprocessor(test_payload)
print(f"✓ {path:<40} → {result}")
elif params_builder := spec.get("params_builder"):
result = params_builder(
{"audio_flow_shift": 3.0, "flow_shift": 12.0, "type": "minimax_h3"},
extra_params={"task": "t2va"},
)
print(f"✓ {path:<40} → {result}")
else:
print(f"✓ {path:<40} → No preprocessor")
print(f"✓ {path:<40} → No preprocessor/params_builder")
else:
print(f"✗ {path:<40} → No match")
apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py
+10
View File
@@ -23,6 +23,14 @@ class WanModelSpecificParams(dict):
pass
class MiniMaxH3ModelSpecificParams(dict):
pass
class VideoReferences(dict):
pass
class ModelMode(Enum):
IMAGE_GENERATION = auto()
VIDEO_GENERATION = auto()
@@ -43,9 +51,11 @@ class ModelModeSpec(TypedDict):
PayloadPreprocessor: TypeAlias = Callable[[dict[str, Any]], dict[str, Any]]
ParamsBuilder: TypeAlias = Callable[..., dict[str, Any]]
class Spec(TypedDict):
stages: list[Literal["diffusion", "autoregression"]]
modes: list[ModelModeSpec]
payload_preprocessor: NotRequired[PayloadPreprocessor]
params_builder: NotRequired[ParamsBuilder]
apps/ComfyUI-vLLM-Omni/docs/images/comfyui-video-generation.jpg
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 129 KiB

After

Width:  |  Height:  |  Size: 161 KiB

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json
+1 -513
View File
@@ -1,513 +1 @@
{
"id": "3aea1d08-7de5-4997-ba71-b2f66ad72a21",
"revision": 0,
"last_node_id": 14,
"last_link_id": 28,
"nodes": [
{
"id": 10,
"type": "MarkdownNote",
"pos": [
227.99306819575278,
-231.24143306069843
],
"size": [
240.3326478922474,
136.3505820791881
],
"flags": {},
"order": 0,
"mode": 0,
"inputs": [],
"outputs": [],
"title": "Note: Task and Input",
"properties": {},
"widgets_values": [
"vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both text-to-video generation and image-to-video generation. The node will create corresponding payload depending on whether an input image is present or not."
],
"color": "#432",
"bgcolor": "#000"
},
{
"id": 4,
"type": "LoadImage",
"pos": [
496.31859627609606,
-229.71277089860084
],
"size": [
270,
314
],
"flags": {},
"order": 1,
"mode": 0,
"inputs": [
{
"localized_name": "image",
"name": "image",
"type": "COMBO",
"widget": {
"name": "image"
},
"link": null
},
{
"localized_name": "choose file to upload",
"name": "upload",
"type": "IMAGEUPLOAD",
"widget": {
"name": "upload"
},
"link": null
}
],
"outputs": [
{
"localized_name": "IMAGE",
"name": "IMAGE",
"type": "IMAGE",
"links": [
25
]
},
{
"localized_name": "MASK",
"name": "MASK",
"type": "MASK",
"links": null
}
],
"properties": {
"Node name for S&R": "LoadImage"
},
"widgets_values": [
"cute-cat.jpg",
"image"
]
},
{
"id": 8,
"type": "VLLMOmniDiffusionSampling",
"pos": [
465.5140218220927,
148.80072646828987
],
"size": [
284.205078125,
226
],
"flags": {},
"order": 2,
"mode": 0,
"inputs": [
{
"localized_name": "n",
"name": "n",
"type": "INT",
"widget": {
"name": "n"
},
"link": null
},
{
"localized_name": "num_inference_steps",
"name": "num_inference_steps",
"type": "INT",
"widget": {
"name": "num_inference_steps"
},
"link": null
},
{
"localized_name": "guidance_scale",
"name": "guidance_scale",
"type": "FLOAT",
"widget": {
"name": "guidance_scale"
},
"link": null
},
{
"localized_name": "true_cfg_scale",
"name": "true_cfg_scale",
"type": "FLOAT",
"widget": {
"name": "true_cfg_scale"
},
"link": null
},
{
"localized_name": "vae_use_slicing",
"name": "vae_use_slicing",
"type": "BOOLEAN",
"widget": {
"name": "vae_use_slicing"
},
"link": null
},
{
"localized_name": "vae_use_tiling",
"name": "vae_use_tiling",
"type": "BOOLEAN",
"widget": {
"name": "vae_use_tiling"
},
"link": null
},
{
"localized_name": "seed",
"name": "seed",
"type": "INT",
"widget": {
"name": "seed"
},
"link": null
}
],
"outputs": [
{
"localized_name": "diffusion sampling params",
"name": "diffusion sampling params",
"type": "SAMPLING_PARAMS",
"links": [
27
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniDiffusionSampling"
},
"widgets_values": [
1,
50,
1,
1,
false,
false,
1992,
"randomize"
]
},
{
"id": 14,
"type": "VLLMOmniWanParams",
"pos": [
518.6005521845747,
428.4455663719664
],
"size": [
270,
106
],
"flags": {},
"order": 3,
"mode": 0,
"inputs": [
{
"localized_name": "guidance_scale_2",
"name": "guidance_scale_2",
"type": "FLOAT",
"widget": {
"name": "guidance_scale_2"
},
"link": null
},
{
"localized_name": "boundary_ratio",
"name": "boundary_ratio",
"type": "FLOAT",
"widget": {
"name": "boundary_ratio"
},
"link": null
},
{
"localized_name": "flow_shift",
"name": "flow_shift",
"type": "FLOAT",
"widget": {
"name": "flow_shift"
},
"link": null
}
],
"outputs": [
{
"localized_name": "Wan video params",
"name": "Wan video params",
"type": "VIDEO_PARAMS",
"links": [
28
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniWanParams"
},
"widgets_values": [
4,
0.875,
5
]
},
{
"id": 12,
"type": "VLLMOmniGenerateVideo",
"pos": [
827.2566336087867,
-73.77449831827545
],
"size": [
400,
346
],
"flags": {},
"order": 4,
"mode": 0,
"inputs": [
{
"localized_name": "image",
"name": "image",
"shape": 7,
"type": "IMAGE",
"link": 25
},
{
"localized_name": "sampling_params",
"name": "sampling_params",
"shape": 7,
"type": "SAMPLING_PARAMS",
"link": 27
},
{
"localized_name": "lora",
"name": "lora",
"shape": 7,
"type": "REMOTE_LORA",
"link": null
},
{
"localized_name": "model_params",
"name": "model_params",
"shape": 7,
"type": "VIDEO_PARAMS",
"link": 28
},
{
"localized_name": "url",
"name": "url",
"type": "STRING",
"widget": {
"name": "url"
},
"link": null
},
{
"localized_name": "model",
"name": "model",
"type": "STRING",
"widget": {
"name": "model"
},
"link": null
},
{
"localized_name": "prompt",
"name": "prompt",
"type": "STRING",
"widget": {
"name": "prompt"
},
"link": null
},
{
"localized_name": "negative_prompt",
"name": "negative_prompt",
"type": "STRING",
"widget": {
"name": "negative_prompt"
},
"link": null
},
{
"localized_name": "width",
"name": "width",
"type": "INT",
"widget": {
"name": "width"
},
"link": null
},
{
"localized_name": "height",
"name": "height",
"type": "INT",
"widget": {
"name": "height"
},
"link": null
},
{
"localized_name": "fps",
"name": "fps",
"type": "INT",
"widget": {
"name": "fps"
},
"link": null
},
{
"localized_name": "num_frames",
"name": "num_frames",
"type": "INT",
"widget": {
"name": "num_frames"
},
"link": null
}
],
"outputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"links": [
26
]
}
],
"properties": {
"Node name for S&R": "VLLMOmniGenerateVideo"
},
"widgets_values": [
"http://localhost:8000/v1",
"Wan-AI/Wan2.2-TI2V-5B-Diffusers",
"Make the cat in the attached video yawning. Eye closed and mouth wide open. Keep the original realistic style.",
"Cartoonish",
900,
1200,
12,
41
]
},
{
"id": 13,
"type": "SaveVideo",
"pos": [
1257.5441369501657,
-44.12956394419863
],
"size": [
374,
372.34896826553796
],
"flags": {},
"order": 5,
"mode": 0,
"inputs": [
{
"localized_name": "video",
"name": "video",
"type": "VIDEO",
"link": 26
},
{
"localized_name": "filename_prefix",
"name": "filename_prefix",
"type": "STRING",
"widget": {
"name": "filename_prefix"
},
"link": null
},
{
"localized_name": "format",
"name": "format",
"type": "COMBO",
"widget": {
"name": "format"
},
"link": null
},
{
"localized_name": "codec",
"name": "codec",
"type": "COMBO",
"widget": {
"name": "codec"
},
"link": null
}
],
"outputs": [],
"properties": {},
"widgets_values": [
"video/ComfyUI",
"auto",
"auto"
]
}
],
"links": [
[
25,
4,
0,
12,
0,
"IMAGE"
],
[
26,
12,
0,
13,
0,
"VIDEO"
],
[
27,
8,
0,
12,
1,
"SAMPLING_PARAMS"
],
[
28,
14,
0,
12,
3,
"VIDEO_PARAMS"
]
],
"groups": [
{
"id": 1,
"title": "Input",
"bounding": [
213.1706010087147,
-313.20095750667554,
560.8246471437027,
407.4246532472182
],
"color": "#3f789e",
"font_size": 24,
"flags": {}
}
],
"config": {},
"extra": {
"workflowRendererVersion": "LG",
"ds": {
"scale": 1.1469075819486894,
"offset": [
101.7632950847089,
492.29121889347806
]
}
},
"version": 0.4
}
{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scLine truncated