From e8f45fcd891aec0ccc1a402fcd6383460db1dc99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Mon, 14 Sep 2026 11:22:05 +0800 Subject: [PATCH] [Frontend] Add ComfyUI FastH3 node, fix t2va aspect ratio and dropped audio (#7456) Signed-off-by: princepride Co-authored-by: Claude Opus 5 --- apps/ComfyUI-vLLM-Omni/README.md | 66 +++- apps/ComfyUI-vLLM-Omni/__init__.py | 9 +- .../comfyui_vllm_omni/nodes.py | 134 +++++++- .../comfyui_vllm_omni/utils/api_client.py | 14 +- .../comfyui_vllm_omni/utils/format.py | 7 + .../comfyui_vllm_omni/utils/models.py | 27 ++ .../comfyui_vllm_omni/utils/types.py | 7 + .../vLLM-Omni FastH3 Text to Video.json | 322 ++++++++++++++++++ .../vLLM-Omni Video Generation.json | 2 +- apps/ComfyUI-vLLM-Omni/web/main.js | 146 +++++++- apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg | 5 + docs/features/comfyui.md | 19 +- .../comfyui/test_comfyui_integration.py | 88 ++++- 13 files changed, 823 insertions(+), 23 deletions(-) create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json create mode 100644 apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index cbd53a9a0..0d607b30a 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -13,7 +13,6 @@ It can send model inference requests to either a locally running vLLM-Omni servi > [!TIP] > If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them. - ## Installation Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`. @@ -21,9 +20,10 @@ Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Y If you are running ComfyUI during copying, you should restart ComfyUI to load this extension. > [!TIP] -> You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info. +> You can use utility websites such as to download a subdirectory of a repo. Also checkout community discussions (e.g., ) for more info. On the device and virtual environment you run ComfyUI, launch ComfyUI with + ```bash cd ComfyUI @@ -35,6 +35,7 @@ python main.py --cpu ``` On the device and virtual environment you run vLLM-Omni, start a model service with + ```bash vllm serve The_Model_ID_to_Serve --omni --port 8000 ``` @@ -48,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C - **Generate Image** for text-to-image and image-to-image tasks - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video +- **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks @@ -56,6 +58,20 @@ This extension also offers example workflows (at **ComfyUI sidebar -> Templates > [!NOTE] > The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support. +Every node carries the vLLM-Omni mark in its title bar and is tinted by what it outputs, so a graph is readable at a glance: + +| Colour | Nodes | What they produce | +| --- | --- | --- | +| Blue | Generate Image, Generate Video, Multimodality Understanding, TTS, TTS Voice Clone | A generated image, video, audio, or text. These are the only nodes that reach a server. | +| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model | +| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts | +| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded | +| Teal | Video References | Reference media | + +Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept. + +**Generate Video** takes a clip length in seconds (`duration`), not a frame count. Frames stay the wire unit and are derived with the node's `fps`, so the length is always measured against the rate that is actually served; models that accept only certain frame counts still round to their own lattice server-side. Graphs saved before this widget existed stored `num_frames` in its place and are converted on load, using the fps recorded alongside it -- the browser console names every node it rewrites. + To build a simple workflow yourself, - Drag a generation node onto the canvas. @@ -123,6 +139,40 @@ You can configure per-stage sampling parameters for multi-stage models. > > Do not use `frame` and `references` together. Task routing is automatic from which inputs you connect. +#### FastH3 text-to-video + +FastH3 is fused into MiniMax-H3 when the vLLM-Omni server starts; it is not a request-switchable LoRA. Download one adapter variant and start a non-offloaded FL2VA server. For the Dense / Data-Free profile: + +```bash +export FASTH3_DIR=/path/to/fasth3 +hf download FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA \ + dense-datafree/adapter_model.safetensors --local-dir "${FASTH3_DIR}" + +vllm serve /path/to/MiniMax-H3 \ + --omni \ + --task-type fl2va \ + --lora-path "${FASTH3_DIR}/dense-datafree/adapter_model.safetensors" \ + --port 8000 +``` + +For the VSA / Data-Free profile, download `vsa-datafree/adapter_model.safetensors`, use that file as `--lora-path`, and add: + +```bash +--diffusion-attention-backend FASTVIDEO_VSA \ +--fastvideo-vsa-topk 64 +``` + +The VSA profile also requires a compatible `fastvideo-kernel` installation. See the [MiniMax-H3 FastH3 recipe](../../recipes/MiniMaxAI/MiniMax-H3.md#fasth3-adapter) for the full serving contract, supported parallel layouts, and kernel requirements. + +Open the **vLLM-Omni FastH3 Text to Video** template, then: + +- Set the server URL and served model name on **FastH3 Deployment**. +- Connect its output to **Generate Video → fast_h3**. When connected, the deployment node's URL and model take precedence over the corresponding Generate Video widgets. +- Keep `frame`, `references`, **LoRA**, and **MiniMax-H3 Video Params** disconnected. FastH3 Preview v1 supports T2VA only, is already fused, and owns both flow shifts. +- A connected **Diffusion Sampling Params** node may set seed and other ordinary sampling options. The integration always enforces four denoising steps and 24 FPS for FastH3. + +The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server. + ### TTS (e.g., Qwen TTS series) (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**) @@ -152,6 +202,8 @@ You can configure per-stage sampling parameters for multi-stage models. Follow the [development convention and rules of vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/). +Node tints and the title-bar mark are applied in `web/main.js`, keyed off each node's declared output types rather than a list of node names. A new node that returns an existing type is themed with no front-end change; a new output type needs one entry in `FAMILY_BY_OUTPUT` there. + ## Limitation and Non-Goals - Single server mode only. No automatic load balancing or failover. @@ -178,11 +230,11 @@ Whenever you find an issue or problem, please Features -- https://github.com/dougbtv/comfyui-vllm-omni/ The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API. -- https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras ComfyUI's built-in node implementations. +- The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API. +- ComfyUI's built-in node implementations. UI/UX design references -- https://github.com/sgl-project/sglang/pull/15271 SGLang Diffusion's official ComfyUI integration for image and video generation. -- https://github.com/SXQBW/ComfyUI-Qwen-Omni A third party ComfyUI integration for Qwen Omni series. -- https://github.com/flybirdxx/ComfyUI-Qwen-TTS https://github.com/DarioFT/ComfyUI-Qwen3-TTS Third party ComfyUI integrations for Qwen TTS series. +- SGLang Diffusion's official ComfyUI integration for image and video generation. +- A third party ComfyUI integration for Qwen Omni series. +- Third party ComfyUI integrations for Qwen TTS series. diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 268107899..55b003ef1 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -1,4 +1,8 @@ -"""Top-level package for comfyui_vllm_omni.""" # noqa: N999 # This is not a python library intended to be imported +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +# ruff: noqa: N999 + +"""Top-level package for comfyui_vllm_omni.""" # This is not a Python library intended to be imported. __all__ = [ "NODE_CLASS_MAPPINGS", @@ -13,6 +17,7 @@ __version__ = "0.0.1" from .comfyui_vllm_omni.nodes import ( VLLMOmniARSampling, VLLMOmniDiffusionSampling, + VLLMOmniFastH3Deployment, VLLMOmniGenerateImage, VLLMOmniGenerateVideo, VLLMOmniMiniMaxH3Params, @@ -40,6 +45,7 @@ NODE_CLASS_MAPPINGS = { "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling, "VLLMOmniSamplingParamsList": VLLMOmniSamplingParamsList, "VLLMOmniRemoteLoRA": VLLMOmniRemoteLoRA, + "VLLMOmniFastH3Deployment": VLLMOmniFastH3Deployment, "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams, "VLLMOmniWanParams": VLLMOmniWanParams, "VLLMOmniMiniMaxH3Params": VLLMOmniMiniMaxH3Params, @@ -59,6 +65,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "VLLMOmniDiffusionSampling": "Diffusion Sampling Params", "VLLMOmniSamplingParamsList": "Multi-Stage Sampling Params List", "VLLMOmniRemoteLoRA": "LoRA", + "VLLMOmniFastH3Deployment": "FastH3 Deployment", "VLLMOmniQwenTTSParams": "Qwen TTS Params", "VLLMOmniWanParams": "Wan Video Params", "VLLMOmniMiniMaxH3Params": "MiniMax-H3 Video Params", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index 8fd00cd9c..df91ece60 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from typing import Literal import torch @@ -10,6 +13,7 @@ from .utils.types import ( AudioFormat, AutoregressionSamplingParams, DiffusionSamplingParams, + FastH3Deployment, MiniMaxH3ModelSpecificParams, QwenTTSModelSpecificParams, VideoReferences, @@ -22,6 +26,23 @@ from .utils.validators import ( logger = get_logger(__name__) +FASTH3_INFERENCE_STEPS = 4 +FASTH3_FPS = 24 +# A FastH3 deployment may expose H3 under any --served-model-name, so the payload +# spec has to be named outright rather than recovered from that alias. +FASTH3_SPEC_MODEL = "MiniMax-H3" + + +def _resolve_fast_h3_deployment(deployment: dict) -> tuple[str, str]: + if not isinstance(deployment, dict): + raise ValueError("FastH3 deployment must be provided by a FastH3 Deployment node.") + + url = str(deployment.get("url") or "").strip().rstrip("/") + model = str(deployment.get("model") or "").strip() + if not url or not model: + raise ValueError("FastH3 deployment requires both URL and model.") + return url, model + class _VLLMOmniGenerateBase: """Base class for vLLM-Omni generation nodes with shared functionality.""" @@ -162,7 +183,20 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): "width": ("INT", {"default": 832, "min": 1}), "height": ("INT", {"default": 480, "min": 1}), "fps": ("INT", {"default": 16, "min": 1}), - "num_frames": ("INT", {"default": 41, "min": 1}), + "duration": ( + "FLOAT", + { + "default": 4.0, + "min": 0.1, + "step": 0.1, + "round": 0.001, + "tooltip": ( + "Clip length in seconds, converted to frames with the fps above. " + "Models that only accept certain frame counts (e.g. MiniMax-H3) round to " + "their own lattice, so the served clip can be slightly longer than requested." + ), + }, + ), }, "optional": { "frame": ("IMAGE",), @@ -170,6 +204,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): "sampling_params": ("SAMPLING_PARAMS",), "lora": ("REMOTE_LORA",), "model_params": ("VIDEO_PARAMS",), + "fast_h3": ("FASTH3_DEPLOYMENT",), }, } @@ -194,20 +229,66 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): width: int, height: int, fps: int, - num_frames: int, + duration: float, negative_prompt: str | None = None, frame: torch.Tensor | None = None, references: dict | None = None, sampling_params: dict | list[dict] | None = None, model_params: dict | None = None, lora: dict | None = None, + fast_h3: dict | None = None, **kwargs, ): if kwargs: logger.info("Uncaught kwargs: %s", kwargs) logger.debug("Got sampling params: %s", sampling_params) logger.debug("Got model params: %s", model_params) - validate_model_and_sampling_params_types(model, sampling_params) + + # Which spec builds the payload. Only a FastH3 deployment separates this + # from the served name; every other path keeps them equal. + spec_model = model + + if fast_h3 is not None: + if frame is not None or references is not None: + raise ValueError("FastH3 Preview supports T2VA only; disconnect frame and references inputs.") + if lora is not None: + raise ValueError( + "FastH3 is already fused into the selected server; disconnect the request-level LoRA input." + ) + + url, model = _resolve_fast_h3_deployment(fast_h3) + logger.info("Using FastH3 deployment at %s", url) + fps = FASTH3_FPS + # The served name is whatever the operator passed to --served-model-name. + # Left alone, lookup_model_spec would miss H3 for an alias such as + # "fasth3", drop the params builder, and send a t2va request carrying no + # aspect_ratio -- which the server refuses. + spec_model = FASTH3_SPEC_MODEL + + if sampling_params is None: + sampling_params = DiffusionSamplingParams() + elif isinstance(sampling_params, list): + if len(sampling_params) != 1: + raise ValueError("FastH3 expects a single diffusion sampling params group.") + sampling_params = sampling_params[0].__class__(sampling_params[0]) + else: + sampling_params = sampling_params.__class__(sampling_params) + sampling_params["num_inference_steps"] = FASTH3_INFERENCE_STEPS + + # FastH3 owns both modality shifts. Sending the ordinary H3 values + # from a connected H3 Params node would turn a deployment choice + # into a request-level override, which the server intentionally + # rejects when it differs from the fused adapter contract. + if model_params is not None: + model_params = model_params.__class__(model_params) + model_params.pop("flow_shift", None) + model_params.pop("audio_flow_shift", None) + + # Frames stay the wire unit. Convert after the FastH3 branch above, so the + # duration is measured against the fps the server will actually apply. + num_frames = max(1, round(duration * fps)) + + validate_model_and_sampling_params_types(spec_model, sampling_params) # Currently, all video generation models are single-stage diffusion models if isinstance(sampling_params, list): @@ -227,6 +308,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): client = VLLMOmniClient(url) output = await client.generate_video( model=model, + spec_model=spec_model, prompt=prompt, frame=frame, # frame present => fl2va / Wan I2V references=references, @@ -695,6 +777,52 @@ class VLLMOmniRemoteLoRA: return (lora,) +class VLLMOmniFastH3Deployment: + """Select a vLLM-Omni service that fused FastH3 at startup.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "url": ( + "STRING", + { + "default": "http://localhost:8000/v1", + "tooltip": "URL of a vLLM-Omni server started with a FastH3 --lora-path.", + }, + ), + "model": ( + "STRING", + { + "default": "MiniMaxAI/MiniMax-H3", + "tooltip": "Model name exposed by the FastH3 deployment.", + }, + ), + } + } + + RETURN_TYPES = ("FASTH3_DEPLOYMENT",) + RETURN_NAMES = ("deployment",) + FUNCTION = "get_deployment" + CATEGORY = "vLLM-Omni" + DESCRIPTION = ( + "Selects a server with FastH3 fused at startup. The connected Generate Video node uses T2VA, " + "four inference steps, and 24 FPS without sending a request-level LoRA." + ) + + @classmethod + def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]: + try: + _resolve_fast_h3_deployment({"url": url, "model": model}) + except ValueError as exc: + return str(exc) + return True + + def get_deployment(self, url: str, model: str): + url, model = _resolve_fast_h3_deployment({"url": url, "model": model}) + return (FastH3Deployment({"url": url, "model": model}),) + + class VLLMOmniQwenTTSParams: @classmethod def INPUT_TYPES(cls): diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index cd509244a..e051e9a09 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API, and transforms the API responses back to ComfyUI formats. @@ -271,8 +274,15 @@ class VLLMOmniClient: sampling_params: dict | None = None, model_params: dict | None = None, lora: dict | None = None, + spec_model: str | None = None, **extra_params, ) -> VideoInput: + """Post a video job and return the decoded result. + + ``spec_model`` names the model whose payload spec builds the request, for + deployments that serve a known model under a different ``model`` alias. It + never reaches the wire; defaults to ``model``. + """ if frame is not None and references is not None: raise ValueError("Provide only one of frame or references, not both.") @@ -358,12 +368,14 @@ class VLLMOmniClient: model_params = dict(model_params) model_params.pop("type", None) - spec, _ = lookup_model_spec(model) + spec, _ = lookup_model_spec(spec_model or model) params_builder = spec.get("params_builder") if spec else None if params_builder is not None: form_fields = params_builder( model_params or {}, extra_params={**extra_params, "task": video_task}, + width=width, + height=height, ) for k, v in form_fields.items(): form.add_field(k, v if isinstance(v, str) else str(v)) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py index 42d396f06..c2aba4cff 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """Image/tensor format helpers. The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv). @@ -170,6 +173,10 @@ def bytes_to_video(video_bytes: bytes) -> VideoInput: audio_stream = container.streams.audio[-1] audio_frames = [] resampler = AudioResampler(format="fltp") + # Decoding the video stream above left the container at EOF. + # Rewind before demuxing again, or this pass yields no frames + # and the audio of a T2VA result is silently dropped. + container.seek(0) for frame in container.decode(audio_stream): if not isinstance(frame, AudioFrame): continue diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py index 3a37db16d..82c7e4a51 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import json import re from typing import Any @@ -34,10 +37,28 @@ def _qwen25_payload_preprocessor(payload: dict) -> dict: return payload +MINIMAX_H3_ASPECT_RATIOS = { + "21:9": 21.0 / 9.0, + "16:9": 16.0 / 9.0, + "4:3": 4.0 / 3.0, + "1:1": 1.0, + "3:4": 3.0 / 4.0, + "9:16": 9.0 / 16.0, +} + + +def _nearest_minimaxh3_aspect_ratio(width: int, height: int) -> str: + """Pick the supported named ratio closest to the requested frame size.""" + target = float(width) / float(height) + return min(MINIMAX_H3_ASPECT_RATIOS, key=lambda name: abs(MINIMAX_H3_ASPECT_RATIOS[name] - target)) + + def _minimaxh3_params_builder( model_params: dict[str, Any], *, extra_params: dict[str, Any], + width: int | None = None, + height: int | None = None, ) -> dict[str, Any]: """Build multipart form fields for MiniMax-H3 from model_params + routed task.""" params = dict(model_params) @@ -53,6 +74,12 @@ def _minimaxh3_params_builder( if params: logger.warning("Unused MiniMax-H3 model params ignored: %s", sorted(params)) + # H3 refuses t2va without a named aspect ratio, and the Generate Video node + # carries width/height rather than a ratio. fl2va takes its ratio from the + # input image and ref2va defaults server-side, so only t2va needs this. + if merged_extra_params.get("task") == "t2va" and "aspect_ratio" not in merged_extra_params and width and height: + merged_extra_params["aspect_ratio"] = _nearest_minimaxh3_aspect_ratio(width, height) + if merged_extra_params: form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False) return form_fields diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py index e1ae43916..bfba44142 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + from collections.abc import Callable from enum import Enum, auto from typing import Any, Literal, TypeAlias @@ -31,6 +34,10 @@ class VideoReferences(dict): pass +class FastH3Deployment(dict): + """Descriptor for a server that fused FastH3 at startup.""" + + class ModelMode(Enum): IMAGE_GENERATION = auto() VIDEO_GENERATION = auto() diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json new file mode 100644 index 000000000..5b3d54f39 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni FastH3 Text to Video.json @@ -0,0 +1,322 @@ +{ + "id": "c6db00da-2ccb-4ff8-8334-d67e1a267781", + "revision": 0, + "last_node_id": 5, + "last_link_id": 3, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [40, 20], + "size": [420, 250], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "FastH3 server prerequisite", + "properties": {}, + "widgets_values": [ + "FastH3 must be fused when the vLLM-Omni server starts. Use --task-type fl2va and --lora-path pointing to exactly one FastH3 adapter file. Do not use CPU or layerwise offload. For VSA / Data-Free, also install fastvideo-kernel and use --diffusion-attention-backend FASTVIDEO_VSA --fastvideo-vsa-topk 64." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "VLLMOmniFastH3Deployment", + "pos": [40, 320], + "size": [420, 160], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": {"name": "url"}, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": {"name": "model"}, + "link": null + } + ], + "outputs": [ + { + "localized_name": "deployment", + "name": "deployment", + "type": "FASTH3_DEPLOYMENT", + "links": [1] + } + ], + "properties": {"Node name for S&R": "VLLMOmniFastH3Deployment"}, + "widgets_values": [ + "http://localhost:8000/v1", + "MiniMaxAI/MiniMax-H3" + ] + }, + { + "id": 3, + "type": "VLLMOmniDiffusionSampling", + "pos": [40, 540], + "size": [420, 230], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": {"name": "n"}, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": {"name": "num_inference_steps"}, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": {"name": "guidance_scale"}, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": {"name": "true_cfg_scale"}, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": {"name": "vae_use_slicing"}, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": {"name": "vae_use_tiling"}, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": {"name": "seed"}, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [2] + } + ], + "properties": {"Node name for S&R": "VLLMOmniDiffusionSampling"}, + "widgets_values": [1, 4, 1, 1, false, false, 1101, "fixed"] + }, + { + "id": 4, + "type": "VLLMOmniGenerateVideo", + "pos": [560, 210], + "size": [540, 610], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 2 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": null + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": 1 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": {"name": "url"}, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": {"name": "model"}, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": {"name": "prompt"}, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": {"name": "negative_prompt"}, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": {"name": "width"}, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": {"name": "height"}, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": {"name": "fps"}, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": {"name": "duration"}, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [3] + } + ], + "properties": {"Node name for S&R": "VLLMOmniGenerateVideo"}, + "widgets_values": [ + "http://localhost:8000/v1", + "MiniMaxAI/MiniMax-H3", + "A cinematic live-action singer performs naturally on a neon-lit stage, with synchronized vocals and environmental sound.", + "", + 1344, + 768, + 24, + 4.458 + ] + }, + { + "id": 5, + "type": "SaveVideo", + "pos": [1200, 280], + "size": [380, 310], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 3 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": {"name": "filename_prefix"}, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMBO", + "widget": {"name": "format"}, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": {"name": "codec"}, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": null + } + ], + "properties": {}, + "widgets_values": ["video/ComfyUI-FastH3", "auto", "auto"] + } + ], + "links": [ + [1, 2, 0, 4, 5, "FASTH3_DEPLOYMENT"], + [2, 3, 0, 4, 2, "SAMPLING_PARAMS"], + [3, 4, 0, 5, 0, "VIDEO"] + ], + "groups": [], + "config": {}, + "extra": { + "workflowRendererVersion": "LG", + "ds": {"scale": 0.9, "offset": [100, 100]} + }, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json index 9785845a5..b03b463d8 100644 --- a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json @@ -1 +1 @@ -{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scale"},"link":null},{"localized_name":"vae_use_slicing","name":"vae_use_slicing","type":"BOOLEAN","widget":{"name":"vae_use_slicing"},"link":null},{"localized_name":"vae_use_tiling","name":"vae_use_tiling","type":"BOOLEAN","widget":{"name":"vae_use_tiling"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"diffusion sampling params","name":"diffusion sampling params","type":"SAMPLING_PARAMS","links":[30]}],"properties":{"Node name for S&R":"VLLMOmniDiffusionSampling"},"widgets_values":[1,50,1,1,false,false,213,"randomize"]},{"id":18,"type":"VLLMOmniMiniMaxH3Params","pos":[483.0944676363692,426.7197881452892],"size":[276.56796875,82],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"audio_flow_shift","name":"audio_flow_shift","type":"FLOAT","widget":{"name":"audio_flow_shift"},"link":null},{"localized_name":"flow_shift","name":"flow_shift","type":"FLOAT","widget":{"name":"flow_shift"},"link":null}],"outputs":[{"localized_name":"MiniMax-H3 video params","name":"MiniMax-H3 video params","type":"VIDEO_PARAMS","links":[33]}],"properties":{"Node name for S&R":"VLLMOmniMiniMaxH3Params"},"widgets_values":[3,12]},{"id":16,"type":"VLLMOmniGenerateVideo","pos":[826.1428043381621,-148.53719710697413],"size":[425.5948726785682,551.767376721426],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"frame","name":"frame","shape":7,"type":"IMAGE","link":29},{"localized_name":"references","name":"references","shape":7,"type":"VIDEO_REFERENCES","link":null},{"localized_name":"sampling_params","name":"sampling_params","shape":7,"type":"SAMPLING_PARAMS","link":30},{"localized_name":"lora","name":"lora","shape":7,"type":"REMOTE_LORA","link":null},{"localized_name":"model_params","name":"model_params","shape":7,"type":"VIDEO_PARAMS","link":33},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null},{"localized_name":"fps","name":"fps","type":"INT","widget":{"name":"fps"},"link":null},{"localized_name":"num_frames","name":"num_frames","type":"INT","widget":{"name":"num_frames"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":[32]}],"properties":{"Node name for S&R":"VLLMOmniGenerateVideo"},"widgets_values":["http://localhost:8000/v1","/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA","A scene of urban fantasy art. A dynamic graffiti art character. A boy made of spray paint comes to life on a concrete wall. He sings an English rap song at high speed while striking a classic, energetic rapper pose. The scene is set under an urban railway bridge at night. The light comes from a single street lamp, creating a cinematic atmosphere full of high energy and amazing detail. The audio of the video consists entirely of the rap, with no other dialogue or noise.","",832,480,24,120]}],"links":[[29,4,0,16,0,"IMAGE"],[30,8,0,16,2,"SAMPLING_PARAMS"],[32,16,0,13,0,"VIDEO"],[33,18,0,16,4,"VIDEO_PARAMS"],[34,23,0,21,0,"IMAGE"],[35,24,0,21,2,"AUDIO"]],"groups":[{"id":1,"title":"Image-to-video input (first-frame-to-video)","bounding":[168.92390079062986,-653.2124402506535,560.8246471437027,407.4246532472182],"color":"#3f789e","flags":{}},{"id":2,"title":"Reference-to-video input","bounding":[-66.75175444110151,-225.24978633666066,805.4597228678848,554.7384400819777],"color":"#3f789e","flags":{}}],"config":{},"extra":{"workflowRendererVersion":"LG","ds":{"scale":0.8,"offset":[1240.4751945653188,770.6151743053379]}},"version":0.4} +{"id":"3aea1d08-7de5-4997-ba71-b2f66ad72a21","revision":0,"last_node_id":24,"last_link_id":35,"nodes":[{"id":13,"type":"SaveVideo","pos":[1315.729153459605,-137.92038948394355],"size":[374,313.7692307692308],"flags":{},"order":9,"mode":0,"inputs":[{"localized_name":"video","name":"video","type":"VIDEO","link":32},{"localized_name":"filename_prefix","name":"filename_prefix","type":"STRING","widget":{"name":"filename_prefix"},"link":null},{"localized_name":"format","name":"format","type":"COMBO","widget":{"name":"format"},"link":null},{"localized_name":"codec","name":"codec","type":"COMFY_DYNAMICCOMBO_V3","widget":{"name":"codec"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":null}],"properties":{},"widgets_values":["video/ComfyUI","auto","auto"]},{"id":10,"type":"MarkdownNote","pos":[183.74636797766794,-571.2529158046766],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":0,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[452.07189605801125,-569.7242536425791],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[29]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["rap.png","image"]},{"id":23,"type":"LoadImage","pos":[9.206110246057657,-7.6955291509170785],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[34]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["white-cat.jpg","image"]},{"id":22,"type":"MarkdownNote","pos":[-9.783113411416666,-148.8790300038605],"size":[297.3012007349086,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Reference to video (r2v)","properties":{},"widgets_values":["For r2v tasks, connect this Video Reference block to the main Generate Video block. Then disconnect the \"frame\" input above."],"color":"#432","bgcolor":"#000"},{"id":24,"type":"LoadAudio","pos":[317.9929742385805,-141.44821500065095],"size":[282.798828125,136],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[35]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["nothing-beats-a-jet2-holiday.mp3",null,""]},{"id":21,"type":"VLLMOmniVideoReferences","pos":[376.6129172897595,105.41557277556292],"size":[156.9380859375,126],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"image_1","name":"image_1","shape":7,"type":"IMAGE","link":34},{"localized_name":"image_2","name":"image_2","shape":7,"type":"IMAGE","link":null},{"localized_name":"audio_1","name":"audio_1","shape":7,"type":"AUDIO","link":35},{"localized_name":"audio_2","name":"audio_2","shape":7,"type":"AUDIO","link":null},{"localized_name":"video_1","name":"video_1","shape":7,"type":"VIDEO","link":null},{"localized_name":"video_2","name":"video_2","shape":7,"type":"VIDEO","link":null}],"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}],"properties":{"Node name for S&R":"VLLMOmniVideoReferences"},"widgets_values":[]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[83.24620208113915,422.08543477542605],"size":[284.205078125,226],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scale"},"link":null},{"localized_name":"vae_use_slicing","name":"vae_use_slicing","type":"BOOLEAN","widget":{"name":"vae_use_slicing"},"link":null},{"localized_name":"vae_use_tiling","name":"vae_use_tiling","type":"BOOLEAN","widget":{"name":"vae_use_tiling"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"diffusion sampling params","name":"diffusion sampling params","type":"SAMPLING_PARAMS","links":[30]}],"properties":{"Node name for S&R":"VLLMOmniDiffusionSampling"},"widgets_values":[1,50,1,1,false,false,213,"randomize"]},{"id":18,"type":"VLLMOmniMiniMaxH3Params","pos":[483.0944676363692,426.7197881452892],"size":[276.56796875,82],"flags":{},"order":6,"mode":0,"inputs":[{"localized_name":"audio_flow_shift","name":"audio_flow_shift","type":"FLOAT","widget":{"name":"audio_flow_shift"},"link":null},{"localized_name":"flow_shift","name":"flow_shift","type":"FLOAT","widget":{"name":"flow_shift"},"link":null}],"outputs":[{"localized_name":"MiniMax-H3 video params","name":"MiniMax-H3 video params","type":"VIDEO_PARAMS","links":[33]}],"properties":{"Node name for S&R":"VLLMOmniMiniMaxH3Params"},"widgets_values":[3,12]},{"id":16,"type":"VLLMOmniGenerateVideo","pos":[826.1428043381621,-148.53719710697413],"size":[425.5948726785682,551.767376721426],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"frame","name":"frame","shape":7,"type":"IMAGE","link":29},{"localized_name":"references","name":"references","shape":7,"type":"VIDEO_REFERENCES","link":null},{"localized_name":"sampling_params","name":"sampling_params","shape":7,"type":"SAMPLING_PARAMS","link":30},{"localized_name":"lora","name":"lora","shape":7,"type":"REMOTE_LORA","link":null},{"localized_name":"model_params","name":"model_params","shape":7,"type":"VIDEO_PARAMS","link":33},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null},{"localized_name":"fps","name":"fps","type":"INT","widget":{"name":"fps"},"link":null},{"localized_name":"duration","name":"duration","type":"FLOAT","widget":{"name":"duration"},"link":null}],"outputs":[{"localized_name":"video","name":"video","type":"VIDEO","links":[32]}],"properties":{"Node name for S&R":"VLLMOmniGenerateVideo"},"widgets_values":["http://localhost:8000/v1","/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA","A scene of urban fantasy art. A dynamic graffiti art character. A boy made of spray paint comes to life on a concrete wall. He sings an English rap song at high speed while striking a classic, energetic rapper pose. The scene is set under an urban railway bridge at night. The light comes from a single street lamp, creating a cinematic atmosphere full of high energy and amazing detail. The audio of the video consists entirely of the rap, with no other dialogue or noise.","",832,480,24,5.0]}],"links":[[29,4,0,16,0,"IMAGE"],[30,8,0,16,2,"SAMPLING_PARAMS"],[32,16,0,13,0,"VIDEO"],[33,18,0,16,4,"VIDEO_PARAMS"],[34,23,0,21,0,"IMAGE"],[35,24,0,21,2,"AUDIO"]],"groups":[{"id":1,"title":"Image-to-video input (first-frame-to-video)","bounding":[168.92390079062986,-653.2124402506535,560.8246471437027,407.4246532472182],"color":"#3f789e","flags":{}},{"id":2,"title":"Reference-to-video input","bounding":[-66.75175444110151,-225.24978633666066,805.4597228678848,554.7384400819777],"color":"#3f789e","flags":{}}],"config":{},"extra":{"workflowRendererVersion":"LG","ds":{"scale":0.8,"offset":[1240.4751945653188,770.6151743053379]}},"version":0.4} diff --git a/apps/ComfyUI-vLLM-Omni/web/main.js b/apps/ComfyUI-vLLM-Omni/web/main.js index da99b2539..9aba6260e 100644 --- a/apps/ComfyUI-vLLM-Omni/web/main.js +++ b/apps/ComfyUI-vLLM-Omni/web/main.js @@ -1,19 +1,153 @@ /** - * @file This file is intended to add dynamic fields to vLLM-Omni nodes - * based on widget (in-node form fields) and input (connection link) values and changes. - * However, this functionality is currently disabled/commented out. - * Because it introduces too much complexity, - * and it may even conflict with the current backend (Python) validation for unknown reasons (pending ComfyUI upstream fixes). + * @file Front-end extension for the vLLM-Omni nodes. + * + * Branding. Every vLLM-Omni node draws the vLLM-Omni "v" mark where LiteGraph + * would otherwise draw its default title dot, and is tinted by the kind of value + * it produces: blue nodes reach a server, amber and purple ones only assemble + * parameters, red ones pin what the server has loaded, teal ones carry media. + * A graph is then readable at a glance, and a node that turns out to be the + * wrong colour for where it sits is usually wired wrong. + * + * The tint is keyed off each node's declared output types rather than a list of + * node names, so a node added later is themed as soon as it returns one of the + * existing types. + * + * It also migrates Generate Video graphs saved before `duration` replaced the + * `num_frames` widget. See migrateNumFramesToDuration below. + * + * This file also used to host dynamic fields on vLLM-Omni nodes, driven by + * widget (in-node form fields) and input (connection link) values and changes. + * That is deliberately absent: it introduced too much complexity, and it even + * conflicted with the current backend (Python) validation for unknown reasons + * (pending ComfyUI upstream fixes). */ import { app } from "../../scripts/app.js"; + +const MARK_URL = new URL("./vllm-omni-mark.svg", import.meta.url).href; +const MARK_ASPECT = 157 / 165; + +/** + * Title bar and body tints, one per node family. + * + * The hues are the logo's: the amber and blue of the "v" mark, and three stops + * of the gradient ring in the "O" (#9128A3, #CC5260, and the teal neighbouring + * its blue). All of them are dark because LiteGraph paints the title bar darker + * than the body and expects both to sit under light text; the ComfyUI light + * theme lightens custom node colours on its own. + */ +const FAMILIES = { + generate: { color: "#152A43", bgcolor: "#1F4269" }, + sampling: { color: "#3A2C12", bgcolor: "#5C461C" }, + model: { color: "#2E1233", bgcolor: "#4A1C52" }, + deployment: { color: "#3A1A1F", bgcolor: "#5C2931" }, + media: { color: "#0F2E2A", bgcolor: "#194A44" }, +}; + +/** Output type -> family. Types absent here fall through to "generate". */ +const FAMILY_BY_OUTPUT = { + SAMPLING_PARAMS: "sampling", + TTS_PARAMS: "model", + VIDEO_PARAMS: "model", + REMOTE_LORA: "deployment", + FASTH3_DEPLOYMENT: "deployment", + VIDEO_REFERENCES: "media", +}; + +function familyOf(nodeData) { + for (const type of nodeData.output ?? []) { + const family = FAMILY_BY_OUTPUT[type]; + if (family) { + return family; + } + } + // What is left returns a generated artifact (IMAGE, VIDEO, AUDIO, STRING), + // which is only ever produced by a node that talks to a vLLM-Omni server. + return "generate"; +} + +/** + * Rewrite a pre-rename Generate Video graph in place. + * + * `duration` (seconds) took over the slot `num_frames` used to occupy, and + * ComfyUI restores widget values positionally, so a graph saved before the + * rename reads its frame count as a duration: the workflow this repository + * shipped stored 120 frames at 24 fps, which would come back as a 120-second + * request. Saved files still name the old widget under `inputs`, and that is + * what makes the rewrite both detectable and safe to apply exactly once -- once + * rewritten the node serializes `duration`, so a later load finds nothing to do. + */ +function migrateNumFramesToDuration(node, info) { + const savedBeforeRename = (info?.inputs ?? []).some((input) => input?.widget?.name === "num_frames"); + if (!savedBeforeRename) { + return + } + const duration = node.widgets?.find((widget) => widget.name === "duration"); + const fps = node.widgets?.find((widget) => widget.name === "fps"); + if (!duration || !fps) { + return + } + // configure() has already placed the stale frame count in the duration widget. + const frames = Number(duration.value); + const rate = Number(fps.value); + if (!Number.isFinite(frames) || !Number.isFinite(rate) || rate <= 0) { + return + } + const seconds = Math.max(duration.options?.min ?? 0.1, Math.round((frames / rate) * 1000) / 1000); + console.info( + `vLLM-Omni: migrated Generate Video "${node.title}" from ${frames} frames to ${seconds}s at ${rate} fps.`, + ); + duration.value = seconds; +} + +const mark = new Image(); +let markReady = false; +mark.addEventListener("load", () => { + markReady = true; + // The image resolves after the first frames are already on screen. + app.graph?.setDirtyCanvas?.(true, true); +}); +mark.src = MARK_URL; + app.registerExtension({ name: "vllm.vllm_omni", async beforeRegisterNodeDef(nodeType, nodeData, app) { if (!nodeData.name.startsWith("VLLMOmni")) { return } - // Stub frontend plugin for now + + const { color, bgcolor } = FAMILIES[familyOf(nodeData)]; + // On the prototype, not the instance: a colour the user picks by hand is + // written to the instance and shadows this, so manual overrides survive. + nodeType.prototype.color = color; + nodeType.prototype.bgcolor = bgcolor; + + if (nodeData.name === "VLLMOmniGenerateVideo") { + const onConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function (info) { + onConfigure?.apply(this, arguments); + migrateNumFramesToDuration(this, info); + }; + } + + // Defining this replaces the default dot outright — LiteGraph draws its + // own square or circle only for nodes that leave onDrawTitleBox unset. + nodeType.prototype.onDrawTitleBox = function (ctx, titleHeight, _size, scale) { + // Zoomed out far enough that LiteGraph itself drops to flat shapes, + // the mark is a few muddy pixels. Leave the title bar clean instead. + if (!markReady || (scale !== undefined && scale < 0.5)) { + return + } + const height = titleHeight * 0.62; + const width = height * MARK_ASPECT; + ctx.drawImage( + mark, + titleHeight * 0.5 - width * 0.5, + titleHeight * -0.5 - height * 0.5, + width, + height, + ); + }; }, async setup() { console.info("vLLM-Omni Setup complete!") diff --git a/apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg b/apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg new file mode 100644 index 000000000..5ce519c1b --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/web/vllm-omni-mark.svg @@ -0,0 +1,5 @@ + + vLLM-Omni + + + diff --git a/docs/features/comfyui.md b/docs/features/comfyui.md index ff766cbb3..b4d6fa101 100644 --- a/docs/features/comfyui.md +++ b/docs/features/comfyui.md @@ -13,7 +13,6 @@ It can send model inference requests to either a locally running vLLM-Omni servi !!! tip If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them. - ## Installation Copy the `apps/ComfyUI-vLLM-Omni` folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`. @@ -21,9 +20,10 @@ Copy the `apps/ComfyUI-vLLM-Omni` folder to the `custom_nodes` subfolder of your If you are running ComfyUI during copying, you should restart ComfyUI to load this extension. !!! tip - You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info. + You can use utility websites such as to download a subdirectory of a repo. Also checkout community discussions (e.g., ) for more info. On the device and virtual environment you run ComfyUI, launch ComfyUI with + ```bash cd ComfyUI @@ -35,6 +35,7 @@ python main.py --cpu ``` On the device and virtual environment you run vLLM-Omni, start a model service with + ```bash vllm serve The_Model_ID_to_Serve --omni --port 8000 ``` @@ -56,6 +57,20 @@ This extension also offers example workflows (at **ComfyUI sidebar -> Templates !!! info The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support. +Every node carries the vLLM-Omni mark in its title bar and is tinted by what it outputs, so a graph is readable at a glance: + +| Colour | Nodes | What they produce | +| --- | --- | --- | +| Blue | Generate Image, Generate Video, Multimodality Understanding, TTS, TTS Voice Clone | A generated image, video, audio, or text. These are the only nodes that reach a server. | +| Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model | +| Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts | +| Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded | +| Teal | Video References | Reference media | + +Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept. + +**Generate Video** takes a clip length in seconds (`duration`), not a frame count. Frames stay the wire unit and are derived with the node's `fps`, so the length is always measured against the rate that is actually served; models that accept only certain frame counts still round to their own lattice server-side. Graphs saved before this widget existed stored `num_frames` in its place and are converted on load, using the fps recorded alongside it -- the browser console names every node it rewrites. + To build a simple workflow yourself, - Drag a generation node onto the canvas. diff --git a/tests/e2e/features/comfyui/test_comfyui_integration.py b/tests/e2e/features/comfyui/test_comfyui_integration.py index 61cf76dcd..0b0f99852 100644 --- a/tests/e2e/features/comfyui/test_comfyui_integration.py +++ b/tests/e2e/features/comfyui/test_comfyui_integration.py @@ -24,6 +24,7 @@ import requests import torch from comfy_api.input import AudioInput, VideoInput from comfyui_vllm_omni.nodes import ( + VLLMOmniFastH3Deployment, VLLMOmniGenerateImage, VLLMOmniGenerateVideo, VLLMOmniTTS, @@ -78,6 +79,7 @@ class SamplingKind(str, Enum): TTS_DIFFUSION_SINGLE = "tts_diffusion_single" VIDEO_NONE = "video_none" VIDEO_DIFFUSION_SINGLE = "video_diffusion_single" + VIDEO_FASTH3 = "video_fasth3" VIDEO_REF2VA_IMAGE_AUDIO = "video_ref2va_image_audio" VIDEO_REF2VA_MULTI_VIDEO = "video_ref2va_multi_video" @@ -89,6 +91,9 @@ VIDEO_WIDTH = 32 VIDEO_HEIGHT = 32 VIDEO_FPS = 8 VIDEO_NUM_FRAMES = 5 +VIDEO_DURATION = VIDEO_NUM_FRAMES / VIDEO_FPS # 0.625 s at the generic 8 fps +# FastH3 pins 24 fps, so the same duration is a different frame count there. +VIDEO_FASTH3_NUM_FRAMES = round(VIDEO_DURATION * 24) DIFFUSION_SINGLE_SAMPLING_PARAMS = DiffusionSamplingParams( { "n": 2, @@ -394,6 +399,24 @@ def _build_mock_outputs(outputs: Iterable[OmniRequestOutput], sampling_case: Sam LORA_PARAMS, ) _assert_model_param_values(received_sampling_params_list[0], VIDEO_MODEL_PARAMS) + elif sampling_case.kind is SamplingKind.VIDEO_FASTH3: + assert len(received_sampling_params_list) == 1 + received = received_sampling_params_list[0] + _assert_sampling_param_values( + received, + { + "width": VIDEO_WIDTH, + "height": VIDEO_HEIGHT, + "num_frames": VIDEO_FASTH3_NUM_FRAMES, + "fps": 24, + "num_inference_steps": 4, + }, + ) + assert received.lora_request is None + # t2va is refused without an explicit ratio; 32x32 derives "1:1". + _assert_model_param_values(received, {"task": "t2va", "aspect_ratio": "1:1"}) + assert "flow_shift" not in received.extra_args + assert "audio_flow_shift" not in received.extra_args elif sampling_case.kind is SamplingKind.VIDEO_REF2VA_IMAGE_AUDIO: assert len(received_sampling_params_list) == 1 assert isinstance(prompt, dict) @@ -861,7 +884,7 @@ async def test_video_generation_node(api_server: str, model: str, image_input: b "width": VIDEO_WIDTH, "height": VIDEO_HEIGHT, "fps": VIDEO_FPS, - "num_frames": VIDEO_NUM_FRAMES, + "duration": VIDEO_DURATION, "model_params": VIDEO_MODEL_PARAMS, } if image_input: @@ -878,6 +901,67 @@ async def test_video_generation_node(api_server: str, model: str, image_input: b assert isinstance(result[0], VideoInput) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "server_case,deployment_model", + [ + pytest.param( + ServerCase( + served_model="MiniMaxAI/MiniMax-H3", + stage_list=["diffusion"], + stage_configs=[{"stage_type": "diffusion", "final_output": True, "final_output_type": "video"}], + outputs=[_build_diffusion_video_output()], + ), + "MiniMaxAI/MiniMax-H3", + id="canonical_model_name", + ), + pytest.param( + ServerCase( + served_model="fasth3", + stage_list=["diffusion"], + stage_configs=[{"stage_type": "diffusion", "final_output": True, "final_output_type": "video"}], + outputs=[_build_diffusion_video_output()], + ), + "fasth3", + # A --served-model-name alias lookup_model_spec cannot resolve to H3. The + # request must still be built by the H3 params builder, or it goes out + # without the aspect_ratio (and task) a t2va request is refused without. + id="served_model_alias", + ), + ], + indirect=["server_case"], +) +@pytest.mark.parametrize( + "sampling_case", + [SamplingCase(kind=SamplingKind.VIDEO_FASTH3, sampling_params=DIFFUSION_VIDEO_SINGLE_SAMPLING_PARAMS)], + indirect=True, +) +async def test_fast_h3_deployment_node(api_server: str, sampling_case: SamplingCase, deployment_model: str): + deployment_node = VLLMOmniFastH3Deployment() + (deployment,) = deployment_node.get_deployment( + url=api_server, + model=deployment_model, + ) + + result = await VLLMOmniGenerateVideo().generate( + url="http://ignored.invalid/v1", + model="ignored-model", + prompt="A singer performs on a neon-lit stage.", + negative_prompt="", + width=VIDEO_WIDTH, + height=VIDEO_HEIGHT, + fps=VIDEO_FPS, + duration=VIDEO_DURATION, + sampling_params=sampling_case.sampling_params, + model_params=H3_MODEL_PARAMS, + fast_h3=deployment, + ) + + assert isinstance(result, tuple) + assert len(result) == 1 + assert isinstance(result[0], VideoInput) + + @pytest.mark.asyncio @pytest.mark.parametrize( "server_case,sampling_case,ref_mode", @@ -944,7 +1028,7 @@ async def test_video_generation_node_minimax_h3_ref2va( width=VIDEO_WIDTH, height=VIDEO_HEIGHT, fps=VIDEO_FPS, - num_frames=VIDEO_NUM_FRAMES, + duration=VIDEO_DURATION, references=references, model_params=H3_MODEL_PARAMS, )