vllm-omni/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

from typing import Literal

import torch
from comfy_api.input import AudioInput, VideoInput

from .utils.api_client import VLLMOmniClient
from .utils.logger import get_logger
from .utils.models import lookup_model_spec
from .utils.types import (
    AudioFormat,
    AutoregressionSamplingParams,
    DiffusionSamplingParams,
    MiniMaxH3ModelSpecificParams,
    QwenTTSModelSpecificParams,
    VideoReferences,
    WanModelSpecificParams,
)
from .utils.validators import (
    add_sampling_parameters_to_stage,
    validate_model_and_sampling_params_types,
)

logger = get_logger(__name__)


class _VLLMOmniGenerateBase:
    """Base class for vLLM-Omni generation nodes with shared functionality."""

    CATEGORY = "vLLM-Omni"

    @classmethod
    def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]:
        """
        Can only validate this model's own input. Cannot check inputs from other nodes.
        See: https://docs.comfy.org/custom-nodes/backend/server_overview#validate_inputs
        """
        if not url:
            return "URL must not be empty"
        if not model:
            return "Model must not be empty"
        return True


class VLLMOmniGenerateImage(_VLLMOmniGenerateBase):
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
                "model": ("STRING", {"default": "Tongyi-MAI/Z-Image-Turbo"}),
                "prompt": ("STRING", {"multiline": True}),
                "negative_prompt": ("STRING", {"multiline": True, "default": ""}),
                "width": ("INT", {"default": 512, "min": 64, "max": 2048}),
                "height": ("INT", {"default": 512, "min": 64, "max": 2048}),
            },
            "optional": {
                "image": ("IMAGE",),
                "mask": ("MASK",),
                # "video": ("VIDEO",),
                # "audio": ("AUDIO",),
                "sampling_params": ("SAMPLING_PARAMS",),
                "lora": ("REMOTE_LORA",),
            },
        }

    RETURN_TYPES = ("IMAGE",)
    RETURN_NAMES = ("image",)
    FUNCTION = "generate"

    async def generate(
        self,
        url: str,
        model: str,
        prompt: str,
        width: int,
        height: int,
        negative_prompt: str | None = None,
        image: torch.Tensor | None = None,
        mask: torch.Tensor | None = None,
        audio: AudioInput | None = None,  # Hidden & unused
        video: VideoInput | None = None,  # Hidden & unused
        sampling_params: dict | list[dict] | None = None,
        lora: dict | None = None,
        **kwargs,
    ):
        if kwargs:
            logger.info("Uncaught kwargs: %s", kwargs)
        logger.debug("Got sampling params: %s", sampling_params)
        validate_model_and_sampling_params_types(model, sampling_params)
        if image is None and mask is not None:
            raise ValueError("Mask input provided without an image input.")

        client = VLLMOmniClient(url)

        spec, pattern = lookup_model_spec(model)
        is_bagel = pattern is not None and "bagel" in pattern.lower()

        # Prefer DALL-E compatible API for simple (one-stage) diffusion models
        if (spec is None or spec["stages"] == ["diffusion"]) and not is_bagel:
            # The number of sampling parameter groups should have been validated.
            # Now, simply convert single-item list to dict.
            if isinstance(sampling_params, list):
                sampling_params = sampling_params[0]
            if audio is None and image is None and video is None:
                # No multimodal input --- use DALL-E image generation
                logger.info("Using DALL-E image generation endpoint")
                output = await client.generate_image(
                    model=model,
                    prompt=prompt,
                    width=width,
                    height=height,
                    negative_prompt=negative_prompt,
                    sampling_params=sampling_params,
                    lora=lora,
                )
                return (output,)
            elif image is not None and audio is None and video is None:
                # Image and text input --- use DALL-E image edit
                logger.info("Using DALL-E image edit endpoint")
                output = await client.edit_image(
                    model=model,
                    prompt=prompt,
                    image=image,
                    width=width,
                    height=height,
                    negative_prompt=negative_prompt,
                    mask=mask,
                    sampling_params=sampling_params,
                    lora=lora,
                )
                return (output,)

        logger.info("Using chat completion endpoint")
        sampling_params = add_sampling_parameters_to_stage(
            model, sampling_params, "diffusion", width=width, height=height
        )
        logger.debug("Edited sampling params: %s", sampling_params)

        output = await client.generate_image_chat_completion(
            model=model,
            prompt=prompt,
            negative_prompt=negative_prompt,
            image=image,
            audio=audio,
            video=video,
            sampling_params=sampling_params,
            lora=lora,
        )

        return (output,)


class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase):
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
                "model": ("STRING", {"default": "Wan-AI/Wan2.2-T2V-A14B-Diffusers"}),
                "prompt": ("STRING", {"multiline": True}),
                "negative_prompt": ("STRING", {"multiline": True, "default": ""}),
                "width": ("INT", {"default": 832, "min": 1}),
                "height": ("INT", {"default": 480, "min": 1}),
                "fps": ("INT", {"default": 16, "min": 1}),
                "num_frames": ("INT", {"default": 41, "min": 1}),
            },
            "optional": {
                "frame": ("IMAGE",),
                "references": ("VIDEO_REFERENCES",),
                "sampling_params": ("SAMPLING_PARAMS",),
                "lora": ("REMOTE_LORA",),
                "model_params": ("VIDEO_PARAMS",),
            },
        }

    RETURN_TYPES = ("VIDEO",)
    RETURN_NAMES = ("video",)
    FUNCTION = "generate"

    @classmethod
    def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs) -> str | Literal[True]:
        base = super().VALIDATE_INPUTS(url, model)
        if base is not True:
            return base
        if frame is not None and references is not None:
            return "Provide only one of frame or references, not both."
        return True

    async def generate(
        self,
        url: str,
        model: str,
        prompt: str,
        width: int,
        height: int,
        fps: int,
        num_frames: int,
        negative_prompt: str | None = None,
        frame: torch.Tensor | None = None,
        references: dict | None = None,
        sampling_params: dict | list[dict] | None = None,
        model_params: dict | None = None,
        lora: dict | None = None,
        **kwargs,
    ):
        if kwargs:
            logger.info("Uncaught kwargs: %s", kwargs)
        logger.debug("Got sampling params: %s", sampling_params)
        logger.debug("Got model params: %s", model_params)
        validate_model_and_sampling_params_types(model, sampling_params)

        # Currently, all video generation models are single-stage diffusion models
        if isinstance(sampling_params, list):
            if len(sampling_params) != 1:
                raise ValueError(
                    "Video generation expects a single sampling params group. "
                    "Please provide one Diffusion sampling node."
                )
            sampling_params = sampling_params[0]

        if sampling_params is not None:
            sampling_params.pop("type", None)  # internal fields
        # model_params["type"] is also an internal field, but is used in api_client
        # if model_params is not None:
        #     model_params.pop("type", None)  # internal fields

        client = VLLMOmniClient(url)
        output = await client.generate_video(
            model=model,
            prompt=prompt,
            frame=frame,  # frame present => fl2va / Wan I2V
            references=references,
            width=width,
            height=height,
            num_frames=num_frames,
            fps=fps,
            negative_prompt=negative_prompt,
            sampling_params=sampling_params,
            lora=lora,
            model_params=model_params,
        )
        return (output,)


class VLLMOmniUnderstanding(_VLLMOmniGenerateBase):
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
                "model": ("STRING", {"default": "Qwen/Qwen2.5-Omni-7B"}),
                "prompt": ("STRING", {"multiline": True}),
                "output_text": ("BOOLEAN", {"default": True}),
                "output_audio": ("BOOLEAN", {"default": True}),
                "use_audio_in_video": ("BOOLEAN", {"default": True}),
            },
            "optional": {
                "image": ("IMAGE",),
                "video": ("VIDEO",),
                "audio": ("AUDIO",),
                "sampling_params": ("SAMPLING_PARAMS",),
            },
        }

    RETURN_TYPES = ("STRING", "AUDIO")
    RETURN_NAMES = ("text_response", "audio_response")
    FUNCTION = "generate"

    @classmethod
    def VALIDATE_INPUTS(cls, url, model, output_text, output_audio) -> str | Literal[True]:  # type: ignore[reportIncompatibleMethodOverride]
        super_validation = super().VALIDATE_INPUTS(url, model)
        if isinstance(super_validation, str):
            return super_validation
        if not output_text and not output_audio:
            return "At least one of output_text or output_audio must be True."
        return True

    async def generate(
        self,
        url: str,
        model: str,
        prompt: str,
        image: torch.Tensor | None = None,
        audio: AudioInput | None = None,
        video: VideoInput | None = None,
        sampling_params: dict | list[dict] | None = None,
        output_text: bool = True,
        output_audio: bool = True,
        use_audio_in_video: bool = True,
        **kwargs,
    ) -> tuple[str, AudioInput]:
        if kwargs:
            logger.info("Uncaught kwargs: %s", kwargs)
        logger.debug("Got sampling params: %s", sampling_params)
        validate_model_and_sampling_params_types(model, sampling_params)

        client = VLLMOmniClient(url)
        spec, pattern = lookup_model_spec(model)
        is_bagel = pattern is not None and "bagel" in pattern.lower()

        if is_bagel:
            # A lot of special handlings here...
            if output_audio:
                raise ValueError("BAGEL models do not support audio output.")
            if audio is not None or video is not None:
                raise ValueError("BAGEL models do not support audio or video input.")
            (
                text_response,
                _,
            ) = await client.generate_understanding_chat_completion(
                model=model,
                prompt=prompt,
                image=image,
                audio=None,
                video=None,
                sampling_params=sampling_params,
                modalities=["text"],
            )
        else:
            modalities = []
            if output_text:
                modalities.append("text")
            if output_audio:
                modalities.append("audio")

            if use_audio_in_video and video is not None:
                use_audio_in_video = True
            else:
                use_audio_in_video = False

            (
                text_response,
                audio,
            ) = await client.generate_understanding_chat_completion(
                model=model,
                prompt=prompt,
                image=image,
                audio=audio,
                video=video,
                sampling_params=sampling_params,
                modalities=modalities,
                # == extra kwargs ==
                mm_processor_kwargs={"use_audio_in_video": use_audio_in_video},
            )

        if text_response is None:
            text_response = ""
        if audio is None:
            channels = 1
            duration = 1
            sample_rate = 44100
            num_samples = int(round(duration * sample_rate))
            waveform = torch.zeros((1, channels, num_samples), dtype=torch.float32)
            audio = {"waveform": waveform, "sample_rate": sample_rate}

        return (text_response, audio)


class VLLMOmniTTS(_VLLMOmniGenerateBase):
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
                "model": (
                    "STRING",
                    {"default": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"},
                ),
                "input": ("STRING", {"multiline": True}),
                "voice": ("STRING", {"default": "Vivian"}),
                "response_format": (["mp3", "opus", "aac", "flac", "wav", "pcm"],),
                "speed": (
                    "FLOAT",
                    {"default": 1.0, "min": 0.25, "max": 4.0, "step": 0.01},
                ),
            },
            "optional": {
                "model_specific_params": ("TTS_PARAMS",),
            },
        }

    RETURN_TYPES = ("AUDIO",)
    RETURN_NAMES = ("audio",)
    FUNCTION = "generate"

    async def generate(
        self,
        url: str,
        model: str,
        input: str,
        voice: str,
        response_format: AudioFormat,
        speed: float,
        model_specific_params: dict | None,
        **kwargs,
    ) -> tuple[AudioInput]:
        logger.info("Got extra kwargs in TTS: %s", kwargs)

        is_qwen_tts = "qwen3-tts" in model.lower()
        if not is_qwen_tts and isinstance(model_specific_params, QwenTTSModelSpecificParams):
            raise ValueError(
                "You have provided Qwen-specific TTS params."
                "However, the model appears to not be a Qwen TTS model (no 'Qwen3-TTS' in model name)."
            )

        combined_params = {**kwargs, **(model_specific_params or {})}

        client = VLLMOmniClient(url)

        audio = await client.generate_speech(
            model=model,
            input=input,
            voice=voice,
            response_format=response_format,
            speed=speed,
            **combined_params,
        )
        return (audio,)


class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
                "model": ("STRING", {"default": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"}),
                "input": ("STRING", {"multiline": True}),
                "voice": ("STRING", {"default": "Vivian"}),
                "response_format": (["mp3", "opus", "aac", "flac", "wav", "pcm"],),
                "speed": (
                    "FLOAT",
                    {"default": 1.0, "min": 0.25, "max": 4.0, "step": 0.01},
                ),
                "ref_audio": ("AUDIO",),
                "ref_text": ("STRING", {"multiline": True}),
                "x_vector_only_mode": ("BOOLEAN", {"default": False}),
            },
            "optional": {
                "model_specific_params": ("TTS_PARAMS",),
            },
        }

    RETURN_TYPES = ("AUDIO",)
    RETURN_NAMES = ("audio",)
    FUNCTION = "generate"

    async def generate(
        self,
        url: str,
        model: str,
        input: str,
        voice: str,
        response_format: AudioFormat,
        speed: float,
        ref_audio: AudioInput,
        ref_text: str,
        x_vector_only_mode: bool,
        model_specific_params: dict | None,
        **kwargs,
    ):
        is_qwen_tts = "qwen3-tts" in model.lower()
        if not is_qwen_tts and isinstance(model_specific_params, QwenTTSModelSpecificParams):
            raise ValueError(
                "You have provided Qwen-specific TTS params."
                "However, the model appears to not be a Qwen TTS model (no 'Qwen3-TTS' in model name)."
            )

        combined_params = {
            "ref_audio": ref_audio,
            "ref_text": ref_text,
            "x_vector_only_mode": x_vector_only_mode,
            **kwargs,
            **(model_specific_params or {}),
        }

        client = VLLMOmniClient(url)

        audio = await client.generate_speech(
            model=model,
            input=input,
            voice=voice,
            response_format=response_format,
            speed=speed,
            **combined_params,
        )
        return (audio,)


class VLLMOmniARSampling:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "max_tokens": ("INT", {"default": 100, "min": 1, "max": 10000}),
                "temperature": (
                    "FLOAT",
                    {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01},
                ),
                "top_p": (
                    "FLOAT",
                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01},
                ),
                "repetition_penalty": (
                    "FLOAT",
                    {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.01},
                ),
                # === Put seed at last. ===
                # Whenever a field named "seed" is present, ComfyUI adds another field called "control after generate"
                "seed": (
                    "INT",
                    {
                        "default": -1,
                        "min": -1,
                        "step": 1,
                        "tooltip": "-1 means to not provide a seed.",
                    },
                ),
            }
        }

    RETURN_TYPES = ("SAMPLING_PARAMS",)
    RETURN_NAMES = ("AR sampling params",)
    FUNCTION = "get_params"
    CATEGORY = "vLLM-Omni/Sampling Params"

    def get_params(self, seed, **kwargs):
        params = AutoregressionSamplingParams(kwargs)
        if seed >= 0:
            params["seed"] = seed
        return (params,)


class VLLMOmniDiffusionSampling:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "n": (
                    "INT",
                    {
                        "default": 1,
                        "min": 0,
                        "max": 10,
                        "step": 1,
                        "tooltip": "Number of images to generate",
                    },
                ),
                "num_inference_steps": (
                    "INT",
                    {
                        "default": 50,
                        "min": 1,
                        "max": 1000,
                        "tooltip": "Number of denoising steps (higher = better quality, slower).",
                    },
                ),
                "guidance_scale": (
                    "FLOAT",
                    {
                        "default": 7.5,
                        "min": 0.0,
                        "max": 20.0,
                        "step": 0.1,
                        "tooltip": "Classifier-free guidance scale (higher = more prompt adherence).",
                    },
                ),
                "true_cfg_scale": (
                    "FLOAT",
                    {
                        "default": 1.0,
                        "min": 0.0,
                        "max": 20.0,
                        "step": 0.5,
                        "tooltip": "True CFG scale for advanced control (model-specific).",
                    },
                ),
                "vae_use_slicing": (
                    "BOOLEAN",
                    {
                        "default": False,
                        "tooltip": "Enable VAE slicing for reduced memory usage (slight quality trade-off)",
                    },
                ),
                "vae_use_tiling": (
                    "BOOLEAN",
                    {
                        "default": False,
                        "tooltip": "Enable VAE tiling for reduced memory usage (slight quality trade-off)",
                    },
                ),
                # === Put seed at last. ===
                # Whenever a field named "seed" is present, ComfyUI adds another field called "control after generate"
                "seed": (
                    "INT",
                    {
                        "default": -1,
                        "min": -1,
                        "step": 1,
                        "tooltip": "-1 means to not provide a seed.",
                    },
                ),
            }
        }

    RETURN_TYPES = ("SAMPLING_PARAMS",)
    RETURN_NAMES = ("diffusion sampling params",)
    FUNCTION = "get_params"
    CATEGORY = "vLLM-Omni/Sampling Params"

    def get_params(self, seed, **kwargs):
        params = DiffusionSamplingParams(kwargs)
        if seed >= 0:
            params["seed"] = seed
        return (params,)


class VLLMOmniSamplingParamsList:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "param1": ("SAMPLING_PARAMS",),
            },
            "optional": {
                "param2": ("SAMPLING_PARAMS",),
                "param3": ("SAMPLING_PARAMS",),
            },
        }

    RETURN_TYPES = ("SAMPLING_PARAMS",)
    RETURN_NAMES = ("param list",)
    FUNCTION = "aggregate"
    CATEGORY = "vLLM-Omni/Sampling Params"

    def aggregate(self, param1: dict, param2: dict | None = None, param3: dict | None = None):
        for i, p in enumerate((param1, param2, param3)):
            if isinstance(p, list):
                raise ValueError(
                    f"Input {i} is a Multi-Stage Sampling Params List. "
                    f"Expected a single sampling parameters node (either AR or Diffusion)."
                )

        params = [param1]
        if param2 is not None:
            params.append(param2)
        if param3 is not None:
            params.append(param3)
        return (params,)


class VLLMOmniRemoteLoRA:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "local_path": ("STRING", {"default": ""}),
                "name": ("STRING", {"default": ""}),
                "scale": (
                    "FLOAT",
                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.1},
                ),
                "int_id": (
                    "INT",
                    {
                        "default": 0,
                        "min": 0,
                        "step": 1,
                        "tooltip": "0 means it is not set and the server can derive it.",
                    },
                ),
            }
        }

    RETURN_TYPES = ("REMOTE_LORA",)
    RETURN_NAMES = ("lora",)
    FUNCTION = "get_lora"
    CATEGORY = "vLLM-Omni"

    @classmethod
    def VALIDATE_INPUTS(cls, local_path, name) -> str | Literal[True]:
        if not local_path.strip() or not name.strip():
            return "Both local_path and name must be provided."
        return True

    def get_lora(self, local_path: str, name: str, scale: float, int_id: int):
        local_path = local_path.strip()
        name = name.strip()
        lora = {
            "local_path": local_path or None,
            "name": name or None,
            "scale": float(scale),
            "int_id": int(int_id) if int_id > 0 else None,
        }
        return (lora,)


class VLLMOmniQwenTTSParams:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "task_type": (
                    ["CustomVoice", "VoiceDesign", "Base"],
                    {"default": "CustomVoice"},
                ),
                "language": (
                    ["Auto", "Chinese", "English", "Japanese", "Korean"],
                    {"default": "Auto"},
                ),
                "instructions": ("STRING", {"multiline": True}),
                "max_new_tokens": ("INT", {"default": 2048, "min": 1}),
            }
        }

    RETURN_TYPES = ("TTS_PARAMS",)
    RETURN_NAMES = ("Qwen TTS params",)
    FUNCTION = "get_params"
    CATEGORY = "vLLM-Omni/TTS Params"

    def get_params(self, **kwargs):
        return (QwenTTSModelSpecificParams(kwargs),)


class VLLMOmniWanParams:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "guidance_scale_2": (
                    "FLOAT",
                    {"default": 4.0, "min": 0.0, "max": 20.0, "step": 0.1},
                ),
                "boundary_ratio": (
                    "FLOAT",
                    {"default": 0.875, "min": 0.0, "max": 1.0, "step": 0.001},
                ),
                "flow_shift": (
                    "FLOAT",
                    {"default": 5.0, "min": 0.0, "max": 100.0, "step": 0.1},
                ),
            }
        }

    RETURN_TYPES = ("VIDEO_PARAMS",)
    RETURN_NAMES = ("Wan video params",)
    FUNCTION = "get_params"
    CATEGORY = "vLLM-Omni/Video Params"

    def get_params(self, **kwargs):
        return (WanModelSpecificParams(kwargs),)


class VLLMOmniMiniMaxH3Params:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "audio_flow_shift": (
                    "FLOAT",
                    {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.1},
                ),
                "flow_shift": (
                    "FLOAT",
                    {"default": 12.0, "min": 0.0, "max": 100.0, "step": 0.1},
                ),
            }
        }

    RETURN_TYPES = ("VIDEO_PARAMS",)
    RETURN_NAMES = ("MiniMax-H3 video params",)
    FUNCTION = "get_params"
    CATEGORY = "vLLM-Omni/Video Params"

    def get_params(self, **kwargs):
        params = MiniMaxH3ModelSpecificParams(kwargs)
        params["type"] = "minimax_h3"
        return (params,)


class VLLMOmniVideoReferences:
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {},
            "optional": {
                "image_1": ("IMAGE",),
                "image_2": ("IMAGE",),
                "audio_1": ("AUDIO",),
                "audio_2": ("AUDIO",),
                "video_1": ("VIDEO",),
                "video_2": ("VIDEO",),
            },
        }

    RETURN_TYPES = ("VIDEO_REFERENCES",)
    RETURN_NAMES = ("references",)
    FUNCTION = "get_references"
    CATEGORY = "vLLM-Omni"

    def get_references(
        self,
        image_1: torch.Tensor | None = None,
        image_2: torch.Tensor | None = None,
        audio_1: AudioInput | None = None,
        audio_2: AudioInput | None = None,
        video_1: VideoInput | None = None,
        video_2: VideoInput | None = None,
        **kwargs,
    ):
        if kwargs:
            logger.info("Uncaught kwargs: %s", kwargs)
        refs = VideoReferences()
        if image_1 is not None:
            refs["image_1"] = image_1
        if image_2 is not None:
            refs["image_2"] = image_2
        if audio_1 is not None:
            refs["audio_1"] = audio_1
        if audio_2 is not None:
            refs["audio_2"] = audio_2
        if video_1 is not None:
            refs["video_1"] = video_1
        if video_2 is not None:
            refs["video_2"] = video_2
        return (refs,)