vllm-omni/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

"""
An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API,
and transforms the API responses back to ComfyUI formats.

The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
Original source at https://github.com/dougbtv/comfyui-vllm-omni, distributed under the MIT License.
"""

import asyncio
import json
from typing import Any

import aiohttp
import av.error
import torch
from comfy_api.input import AudioInput, VideoInput

from .format import (
    audio_to_base64,
    base64_to_audio,
    base64_to_image_tensor,
    bytes_to_audio,
    bytes_to_video,
    image_tensor_to_base64,
    image_tensor_to_png_bytes,
    video_to_base64,
    video_to_bytes,
)
from .logger import get_logger, pretty_printer
from .models import lookup_model_spec
from .types import AudioFormat

logger = get_logger(__name__)


async def url_json(session: aiohttp.ClientSession, url: str, verb: str = "get", **kwargs) -> dict[str, Any]:
    try:
        async with getattr(session, verb)(url, **kwargs) as response:
            if not response.ok:
                error_text = await response.text()
                raise (ValueError if response.status < 500 else RuntimeError)(
                    f"vLLM-Omni API returned status {response.status}: {error_text}"
                )
            try:
                return await response.json()
            except aiohttp.ContentTypeError as e:
                raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
    except aiohttp.ClientError as e:
        raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")


async def url_bytes(session: aiohttp.ClientSession, url: str, verb: str = "get", **kwargs) -> bytes:
    try:
        async with getattr(session, verb)(url, **kwargs) as response:
            if not response.ok:
                error_text = await response.text()
                raise (ValueError if response.status < 500 else RuntimeError)(
                    f"vLLM-Omni API returned status {response.status}: {error_text}"
                )
            return await response.read()
    except aiohttp.ClientError as e:
        raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")


class VLLMOmniClient:
    def __init__(
        self,
        base_url: str,
        timeout: float | None = None,
        poll_interval: float = 5.0,
        max_poll_duration: float = 60 * 30,
    ):
        self.base_url = base_url
        self.timeout = aiohttp.ClientTimeout(total=timeout)
        self.poll_interval = poll_interval
        self.max_poll_duration = max_poll_duration

    async def generate_image(
        self,
        *,
        model: str,
        prompt: str,
        width: int,
        height: int,
        negative_prompt: str | None = None,
        sampling_params: dict | None = None,
        lora: dict | None = None,
    ) -> torch.Tensor:
        """Run text-to-image generation via DALLE API"""
        await self._check_model_exist(model)

        size = f"{width}x{height}"
        payload: dict[str, Any] = {
            "model": model,
            "prompt": prompt,
            "size": size,
            "response_format": "b64_json",
        }
        if negative_prompt:
            payload["negative_prompt"] = negative_prompt
        if sampling_params is not None:
            payload.update(sampling_params)
        if lora is not None:
            payload["lora"] = lora
        logger.debug("img gen payload: %s", payload)

        url = self.base_url + "/images/generations"
        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            try:
                async with session.post(
                    url,
                    json=payload,
                    headers={"Content-Type": "application/json"},
                ) as response:
                    if not response.ok:
                        error_text = await response.text()
                        raise (ValueError if response.status < 500 else RuntimeError)(
                            f"vLLM-Omni API returned status {response.status}: {error_text}"
                        )

                    try:
                        data = await response.json()
                    except aiohttp.ContentTypeError as e:
                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
                    if "data" not in data:
                        raise RuntimeError("API response missing 'data' field - expected OpenAI DALL-E format")
                    if not data["data"]:
                        raise RuntimeError("API returned empty data array")

                    image_tensors = []
                    for idx, img in enumerate(data["data"]):
                        if "b64_json" not in img:
                            raise RuntimeError(f"API returned image #{idx} without 'b64_json' field")
                        base64_str = img["b64_json"]
                        tensor = base64_to_image_tensor(base64_str)
                        image_tensors.append(tensor)
                        logger.debug("Image #%d has shape %s", idx, tensor.shape)

                    batch_tensor = torch.stack(image_tensors, dim=0)
                    logger.debug("batch_tensor output has shape: %s", batch_tensor.shape)
                    return batch_tensor

            except aiohttp.ClientError as e:
                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")

    async def edit_image(
        self,
        *,
        model: str,
        prompt: str,
        image: torch.Tensor,
        width: int,
        height: int,
        negative_prompt: str | None = None,
        mask: torch.Tensor | None = None,
        sampling_params: dict | None = None,
        lora: dict | None = None,
    ) -> torch.Tensor:
        """Run image editing via DALLE API"""
        await self._check_model_exist(model)

        size = f"{width}x{height}"
        image_filename = "image.png"  # Required for multipart form
        form = aiohttp.FormData()
        form.add_field("model", model)
        form.add_field(
            "image",
            image_tensor_to_png_bytes(image, image_filename),
            filename=image_filename,
            content_type="image/png",
        )
        form.add_field("prompt", prompt)
        form.add_field("size", size)
        if negative_prompt:
            form.add_field("negative_prompt", negative_prompt)
        if sampling_params is not None:
            for k, v in sampling_params.items():
                form.add_field(k, str(v))
        if lora is not None:
            form.add_field("lora", json.dumps(lora, ensure_ascii=False))
        if mask is not None:
            mask_filename = "mask.png"
            form.add_field(
                "mask",
                image_tensor_to_png_bytes(mask, mask_filename),
                filename=mask_filename,
                content_type="image/png",
            )

        url = self.base_url + "/images/edits"
        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            try:
                async with session.post(url, data=form) as response:
                    if not response.ok:
                        error_text = await response.text()
                        raise (ValueError if response.status < 500 else RuntimeError)(
                            f"vLLM-Omni API returned status {response.status}: {error_text}"
                        )

                    try:
                        data = await response.json()
                    except aiohttp.ContentTypeError as e:
                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")

                    if "data" not in data:
                        raise RuntimeError("API response missing 'data' field - expected OpenAI DALL-E format")
                    if not data["data"]:
                        raise RuntimeError("API returned empty data array")

                    image_tensors = []
                    for idx, img in enumerate(data["data"]):
                        if "b64_json" not in img:
                            raise RuntimeError(f"API returned image #{idx} without 'b64_json' field")
                        base64_str = img["b64_json"]
                        tensor = base64_to_image_tensor(base64_str)
                        image_tensors.append(tensor)

                    return torch.stack(image_tensors, dim=0)

            except aiohttp.ClientError as e:
                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")

    async def generate_image_chat_completion(
        self,
        *,
        model: str,
        prompt: str,
        negative_prompt: str | None = None,
        image: torch.Tensor | None = None,
        audio: AudioInput | None = None,
        video: VideoInput | None = None,
        sampling_params: dict | list[dict] | None = None,
        lora: dict | None = None,
    ) -> torch.Tensor:
        payload = VLLMOmniClient._prepare_chat_completion_messages(
            model=model,
            prompt=prompt,
            negative_prompt=negative_prompt,
            image=image,
            audio=audio,
            video=video,
            sampling_params=sampling_params,
            modalities=["image"],
            # === below are additional `extra_body` fields, handled by **kwargs ===
            lora=lora,
        )
        choices = await self._generate_base_chat_completion(model, payload)

        image_tensors = []
        for idx, img_content in enumerate(choices[0]["message"]["content"]):
            base64_str = img_content.get("image_url", {}).get("url", "")
            if not base64_str:
                raise RuntimeError(f"API returned image #{idx} without image url")
            tensor = base64_to_image_tensor(base64_str)
            image_tensors.append(tensor)

        return torch.stack(image_tensors, dim=0)

    async def generate_video(
        self,
        *,
        model: str,
        prompt: str,
        width: int,
        height: int,
        num_frames: int,
        fps: int,
        negative_prompt: str | None = None,
        frame: torch.Tensor | None = None,
        references: dict | None = None,
        sampling_params: dict | None = None,
        model_params: dict | None = None,
        lora: dict | None = None,
        **extra_params,
    ) -> VideoInput:
        if frame is not None and references is not None:
            raise ValueError("Provide only one of frame or references, not both.")

        # === regular payload fields ===
        form = aiohttp.FormData()
        form.add_field("model", model)
        form.add_field("prompt", prompt)
        form.add_field("width", str(width))
        form.add_field("height", str(height))
        form.add_field("num_frames", str(num_frames))
        form.add_field("fps", str(fps))
        if negative_prompt:
            form.add_field("negative_prompt", negative_prompt)
        if sampling_params is not None:
            for k, v in sampling_params.items():
                form.add_field(k, str(v))
        if lora is not None:
            form.add_field("lora", json.dumps(lora, ensure_ascii=False))

        # === multimodal inputs (first-last-frames, references, etc.) ===
        input_reference_image: torch.Tensor | None = None
        audio_reference: AudioInput | None = None
        reference_videos: list[VideoInput] = []
        video_task: str | None = None

        if frame is not None:
            input_reference_image = frame
            video_task = "fl2va"
        elif references is not None:
            images = [references[k] for k in ("image_1", "image_2") if k in references and references[k] is not None]
            audios = [references[k] for k in ("audio_1", "audio_2") if k in references and references[k] is not None]
            videos = [references[k] for k in ("video_1", "video_2") if k in references and references[k] is not None]

            if not images and not audios and not videos:
                raise ValueError("references is empty; connect at least one image/audio/video.")

            if videos:
                if images or audios:
                    raise ValueError(
                        "Video references cannot be combined with image or audio references. "
                        "Connect only video_1/video_2 for multi-video Ref2VA."
                    )
                reference_videos = videos
                video_task = "ref2va"
            elif len(images) == 1 and len(audios) == 1:
                input_reference_image = images[0]
                audio_reference = audios[0]
                video_task = "ref2va"
            else:
                raise ValueError(
                    "Invalid references combination. Supported modes: "
                    "(1) one or more videos only, or (2) exactly one image and one audio."
                )
        else:
            video_task = "t2va"

        if input_reference_image is not None:
            image_filename = "image.png"  # Required for multipart form
            form.add_field(
                "input_reference",
                image_tensor_to_png_bytes(input_reference_image, image_filename),
                filename=image_filename,
                content_type="image/png",
            )

        if audio_reference is not None:
            form.add_field(
                "audio_reference",
                json.dumps({"audio_url": audio_to_base64(audio_reference)}, ensure_ascii=False),
            )

        for idx, video in enumerate(reference_videos, start=1):
            video_filename = f"reference_{idx}.mp4"
            form.add_field(
                "input_references",
                video_to_bytes(video, video_filename),
                filename=video_filename,
                content_type="video/mp4",
            )

        # === model specific params. Either use a specialized builder, or add flattened fields as-is ===
        if model_params is not None:
            model_params = dict(model_params)
            model_params.pop("type", None)

        spec, _ = lookup_model_spec(model)
        params_builder = spec.get("params_builder") if spec else None
        if params_builder is not None:
            form_fields = params_builder(
                model_params or {},
                extra_params={**extra_params, "task": video_task},
            )
            for k, v in form_fields.items():
                form.add_field(k, v if isinstance(v, str) else str(v))
        else:
            if model_params is not None:
                for k, v in model_params.items():
                    form.add_field(k, str(v))
            if extra_params:
                form.add_field("extra_params", json.dumps(extra_params, ensure_ascii=False))

        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            # Start the video generation job
            url = f"{self.base_url}/videos"
            data = await url_json(session, url, "post", data=form)
            if (job_id := data.get("id", None)) is None:
                raise RuntimeError("API response missing job 'id' field - expected OpenAI compliant format")
            if (job_status := data.get("status", None)) is None:
                raise RuntimeError("API response missing job 'status' field - expected OpenAI compliant format")

            # Poll for video generation job completion
            deadline = asyncio.get_running_loop().time() + self.max_poll_duration
            url = f"{self.base_url}/videos/{job_id}"
            while job_status not in {"completed", "failed"}:
                await asyncio.sleep(self.poll_interval)

                data = await url_json(session, url)
                if (job_status := data.get("status", None)) is None:
                    raise RuntimeError("API response missing job 'status' field - expected OpenAI compliant format")
                if asyncio.get_running_loop().time() >= deadline:
                    raise RuntimeError(f"Timed out waiting for video job {job_id} to complete")

            if job_status == "failed":
                raise RuntimeError(f"Video job failed: {data}")

            # Retrieve completed content
            video_bytes = await url_bytes(session, f"{url}/content")

            # Decode video and make a best effort at cleaning up server resources
            try:
                return bytes_to_video(video_bytes)
            finally:
                try:
                    await url_json(session, url, "delete")
                except Exception as exc:
                    logger.warning("Failed to clean up video job %s: %s", job_id, exc)

    async def generate_understanding_chat_completion(
        self,
        *,
        model: str,
        prompt: str,
        image: torch.Tensor | None = None,
        audio: AudioInput | None = None,
        video: VideoInput | None = None,
        sampling_params: dict | list[dict] | None = None,
        modalities: list[str] = ["text", "audio"],
        **extra_body,
    ) -> tuple[str | None, AudioInput | None]:
        # Response may contain two choices: one with text, one with audio
        payload = VLLMOmniClient._prepare_chat_completion_messages(
            model=model,
            prompt=prompt,
            negative_prompt=None,
            image=image,
            audio=audio,
            video=video,
            sampling_params=sampling_params,
            modalities=modalities,
            **extra_body,
        )

        choices = await self._generate_base_chat_completion(model, payload)
        text_response = None
        audio_base64 = None
        for choice in choices:
            try:
                text_response = choice["message"]["content"]
            except (KeyError, TypeError):
                # Either this case (text response) or the audio response case will be hit. Checking None's later.
                pass
            try:
                audio_base64 = choice["message"]["audio"]["data"]
            except (KeyError, TypeError):
                # Either this case (text response) or the audio response case will be hit. Checking None's later.
                pass
        if audio_base64 is None and text_response is None:
            raise RuntimeError(
                "API response missing both '.message.audio' and 'message.content' fields."
                f"The choices object is {choices}"
            )
        if audio_base64 is not None:
            audio = base64_to_audio(audio_base64)
            logger.debug(
                "audio sample rate %d, audio shape %s, duration in second %f",
                audio["sample_rate"],
                audio["waveform"].shape,
                audio["waveform"].shape[2] / audio["sample_rate"],
            )
        else:
            audio = None
        return text_response, audio

    async def generate_speech(
        self,
        *,
        model: str,
        input: str,
        voice: str,
        response_format: AudioFormat,
        speed: float,
        **extra_params,
    ) -> AudioInput:
        await self._check_model_exist(model)

        ref_audio: AudioInput | None = extra_params.pop("ref_audio", None)

        payload = {
            "model": model,
            "input": input,
            "voice": voice,
            "response_format": response_format,
            "speed": speed,
            **extra_params,
        }

        if ref_audio is not None:
            audio_base64 = audio_to_base64(ref_audio)
            payload["ref_audio"] = audio_base64

        logger.debug("Omni TTS payload: %s", pretty_printer.pformat(payload))

        url = self.base_url + "/audio/speech"
        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            try:
                async with session.post(
                    url,
                    json=payload,
                    headers={"Content-Type": "application/json"},
                ) as response:
                    if not response.ok:
                        error_text = await response.text()
                        raise (ValueError if response.status < 500 else RuntimeError)(
                            f"vLLM-Omni API returned status {response.status}: {error_text}"
                        )

                    try:
                        audio_bytes = await response.read()
                    except aiohttp.ContentTypeError as e:
                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")

                    try:
                        audio = bytes_to_audio(audio_bytes)
                    except av.error.InvalidDataError as e:
                        raise ValueError(
                            f"Invalid audio data received from vLLM-Omni: {e}"
                            "Check if you have input unsupported arguments (such as 'voice')"
                        )
                    return audio

            except aiohttp.ClientError as e:
                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")

    async def _generate_base_chat_completion(self, model: str, payload: dict[str, Any]) -> list[dict[str, Any]]:
        logger.debug("Omni payload: %s", pretty_printer.pformat(payload))
        await self._check_model_exist(model)

        url = self.base_url + "/chat/completions"
        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            try:
                async with session.post(
                    url,
                    json=payload,
                    headers={"Content-Type": "application/json"},
                ) as response:
                    if not response.ok:
                        error_text = await response.text()
                        raise (ValueError if response.status < 500 else RuntimeError)(
                            f"vLLM-Omni API returned status {response.status}: {error_text}"
                        )

                    try:
                        data = await response.json()
                    except aiohttp.ContentTypeError as e:
                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")

                    logger.debug(
                        "chat completion response: %s",
                        pretty_printer.pformat(data),
                    )

                    try:
                        return data["choices"]
                    except (KeyError, TypeError):
                        raise RuntimeError("Invalid JSON response from vLLM-Omni: missing 'choices' field")

            except aiohttp.ClientError as e:
                raise RuntimeError(f"Network error connecting to vLLM-Omni at {self.base_url}: {e}")

    async def _check_model_exist(self, model: str):
        url = self.base_url + "/models"
        async with aiohttp.ClientSession(timeout=self.timeout) as session:
            try:
                async with session.get(
                    url,
                    headers={"Content-Type": "application/json"},
                ) as response:
                    if not response.ok:
                        error_text = await response.text()
                        raise (ValueError if response.status < 500 else RuntimeError)(
                            f"vLLM-Omni API returned status {response.status} "
                            f"when getting hosted model list: {error_text}"
                        )

                    try:
                        data = await response.json()
                    except aiohttp.ContentTypeError as e:
                        raise RuntimeError(f"Invalid JSON response when getting hosted model list from vLLM-Omni: {e}")

            except aiohttp.ClientError as e:
                raise RuntimeError(f"Network error connecting to vLLM-Omni at {self.base_url}: {e}")
        try:
            model_list = data["data"]
            model_found = next((True for m in model_list if m["id"] == model), False)
        except (KeyError, TypeError):
            raise RuntimeError(f"Invalid JSON response of the hosted model list: {data}")

        if not model_found:
            raise ValueError(f"Model {model} not served at {self.base_url}.")

    @staticmethod
    def _prepare_chat_completion_messages(
        *,
        model: str,
        prompt: str,
        negative_prompt: str | None,
        image: torch.Tensor | None = None,
        audio: AudioInput | None = None,
        video: VideoInput | None = None,
        sampling_params: dict | list[dict] | None = None,
        modalities: list[str] | None = None,  # diffusion don't have this field
        **extra_body,
    ):
        message_content: list[dict] = [{"type": "text", "text": prompt}]
        if image is not None:
            message_content.append(
                {
                    "type": "image_url",
                    "image_url": {"url": image_tensor_to_base64(image)},
                }
            )
        if audio is not None:
            message_content.append({"type": "audio_url", "audio_url": {"url": audio_to_base64(audio)}})
        if video is not None:
            message_content.append({"type": "video_url", "video_url": {"url": video_to_base64(video)}})
        messages = [{"role": "user", "content": message_content}]

        payload: dict[str, Any] = {"messages": messages, "model": model}
        if modalities:
            payload["modalities"] = modalities

        combined_extra_body: dict[str, Any] = {}
        if sampling_params is not None:
            spec, _ = lookup_model_spec(model)
            is_single_sampling_param = isinstance(sampling_params, dict) or len(sampling_params) == 1

            if (spec is None and is_single_sampling_param) or (spec is not None and spec["stages"] == ["diffusion"]):
                # Diffusion format: extra_body directly contains sampling params.
                # Validation has already taken care of matching sampling params' types and length. Safe to take [0].
                # * Use this mode if the model is a simple one-stage diffusion model.
                # * Fallback to this mode if model is not registered and a single sampling param is provided.
                sampling_params = sampling_params if isinstance(sampling_params, dict) else sampling_params[0]
                combined_extra_body: dict[str, Any] = sampling_params.copy()
                if "n" in combined_extra_body:
                    combined_extra_body["num_outputs_per_prompt"] = combined_extra_body.pop("n")
            else:
                # AR format: the payload has a sampling_params_list field, containing a list.
                sampling_params_list = sampling_params if isinstance(sampling_params, list) else [sampling_params]
                payload["sampling_params_list"] = sampling_params_list

        if negative_prompt:
            combined_extra_body["negative_prompt"] = negative_prompt

        if extra_body:
            combined_extra_body.update(extra_body)

        # Add extra_body only if it has any content.
        if combined_extra_body:
            payload["extra_body"] = combined_extra_body

        # Place to inject any model-specific payload adjustment
        spec, _ = lookup_model_spec(model)
        if spec:
            preprocessor = spec.get("payload_preprocessor", None)
            if preprocessor is not None:
                payload = preprocessor(payload)

        return payload