vllm-omni/vllm_omni/inputs/preprocess.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Omni renderer: the upstream renderer class with Omni engine-input routing.

``OmniRenderer`` is a mixin. ``omni_renderer_cls`` puts it in front of a
concrete ``BaseRenderer`` subclass, so the Omni instance *is* an upstream
renderer (tokenizer, executors, multimodal cache and every public entry point
come from upstream unchanged) and only ``_process_singleton`` /
``_process_singleton_async`` are extended. ``build_omni_renderer`` resolves
the concrete class the same way upstream ``renderer_from_config`` does.
"""

from __future__ import annotations

from typing import TYPE_CHECKING, cast

from vllm.renderers import BaseRenderer
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.registry import cached_tokenizer_from_config, tokenizer_args_from_config

if TYPE_CHECKING:
    from vllm.config import VllmConfig
    from vllm.tokenizers import TokenizerLike


class OmniRenderer:
    """Mixin that routes Omni prompts through the upstream renderer.

    It has no base class of its own; ``omni_renderer_cls`` prepends it to a
    concrete ``BaseRenderer`` subclass, so ``self`` and ``super()`` are that
    renderer at runtime (hence the casts below).

    Two extensions over upstream ``_process_singleton``:

    * a token prompt that carries ``mm_processor_kwargs`` but no
      ``multi_modal_data`` still goes through the multimodal processor (AR
      image-generation models such as GLM-Image read their target size from
      those kwargs);
    * Omni-only prompt keys (``prompt``, ``cache_salt``,
      ``additional_information``, ``model_intermediate_buffer``) are copied
      onto the engine input.
    """

    @staticmethod
    def _with_omni_extras(inputs, prompt):
        for key in ("prompt", "cache_salt", "additional_information", "model_intermediate_buffer"):
            if key in prompt:
                inputs[key] = prompt[key]
        return inputs

    @staticmethod
    def _routes_no_media_kwargs(prompt) -> bool:
        return "prompt_embeds" not in prompt and "mm_processor_kwargs" in prompt and not prompt.get("multi_modal_data")

    def _process_singleton(self, prompt, *, skip_mm_cache: bool = False):
        renderer = cast(BaseRenderer, self)
        if self._routes_no_media_kwargs(prompt):
            inputs = renderer._process_multimodal(
                prompt["prompt_token_ids"],
                {},
                mm_processor_kwargs=prompt["mm_processor_kwargs"],
                mm_uuids=prompt.get("multi_modal_uuids"),
                skip_mm_cache=skip_mm_cache,
            )
        else:
            inputs = cast(BaseRenderer, super())._process_singleton(prompt, skip_mm_cache=skip_mm_cache)
        return self._with_omni_extras(inputs, prompt)

    async def _process_singleton_async(self, prompt, *, skip_mm_cache: bool = False):
        renderer = cast(BaseRenderer, self)
        if self._routes_no_media_kwargs(prompt):
            inputs = await renderer._process_multimodal_async(
                prompt["prompt_token_ids"],
                {},
                mm_processor_kwargs=prompt["mm_processor_kwargs"],
                mm_uuids=prompt.get("multi_modal_uuids"),
                skip_mm_cache=skip_mm_cache,
            )
        else:
            inputs = await cast(BaseRenderer, super())._process_singleton_async(prompt, skip_mm_cache=skip_mm_cache)
        return self._with_omni_extras(inputs, prompt)


_OMNI_RENDERER_CLASSES: dict[type[BaseRenderer], type[BaseRenderer]] = {}


def omni_renderer_cls(base_cls: type[BaseRenderer]) -> type[BaseRenderer]:
    """Return the Omni subclass of ``base_cls`` (created once per base class)."""
    if issubclass(base_cls, OmniRenderer):
        return base_cls
    cls = _OMNI_RENDERER_CLASSES.get(base_cls)
    if cls is None:
        name = f"Omni{base_cls.__name__}"
        cls = type(name, (OmniRenderer, base_cls), {"__module__": __name__, "__qualname__": name})
        _OMNI_RENDERER_CLASSES[base_cls] = cls
    return cls


def build_omni_renderer(
    vllm_config: VllmConfig,
    *,
    renderer_cls: type[BaseRenderer] | None = None,
    tokenizer: TokenizerLike | None = None,
) -> BaseRenderer:
    """Build the Omni renderer for ``vllm_config``.

    With ``renderer_cls`` unset this mirrors upstream ``renderer_from_config``:
    the tokenizer and renderer mode come from the model config and the class
    from ``RENDERER_REGISTRY``. Pass ``renderer_cls`` (and optionally
    ``tokenizer``) to skip that resolution, e.g. for tokenizer-less stages.
    """
    model_config = vllm_config.model_config
    if renderer_cls is None:
        if tokenizer is None:
            tokenizer = cached_tokenizer_from_config(model_config)
        renderer_mode, *_ = tokenizer_args_from_config(model_config)
        renderer_cls = RENDERER_REGISTRY.load_renderer_cls(renderer_mode)
    return omni_renderer_cls(renderer_cls)(vllm_config, tokenizer)