vllm-omni/vllm_omni/model_executor/models/utils.py

from collections.abc import Callable
from contextlib import contextmanager

import torch
from vllm.model_executor.models.utils import maybe_prefix


@contextmanager
def transformers_keys_to_ignore_compat():
    """Make ``trust_remote_code`` weight loading robust to the transformers 5.9
    ``_keys_to_ignore_on_load_unexpected`` list-vs-set change.

    transformers 5.9 rewrote ``PreTrainedModel._adjust_missing_and_unexpected_keys``
    from ``(attr or []) + patterns`` (list concatenation) to
    ``(attr or set()) | patterns`` (set union). Remote-code models such as
    ``OpenMOSS-Team/MOSS-TTS-Nano`` still declare
    ``_keys_to_ignore_on_load_unexpected`` as a *list*, so ``list | set`` raises
    ``TypeError: unsupported operand type(s) for |: 'list' and 'set'`` and the
    engine core dies during model load.

    Wrap any ``from_pretrained(..., trust_remote_code=True)`` call whose remote
    code may declare the attribute as a list. The guard keeps such models
    loadable regardless of which transformers version is installed, while
    preserving the model's ignore patterns.
    """
    try:
        from transformers.modeling_utils import PreTrainedModel
    except Exception:  # pragma: no cover - transformers is always present here
        yield
        return

    orig = getattr(PreTrainedModel, "_adjust_missing_and_unexpected_keys", None)
    if orig is None:
        yield
        return

    def _wrapper(self, *args, **kwargs):
        try:
            return orig(self, *args, **kwargs)
        except TypeError as exc:
            if "unsupported operand type" not in str(exc):
                raise
            attr = getattr(self, "_keys_to_ignore_on_load_unexpected", None)
            if not isinstance(attr, (list, tuple)):
                raise
            # transformers >=5.9 combines patterns with ``set | ...``. The
            # ignore-pattern combine runs before any mutation of ``loading_info``
            # (see PreTrainedModel._adjust_missing_and_unexpected_keys), so it is
            # safe to coerce the list to a set and re-run the original once.
            self._keys_to_ignore_on_load_unexpected = set(attr)
            try:
                return orig(self, *args, **kwargs)
            finally:
                self._keys_to_ignore_on_load_unexpected = attr

    PreTrainedModel._adjust_missing_and_unexpected_keys = _wrapper
    try:
        yield
    finally:
        PreTrainedModel._adjust_missing_and_unexpected_keys = orig


def add_prefix_to_loaded_weights(weights: set[str], prefix: str) -> set[str]:
    """
    Add a prefix to the names of the loaded weights.
    """
    return {maybe_prefix(prefix, name) for name in weights}


def split_list_into_ranges(lst: torch.Tensor, interval: int) -> list[list[int]]:
    if lst.numel() == 0:
        return []

    # Move to CPU and convert to list once (High Speedup)
    # using .item() inside a loop is very slow.
    data_list = lst.detach().cpu().tolist()

    # Calculate max on the list or tensor (Tensor max is fast enough)
    max_val = int(torch.max(lst).item())

    # Pre-allocate buckets
    ranges: list[list[int]] = [[] for _ in range((max_val // interval) + 1)]

    for num in data_list:
        index = int(num // interval)
        ranges[index].append(num)

    return ranges


def safe_tensor_reshape(tensor: torch.Tensor, shape: tuple) -> torch.Tensor:
    """
    Reshape a tensor safely.
    """
    if tensor is None:
        return None
    return tensor.reshape(shape)


def reinit_rotary_inv_freq(
    model: torch.nn.Module,
    base: float = 10000.0,
    match: Callable[[str, torch.nn.Module], bool] | None = None,
) -> int:
    """Recompute ``inv_freq`` buffers on RoPE modules in-place.

    Custom RoPE classes loaded via ``trust_remote_code`` that register
    ``inv_freq`` with ``persistent=False`` and are not in
    ``ROPE_INIT_FUNCTIONS`` come out of ``from_pretrained`` with garbage
    buffer values (shape and dtype correct, contents not). ``cos()`` /
    ``sin()`` of those values produce NaN, so the first forward emits
    NaN logits. Mainstream HF RoPE classes avoid this via
    ``_rope_init_function`` framework integration.

    Recomputes ``1.0 / base^(arange(0, head_dim, 2) / head_dim)``.
    ``head_dim`` is inferred from ``2 * inv_freq.numel()``. Pass
    ``match`` to override the default selector (modules whose
    qualified name ends in ``"rotary_emb"`` and that expose a 1-D
    float ``inv_freq`` tensor). Returns the number of buffers
    re-initialised.
    """
    n_fixed = 0
    for name, module in model.named_modules():
        if match is not None:
            if not match(name, module):
                continue
        elif not name.endswith("rotary_emb"):
            continue
        inv_freq = getattr(module, "inv_freq", None)
        if not isinstance(inv_freq, torch.Tensor) or inv_freq.ndim != 1:
            continue
        head_dim = inv_freq.numel() * 2
        new_inv_freq = 1.0 / (
            base ** (torch.arange(0, head_dim, 2, dtype=torch.float32, device=inv_freq.device) / head_dim)
        )
        with torch.no_grad():
            inv_freq.copy_(new_inv_freq.to(dtype=inv_freq.dtype))
        n_fixed += 1
    return n_fixed


def is_interleaved(config) -> bool:
    """Detect if the model with this config is used with interleaved attention.

    Replicates the helper that was removed from upstream
    ``vllm.transformers_utils.config`` (vLLM commit 26d725c334,
    "[Model] Add VaultGemma via Transformers modeling backend") so models
    that still need the check can share one copy.
    """
    text_config = config.get_text_config()
    if layer_types := getattr(text_config, "layer_types", None):
        return len(set(layer_types)) > 1
    return False