vllm-omni/vllm_omni/diffusion/models/ltx2/ltx2_adapter_parser.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""Parse official LTX distilled adapters without materializing their tensors.

The official LTX adapters are raw safetensors files rather than PEFT
checkpoints.  This module is deliberately LTX-specific: it translates the
official names into the locally constructed Diffusers transformer names and
describes the resulting adapter targets.  Applying those targets is the job
of :mod:`ltx2_phase_adapter`.
"""

from __future__ import annotations

from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import Path

import torch
import torch.nn as nn
from safetensors import safe_open

_IGNORED_OFFICIAL_LORA_MODULES = {"text_embedding_projection.aggregate_embed"}
_OFFICIAL_TO_DIFFUSERS_PREFIXES = (
    ("audio_prompt_adaln_single.", "audio_prompt_adaln."),
    ("prompt_adaln_single.", "prompt_adaln."),
    ("audio_adaln_single.", "audio_time_embed."),
    ("adaln_single.", "time_embed."),
    ("audio_patchify_proj", "audio_proj_in"),
    ("patchify_proj", "proj_in"),
    ("av_ca_video_scale_shift_adaln_single.", "av_cross_attn_video_scale_shift."),
    ("av_ca_audio_scale_shift_adaln_single.", "av_cross_attn_audio_scale_shift."),
    ("av_ca_a2v_gate_adaln_single.", "av_cross_attn_video_a2v_gate."),
    ("av_ca_v2a_gate_adaln_single.", "av_cross_attn_audio_v2a_gate."),
)


@dataclass(frozen=True)
class AdapterTarget:
    """One A/B pair mapped onto a local transformer linear layer."""

    # Keep the original Diffusers module name for diagnostics after LTX packs
    # Q/K/V into the runtime module path.
    source_module: str
    module: str
    slice: str | None
    a_key: str
    b_key: str
    a_shape: tuple[int, ...]
    b_shape: tuple[int, ...]


@dataclass(frozen=True)
class AdapterManifest:
    """Tensor-free description of the fixed refinement adapter."""

    path: str
    targets: tuple[AdapterTarget, ...]


def _to_diffusers_module_name(name: str) -> str:
    for official_prefix, diffusers_prefix in _OFFICIAL_TO_DIFFUSERS_PREFIXES:
        if name.startswith(official_prefix):
            return diffusers_prefix + name[len(official_prefix) :]
    return name


def _resolve_lora_target(transformer: nn.Module, module_name: str) -> tuple[str, str | int | None] | None:
    modules = dict(transformer.named_modules())
    if module_name in modules:
        return module_name, None
    for packed_suffix, sub_suffix, shard_id in getattr(transformer, "stacked_params_mapping", ()):
        if sub_suffix in module_name:
            packed_name = module_name.replace(sub_suffix, packed_suffix)
            if packed_name in modules:
                return packed_name, shard_id
    return None


def parse_ltx_adapter(transformer: nn.Module, path: str) -> AdapterManifest:
    """Translate one official raw LTX adapter into local Transformer targets."""
    path = str(Path(path))
    targets: list[AdapterTarget] = []
    unresolved: list[str] = []
    seen: set[tuple[str, str | int | None]] = set()

    with safe_open(path, framework="pt", device="cpu") as handle:
        tensor_names = set(handle.keys())
        official_name_set = {
            tensor_name.removeprefix("diffusion_model.").rsplit(".lora_", 1)[0]
            for tensor_name in tensor_names
            if tensor_name.endswith(".lora_A.weight")
        }
        official_b_names = {
            tensor_name.removeprefix("diffusion_model.").rsplit(".lora_", 1)[0]
            for tensor_name in tensor_names
            if tensor_name.endswith(".lora_B.weight")
        }
        missing_a = sorted(official_b_names - official_name_set - _IGNORED_OFFICIAL_LORA_MODULES)
        if missing_a:
            raise ValueError(f"Missing paired LoRA A tensors for {missing_a} in {path}.")

        official_names = sorted(official_name_set)
        for official_name in official_names:
            if official_name in _IGNORED_OFFICIAL_LORA_MODULES:
                continue
            key_prefix = f"diffusion_model.{official_name}"
            key_a = f"{key_prefix}.lora_A.weight"
            key_b = f"{key_prefix}.lora_B.weight"
            if key_b not in tensor_names:
                raise ValueError(f"Missing paired LoRA tensor {key_b!r} in {path}.")

            a_shape = tuple(handle.get_slice(key_a).get_shape())
            b_shape = tuple(handle.get_slice(key_b).get_shape())
            if len(a_shape) != 2 or len(b_shape) != 2 or b_shape[1] != a_shape[0]:
                raise ValueError(f"Invalid LoRA pair for {official_name}: A={a_shape}, B={b_shape}.")

            source_module = _to_diffusers_module_name(official_name)
            resolved = _resolve_lora_target(transformer, source_module)
            if resolved is None:
                unresolved.append(official_name)
                continue
            module, shard_id = resolved
            target_id = (module, shard_id)
            if target_id in seen:
                raise ValueError(f"Official LTX LoRA contains duplicate target {module!r}, slice {shard_id!r}.")
            seen.add(target_id)
            targets.append(
                AdapterTarget(
                    source_module=source_module,
                    module=module,
                    slice=shard_id,
                    a_key=key_a,
                    b_key=key_b,
                    a_shape=a_shape,
                    b_shape=b_shape,
                )
            )

    if unresolved:
        raise ValueError(f"Official LTX LoRA contains unmapped modules: {unresolved}.")
    if not targets:
        raise ValueError(f"No Transformer LoRA tensors were found in {path}.")
    return AdapterManifest(path=path, targets=tuple(targets))


def iter_adapter_tensors(manifest: AdapterManifest) -> Iterator[tuple[AdapterTarget, torch.Tensor, torch.Tensor]]:
    """Stream adapter pairs in manifest order without a full checkpoint copy."""

    with safe_open(manifest.path, framework="pt", device="cpu") as handle:
        for target in manifest.targets:
            yield (
                target,
                handle.get_tensor(target.a_key),
                handle.get_tensor(target.b_key),
            )