[Diffusion] Add LTX-2.5 Diffusion VAE decoder support (#6189)
Signed-off-by: mglyn <1203789601@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
262ff14693
commit
5e4daaf197
22 files changed
+1832
-61
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared component construction helpers for the LTX model family."""
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from diffusers import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoVaeNeighborhoodNattenProcessor
|
||||
from diffusers.pipelines.ltx2 import LTX2TextConnectors
|
||||
from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
|
||||
from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder
|
||||
@@ -35,6 +36,12 @@ from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
from .ltx2_diffusion_decoder import (
|
||||
LTX25_NATIVE_ARTIFACT_REVISION,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_FILENAME,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_REPO_ID,
|
||||
)
|
||||
from .ltx2_diffusion_decoder_distributed import DistributedLTX2VideoDiffusionDecoderModel
|
||||
from .ltx2_request import LTXCheckpointKind, validate_ltx_checkpoint
|
||||
from .ltx2_transformer import (
|
||||
LTX2VideoTransformer3DModel,
|
||||
@@ -69,6 +76,18 @@ _LTX_COMPONENT_SUBFOLDERS = (
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LTX2_CONV_VAE_EXTRA = "ltx2_use_conv_vae"
|
||||
_LTX2_DIFFUSION_DECODER_SUBFOLDER = "diffusion_decoder"
|
||||
|
||||
|
||||
def _ltx2_use_diffusion_decoder(od_config: Any, model_version: str) -> bool:
|
||||
"""Select DiffVAE by default for LTX-2.5, with an explicit ConvVAE opt-in."""
|
||||
extras = getattr(od_config, "extras", {}) or {}
|
||||
use_conv_vae = extras.get(_LTX2_CONV_VAE_EXTRA, False)
|
||||
if not isinstance(use_conv_vae, bool):
|
||||
raise TypeError(f"{_LTX2_CONV_VAE_EXTRA} must be a bool, got {type(use_conv_vae)!r}")
|
||||
return model_version == "2.5" and not use_conv_vae
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LTXComponentProfile:
|
||||
@@ -84,6 +103,7 @@ class LTXComponentProfile:
|
||||
text_encoder_cls: type | None = Gemma3ForConditionalGeneration
|
||||
vocoder_fallback_cls: type | None = None
|
||||
artifact_repo_id: str | None = None
|
||||
artifact_revision: str | None = None
|
||||
latent_upsampler_filename: str | None = None
|
||||
distilled_lora_filename: str | None = None
|
||||
transformer_subfolder: str = "transformer"
|
||||
@@ -202,6 +222,7 @@ LTX25_TWO_STAGE_COMPONENT_PROFILE = replace(
|
||||
name="ltx2_5_two_stage",
|
||||
resident_modules=(*LTX25_FULL_COMPONENT_PROFILE.resident_modules, "latent_upsampler"),
|
||||
artifact_repo_id="Lightricks/LTX-2.5",
|
||||
artifact_revision=LTX25_NATIVE_ARTIFACT_REVISION,
|
||||
latent_upsampler_filename=("latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors"),
|
||||
distilled_lora_filename="loras/ltx-2.5-22b-distilled-lora-450-bf16.safetensors",
|
||||
)
|
||||
@@ -239,14 +260,30 @@ def resolve_ltx_artifact(
|
||||
model: str,
|
||||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
model_revision: str | None,
|
||||
artifact_revision: str | None,
|
||||
) -> str:
|
||||
"""Resolve an official LTX sidecar from the model root or its Hub repository."""
|
||||
"""Resolve an official LTX sidecar without crossing repository revisions.
|
||||
|
||||
A local model path selects where the primary components are loaded from;
|
||||
it does not imply that independently hosted sidecars must be offline. Hub
|
||||
offline behavior remains controlled by huggingface_hub itself.
|
||||
"""
|
||||
candidate = Path(model) / filename
|
||||
if candidate.is_file():
|
||||
return str(candidate)
|
||||
|
||||
# Hub revisions are repository-scoped. Reuse the model revision only when
|
||||
# the model and artifact are in the same repository; otherwise use the
|
||||
# independently pinned artifact revision.
|
||||
revision = model_revision if model == repo_id else artifact_revision
|
||||
try:
|
||||
return hf_hub_download(repo_id=repo_id, filename=filename)
|
||||
return hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
revision=revision,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise FileNotFoundError(
|
||||
f"Unable to resolve LTX artifact {filename!r}. Searched {candidate}; "
|
||||
@@ -254,6 +291,18 @@ def resolve_ltx_artifact(
|
||||
) from exc
|
||||
|
||||
|
||||
def _create_ltx25_natten_processor() -> LTX2VideoVaeNeighborhoodNattenProcessor:
|
||||
"""Create the required production attention backend with an actionable error."""
|
||||
try:
|
||||
return LTX2VideoVaeNeighborhoodNattenProcessor()
|
||||
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||
raise RuntimeError(
|
||||
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
||||
"Install kernels==0.14.1, use a supported GPU, leave "
|
||||
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
||||
) from exc
|
||||
|
||||
|
||||
def _load_ltx_latent_upsampler_single_file(path: str, dtype: torch.dtype) -> LTX2LatentUpsamplerModel:
|
||||
"""Load an official single-file upsampler into the Diffusers module."""
|
||||
with safe_open(path, framework="pt", device="cpu") as handle:
|
||||
@@ -519,18 +568,49 @@ def _load_component(
|
||||
local_files_only: bool,
|
||||
dtype: torch.dtype,
|
||||
revision: str | None,
|
||||
prefetch_list: tuple[str, ...] = _LTX_COMPONENT_SUBFOLDERS,
|
||||
) -> Any:
|
||||
return from_pretrained_with_prefetch(
|
||||
component_cls.from_pretrained,
|
||||
model,
|
||||
subfolder=subfolder,
|
||||
prefetch_list=_LTX_COMPONENT_SUBFOLDERS,
|
||||
prefetch_list=prefetch_list,
|
||||
local_files_only=local_files_only,
|
||||
revision=revision,
|
||||
torch_dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
def _load_ltx25_native_diffusion_decoder(
|
||||
model: str,
|
||||
*,
|
||||
local_files_only: bool,
|
||||
dtype: torch.dtype,
|
||||
revision: str | None,
|
||||
) -> DistributedLTX2VideoDiffusionDecoderModel:
|
||||
"""Load canonical Native weights into the local Diffusers-compatible class."""
|
||||
config = DistributedLTX2VideoDiffusionDecoderModel.load_config(
|
||||
model,
|
||||
subfolder=_LTX2_DIFFUSION_DECODER_SUBFOLDER,
|
||||
local_files_only=local_files_only,
|
||||
revision=revision,
|
||||
)
|
||||
checkpoint_path = resolve_ltx_artifact(
|
||||
model,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_REPO_ID,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_FILENAME,
|
||||
model_revision=revision,
|
||||
artifact_revision=LTX25_NATIVE_ARTIFACT_REVISION,
|
||||
)
|
||||
decoder = DistributedLTX2VideoDiffusionDecoderModel.from_ltx25_native_checkpoint(
|
||||
checkpoint_path,
|
||||
config,
|
||||
dtype,
|
||||
)
|
||||
decoder.init_distributed()
|
||||
return decoder
|
||||
|
||||
|
||||
def _place_aux_components(pipeline: Any) -> None:
|
||||
parallel_config = getattr(pipeline.od_config, "parallel_config", None)
|
||||
use_managed_placement = bool(
|
||||
@@ -555,6 +635,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
model = od_config.model
|
||||
revision = getattr(od_config, "revision", None)
|
||||
local_files_only = os.path.exists(model)
|
||||
use_diffusion_decoder = pipeline.use_diffusion_decoder
|
||||
|
||||
pipeline.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
@@ -604,6 +685,27 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
dtype=dtype,
|
||||
revision=revision,
|
||||
)
|
||||
if use_diffusion_decoder:
|
||||
pipeline.diffusion_decoder = _load_ltx25_native_diffusion_decoder(
|
||||
model,
|
||||
local_files_only=local_files_only,
|
||||
dtype=dtype,
|
||||
revision=revision,
|
||||
)
|
||||
# The canonical decoder is trained and validated with NATTEN. Its
|
||||
# portable FlexAttention fallback materializes an impractically large
|
||||
# block mask at production video sizes, so fail early if the matching
|
||||
# Hub kernel cannot be loaded instead of failing later during decode.
|
||||
pipeline.diffusion_decoder.set_attn_processor(_create_ltx25_natten_processor())
|
||||
vae_patch_parallel_size = int(
|
||||
getattr(getattr(od_config, "parallel_config", None), "vae_patch_parallel_size", 1)
|
||||
)
|
||||
if vae_patch_parallel_size > 1 or getattr(od_config, "vae_use_tiling", False):
|
||||
pipeline.diffusion_decoder.enable_tiling()
|
||||
pipeline.diffusion_decoder.set_parallel_size(
|
||||
vae_patch_parallel_size,
|
||||
mode=getattr(getattr(od_config, "parallel_config", None), "vae_parallel_mode", "tile"),
|
||||
)
|
||||
pipeline.audio_vae = _load_component(
|
||||
AutoencoderKLLTX2Audio,
|
||||
model,
|
||||
@@ -656,6 +758,8 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
model,
|
||||
profile.artifact_repo_id,
|
||||
profile.latent_upsampler_filename,
|
||||
model_revision=revision,
|
||||
artifact_revision=profile.artifact_revision,
|
||||
)
|
||||
pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype)
|
||||
else:
|
||||
@@ -665,6 +769,8 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
model,
|
||||
profile.artifact_repo_id,
|
||||
profile.latent_upsampler_filename,
|
||||
model_revision=revision,
|
||||
artifact_revision=profile.artifact_revision,
|
||||
)
|
||||
pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Task-specific conditioning shared by LTX model versions."""
|
||||
|
||||
@@ -502,10 +502,12 @@ class LTXI2VConditioningMixin:
|
||||
init_latents = torch.cat(init_latents, dim=0).to(dtype=dtype)
|
||||
if num_videos_per_prompt > 1:
|
||||
init_latents = init_latents.repeat_interleave(num_videos_per_prompt, dim=0)
|
||||
latents_mean, latents_std, scaling_factor = latent_ops.resolve_video_latent_statistics(self)
|
||||
return latent_ops.normalize_latents(
|
||||
init_latents,
|
||||
self.vae.latents_mean,
|
||||
self.vae.latents_std,
|
||||
latents_mean,
|
||||
latents_std,
|
||||
scaling_factor,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -570,11 +572,12 @@ class LTXI2VConditioningMixin:
|
||||
)
|
||||
conditioning_mask = latents.new_zeros(mask_shape)
|
||||
conditioning_mask[:, :, 0] = 1.0
|
||||
latents_mean, latents_std, scaling_factor = latent_ops.resolve_video_latent_statistics(self)
|
||||
latents = latent_ops.normalize_latents(
|
||||
latents,
|
||||
self.vae.latents_mean,
|
||||
self.vae.latents_std,
|
||||
self.vae.config.scaling_factor,
|
||||
latents_mean,
|
||||
latents_std,
|
||||
scaling_factor,
|
||||
)
|
||||
clean_latents = latents
|
||||
if image is not None:
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# SPDX-FileCopyrightText: Copyright 2026 Lightricks and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Native conversion rules and decoder behavior copied and modified from
|
||||
# Diffusers commit d035dcd7cc7c88e0a154609b62887d50bba9fdc2 (Apache-2.0).
|
||||
|
||||
"""LTX-2.5 Native-weight adapter for Diffusers' diffusion VAE decoder."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from diffusers.models.autoencoders import (
|
||||
LTX2VideoDiffusionDecoderModel as DiffusersLTX2VideoDiffusionDecoderModel,
|
||||
)
|
||||
from diffusers.models.autoencoders.ltx2_diffusion_decoder import (
|
||||
LTX2VideoDiffusionDecoder3d as DiffusersLTX2VideoDiffusionDecoder3d,
|
||||
)
|
||||
from diffusers.models.autoencoders.vae import DecoderOutput
|
||||
from safetensors import safe_open
|
||||
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_REPO_ID = "Lightricks/LTX-2.5"
|
||||
LTX25_NATIVE_ARTIFACT_REVISION = "8a4ff96f581e72bedc1b44367581c49d544a05f1"
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_FILENAME = "vae/ltx-2.5-video-vae-bf16.safetensors"
|
||||
|
||||
_NATIVE_DECODER_PREFIXES = ("vae.decoder.", "decoder.")
|
||||
_NATIVE_STATISTICS_KEYS = {
|
||||
"per_channel_statistics.mean-of-means": "latents_mean",
|
||||
"per_channel_statistics.std-of-means": "latents_std",
|
||||
}
|
||||
_NATIVE_KEY_REPLACEMENTS = (
|
||||
("t_embedder.mlp.0.", "t_embedder.timestep_embedder.linear_1."),
|
||||
("t_embedder.mlp.2.", "t_embedder.timestep_embedder.linear_2."),
|
||||
(".attn.proj.", ".attn.to_out.0."),
|
||||
(".attn.q_norm.", ".attn.norm_q."),
|
||||
(".attn.k_norm.", ".attn.norm_k."),
|
||||
)
|
||||
_GATE_FOLD_TARGETS = {
|
||||
".attn.to_out.0.weight": ".gate_msa",
|
||||
".attn.to_out.0.bias": ".gate_msa",
|
||||
".mlp.w_down.weight": ".gate_mlp",
|
||||
".context_proj.weight": ".gate_ctx",
|
||||
".context_proj.bias": ".gate_ctx",
|
||||
}
|
||||
_GATE_SUFFIXES = tuple(_GATE_FOLD_TARGETS.values())
|
||||
|
||||
|
||||
def _strip_native_decoder_prefix(key: str) -> str | None:
|
||||
for prefix in _NATIVE_DECODER_PREFIXES:
|
||||
if key.startswith(prefix):
|
||||
return key.removeprefix(prefix)
|
||||
return None
|
||||
|
||||
|
||||
def _native_statistics_target(key: str) -> str | None:
|
||||
return _NATIVE_STATISTICS_KEYS.get(key.removeprefix("vae."))
|
||||
|
||||
|
||||
def _rename_native_decoder_key(key: str) -> str:
|
||||
for source, target in _NATIVE_KEY_REPLACEMENTS:
|
||||
key = key.replace(source, target)
|
||||
return key
|
||||
|
||||
|
||||
def _fold_native_gate(key: str, value: torch.Tensor, gates: Mapping[str, torch.Tensor]) -> torch.Tensor:
|
||||
target_suffix = next((suffix for suffix in _GATE_FOLD_TARGETS if key.endswith(suffix)), None)
|
||||
if target_suffix is None:
|
||||
return value
|
||||
gate_key = key[: -len(target_suffix)] + _GATE_FOLD_TARGETS[target_suffix]
|
||||
gate = gates.get(gate_key)
|
||||
if gate is None:
|
||||
return value
|
||||
gate = gate.to(device=value.device, dtype=torch.float32)
|
||||
value_float = value.to(dtype=torch.float32)
|
||||
folded = gate.unsqueeze(1) * value_float if value.ndim == 2 else gate * value_float
|
||||
return folded.to(dtype=value.dtype)
|
||||
|
||||
|
||||
def convert_ltx25_native_diffusion_decoder_state_dict(
|
||||
native_state_dict: Mapping[str, torch.Tensor],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Convert the canonical Native LTX-2.5 DiffVAE state dict to Diffusers."""
|
||||
decoder_state_dict: dict[str, torch.Tensor] = {}
|
||||
converted: dict[str, torch.Tensor] = {}
|
||||
for key, value in native_state_dict.items():
|
||||
statistics_target = _native_statistics_target(key)
|
||||
if statistics_target is not None:
|
||||
converted[statistics_target] = value
|
||||
continue
|
||||
decoder_key = _strip_native_decoder_prefix(key)
|
||||
if decoder_key is not None:
|
||||
decoder_state_dict[decoder_key] = value
|
||||
|
||||
gates = {key: value for key, value in decoder_state_dict.items() if key.endswith(_GATE_SUFFIXES)}
|
||||
for key, value in decoder_state_dict.items():
|
||||
# ``type_emb`` is an unused shipping-checkpoint residue. Preview heads
|
||||
# and legacy static gates are not parameters of the runtime decoder.
|
||||
if key == "type_emb" or key.startswith("coarse_") or ".coarse_" in key or key in gates:
|
||||
continue
|
||||
|
||||
converted_key = _rename_native_decoder_key(key)
|
||||
value = _fold_native_gate(converted_key, value, gates)
|
||||
if converted_key.endswith((".qkv.weight", ".qkv.bias")):
|
||||
leaf = "weight" if converted_key.endswith(".weight") else "bias"
|
||||
prefix = converted_key[: -len(f"qkv.{leaf}")]
|
||||
if value.shape[0] % 3 != 0:
|
||||
raise ValueError(
|
||||
f"Fused LTX-2.5 DiffVAE parameter {key!r} has leading dimension "
|
||||
f"{value.shape[0]}, which is not divisible by 3."
|
||||
)
|
||||
chunk = value.shape[0] // 3
|
||||
converted[f"decoder.{prefix}to_q.{leaf}"] = value[:chunk].clone()
|
||||
converted[f"decoder.{prefix}to_k.{leaf}"] = value[chunk : 2 * chunk].clone()
|
||||
converted[f"decoder.{prefix}to_v.{leaf}"] = value[2 * chunk :].clone()
|
||||
continue
|
||||
|
||||
converted[f"decoder.{converted_key}"] = value
|
||||
|
||||
return converted
|
||||
|
||||
|
||||
def load_ltx25_native_diffusion_decoder_state_dict(path: str) -> dict[str, torch.Tensor]:
|
||||
"""Read and convert only DiffVAE tensors from the canonical Native VAE file."""
|
||||
with safe_open(path, framework="pt", device="cpu") as handle:
|
||||
native_state_dict = {
|
||||
key: handle.get_tensor(key)
|
||||
for key in handle.keys()
|
||||
if _strip_native_decoder_prefix(key) is not None or _native_statistics_target(key) is not None
|
||||
}
|
||||
return convert_ltx25_native_diffusion_decoder_state_dict(native_state_dict)
|
||||
|
||||
|
||||
class LTX2VideoDiffusionDecoder3d(DiffusersLTX2VideoDiffusionDecoder3d):
|
||||
"""Diffusers decoder core with the short-clip NATTEN context fix."""
|
||||
|
||||
def forward_stage_4(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
drop_leading_frame: bool = True,
|
||||
crop_trailing_ghost: bool = True,
|
||||
) -> torch.Tensor:
|
||||
blocks = self.det_stages[-1]
|
||||
block_mask = blocks[0].attn.build_block_mask(hidden_states)
|
||||
for block in blocks:
|
||||
hidden_states = block(hidden_states, block_mask)
|
||||
hidden_states = self.upsamples[-1](hidden_states, drop_leading_frame=drop_leading_frame)
|
||||
|
||||
num_pad = self.trailing_pad_latent_frames
|
||||
if crop_trailing_ghost and num_pad > 0:
|
||||
content_frames = max(hidden_states.shape[1] - num_pad * self.temporal_compression_ratio, 1)
|
||||
keep_frames = min(hidden_states.shape[1], max(content_frames, self.stage5_kernel[0]))
|
||||
hidden_states = hidden_states[:, :keep_frames]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class LTX2VideoDiffusionDecoderModel(DiffusersLTX2VideoDiffusionDecoderModel):
|
||||
"""Diffusers 0.40 decoder plus canonical Native loading and short-clip support."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
out_channels: int = 3,
|
||||
latent_channels: int = 128,
|
||||
patch_size: int = 4,
|
||||
scaling_factor: float = 1.0,
|
||||
decoder_head_dim: int = 64,
|
||||
decoder_stage_channels: tuple[int, ...] = (2048, 1024, 512, 512, 256),
|
||||
decoder_stage_depths: tuple[int, ...] = (4, 6, 4, 2, 8),
|
||||
decoder_stage_kernels: tuple[tuple[int, int, int], ...] = ((3, 7, 7), (3, 7, 7), (3, 5, 5), (3, 5, 5)),
|
||||
decoder_upsample_strides: tuple[tuple[int, int, int], ...] = ((1, 2, 2), (2, 1, 1), (2, 2, 2), (2, 2, 2)),
|
||||
decoder_upsample_channel_reductions: tuple[int, ...] = (2, 2, 1, 2),
|
||||
decoder_stage5_kernel: tuple[int, int, int] = (11, 11, 11),
|
||||
decoder_t_emb_dim: int = 384,
|
||||
decoder_timestep_scale_multiplier: float = 1000.0,
|
||||
decoder_model_output_type: str = "x0",
|
||||
decoder_num_inference_steps: int = 1,
|
||||
spatial_compression_ratio: int = 32,
|
||||
temporal_compression_ratio: int = 8,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
out_channels=out_channels,
|
||||
latent_channels=latent_channels,
|
||||
patch_size=patch_size,
|
||||
scaling_factor=scaling_factor,
|
||||
decoder_head_dim=decoder_head_dim,
|
||||
decoder_stage_channels=decoder_stage_channels,
|
||||
decoder_stage_depths=decoder_stage_depths,
|
||||
decoder_stage_kernels=decoder_stage_kernels,
|
||||
decoder_upsample_strides=decoder_upsample_strides,
|
||||
decoder_upsample_channel_reductions=decoder_upsample_channel_reductions,
|
||||
decoder_stage5_kernel=decoder_stage5_kernel,
|
||||
decoder_t_emb_dim=decoder_t_emb_dim,
|
||||
decoder_timestep_scale_multiplier=decoder_timestep_scale_multiplier,
|
||||
decoder_model_output_type=decoder_model_output_type,
|
||||
decoder_num_inference_steps=decoder_num_inference_steps,
|
||||
spatial_compression_ratio=spatial_compression_ratio,
|
||||
temporal_compression_ratio=temporal_compression_ratio,
|
||||
)
|
||||
# Diffusers constructs its private decoder core internally. Switching
|
||||
# to this behavior-only subclass preserves parameters and state-dict keys.
|
||||
self.decoder.__class__ = LTX2VideoDiffusionDecoder3d
|
||||
self.decoder.stage5_kernel = tuple(decoder_stage5_kernel)
|
||||
|
||||
@classmethod
|
||||
def from_ltx25_native_checkpoint(
|
||||
cls,
|
||||
checkpoint_path: str,
|
||||
config: Mapping[str, Any],
|
||||
dtype: torch.dtype,
|
||||
) -> "LTX2VideoDiffusionDecoderModel":
|
||||
"""Construct this decoder class directly from the canonical Native checkpoint."""
|
||||
with torch.device("meta"):
|
||||
model = cls.from_config(dict(config))
|
||||
state_dict = load_ltx25_native_diffusion_decoder_state_dict(checkpoint_path)
|
||||
try:
|
||||
model.load_state_dict(state_dict, strict=True, assign=True)
|
||||
except RuntimeError as exc:
|
||||
raise ValueError(f"Invalid LTX-2.5 Native DiffVAE checkpoint {checkpoint_path!r}.") from exc
|
||||
model.to(device="cpu", dtype=dtype)
|
||||
return model
|
||||
|
||||
def decode(
|
||||
self,
|
||||
z: torch.Tensor,
|
||||
generator: torch.Generator | None = None,
|
||||
num_inference_steps: int | None = None,
|
||||
return_dict: bool = True,
|
||||
) -> DecoderOutput | tuple[torch.Tensor]:
|
||||
decoded = super().decode(
|
||||
z,
|
||||
generator=generator,
|
||||
num_inference_steps=num_inference_steps,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
# Short clips retain replicated trailing context through stage 5. Do
|
||||
# not expose those context pixels to the caller.
|
||||
target_num_frames = (z.shape[2] - 1) * self.temporal_compression_ratio + 1
|
||||
target_height = z.shape[3] * self.spatial_compression_ratio
|
||||
target_width = z.shape[4] * self.spatial_compression_ratio
|
||||
decoded = decoded[:, :, :target_num_frames, :target_height, :target_width]
|
||||
|
||||
if not return_dict:
|
||||
return (decoded,)
|
||||
return DecoderOutput(sample=decoded)
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# SPDX-FileCopyrightText: Copyright 2026 Lightricks and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Distributed tiling is copied and modified from Diffusers' serial LTX-2.5
|
||||
# tiling at commit d035dcd7cc7c88e0a154609b62887d50bba9fdc2 (Apache-2.0).
|
||||
|
||||
"""LTX-2.5-specific distributed execution for the diffusion VAE decoder."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from diffusers.utils import logging
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedOperator,
|
||||
DistributedVaeMixin,
|
||||
GridSpec,
|
||||
TileTask,
|
||||
)
|
||||
|
||||
from .ltx2_diffusion_decoder import LTX2VideoDiffusionDecoderModel
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
|
||||
|
||||
def _tile_intervals(length: int, tile_size: int, stride: int, min_size: int) -> list[tuple[int, int]]:
|
||||
"""Build overlapping intervals, merging a too-small trailing remnant."""
|
||||
if length <= tile_size:
|
||||
return [(0, length)]
|
||||
starts = list(range(0, length, stride))
|
||||
while len(starts) > 1 and length - starts[-1] < min_size:
|
||||
starts.pop()
|
||||
return [(start, min(start + tile_size, length)) for start in starts[:-1]] + [(starts[-1], length)]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LTX2VideoDiffusionTilePlan:
|
||||
"""Geometry shared by all ranks for one distributed DiffVAE decode."""
|
||||
|
||||
temporal_tiles: tuple[tuple[int, int], ...]
|
||||
height_tiles: tuple[tuple[int, int], ...]
|
||||
width_tiles: tuple[tuple[int, int], ...]
|
||||
num_frames: int
|
||||
height: int
|
||||
width: int
|
||||
scale_t: int
|
||||
scale_h: int
|
||||
scale_w: int
|
||||
stride_t: int
|
||||
stride_h: int
|
||||
stride_w: int
|
||||
blend_frames: int
|
||||
blend_height: int
|
||||
blend_width: int
|
||||
single_step_x0: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class LTX2VideoDiffusionTileTask(TileTask):
|
||||
"""One stage-4 + stage-5 tile and its rank-independent noise."""
|
||||
|
||||
drop_leading_frame: bool = False
|
||||
crop_trailing_ghost: bool = False
|
||||
noise_generator: torch.Generator | list[torch.Generator] | None = None
|
||||
x_t: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DistributedLTX2VideoDiffusionDecoderModel(LTX2VideoDiffusionDecoderModel, DistributedVaeMixin):
|
||||
"""LTX-2.5 diffusion decoder with model-specific tile parallelism.
|
||||
|
||||
Stages 1-3 run over the complete low-resolution feature volume on every
|
||||
participating rank. Stage 4 and the diffusion stage run as independent
|
||||
overlapping tile tasks, and rank 0 performs the reference blend/merge.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args: Any, **kwargs: Any):
|
||||
model = super().from_pretrained(*args, **kwargs)
|
||||
model.init_distributed()
|
||||
return model
|
||||
|
||||
def set_parallel_size(self, parallel_size: int, mode: str = "tile") -> None:
|
||||
if mode != "tile":
|
||||
raise ValueError(f"LTX-2.5 DiffVAE only supports vae_parallel_mode='tile', got {mode!r}.")
|
||||
super().set_parallel_size(parallel_size, mode=mode)
|
||||
|
||||
def _build_tiled_decode_plan(
|
||||
self,
|
||||
features: torch.Tensor,
|
||||
num_inference_steps: int,
|
||||
) -> LTX2VideoDiffusionTilePlan:
|
||||
"""Build the rank-independent tile schedule around Diffusers' decoder."""
|
||||
decoder = self.decoder
|
||||
patch_size = decoder.patch_size
|
||||
upsample_stride = decoder.upsamples[-1].stride
|
||||
scale_t, scale_h, scale_w = (
|
||||
upsample_stride[0],
|
||||
upsample_stride[1] * patch_size,
|
||||
upsample_stride[2] * patch_size,
|
||||
)
|
||||
tile_t = self.tile_sample_min_num_frames // scale_t
|
||||
tile_h = self.tile_sample_min_height // scale_h
|
||||
tile_w = self.tile_sample_min_width // scale_w
|
||||
stride_t = self.tile_sample_stride_num_frames // scale_t
|
||||
stride_h = self.tile_sample_stride_height // scale_h
|
||||
stride_w = self.tile_sample_stride_width // scale_w
|
||||
min_sizes = [
|
||||
max(kernel_4, -(-kernel_5 // stride))
|
||||
for kernel_4, kernel_5, stride in zip(
|
||||
self.config.decoder_stage_kernels[-1],
|
||||
self.config.decoder_stage5_kernel,
|
||||
upsample_stride,
|
||||
)
|
||||
]
|
||||
|
||||
ghost_frames = decoder.trailing_pad_latent_frames * math.prod(
|
||||
upsample.stride[0] for upsample in decoder.upsamples[:-1]
|
||||
)
|
||||
num_frames = features.shape[1] - ghost_frames
|
||||
height, width = features.shape[2:4]
|
||||
return LTX2VideoDiffusionTilePlan(
|
||||
temporal_tiles=tuple(_tile_intervals(num_frames, tile_t, stride_t, min_sizes[0])),
|
||||
height_tiles=tuple(_tile_intervals(height, tile_h, stride_h, min_sizes[1])),
|
||||
width_tiles=tuple(_tile_intervals(width, tile_w, stride_w, min_sizes[2])),
|
||||
num_frames=num_frames,
|
||||
height=height,
|
||||
width=width,
|
||||
scale_t=scale_t,
|
||||
scale_h=scale_h,
|
||||
scale_w=scale_w,
|
||||
stride_t=stride_t,
|
||||
stride_h=stride_h,
|
||||
stride_w=stride_w,
|
||||
blend_frames=(tile_t - stride_t) * scale_t,
|
||||
blend_height=(tile_h - stride_h) * scale_h,
|
||||
blend_width=(tile_w - stride_w) * scale_w,
|
||||
single_step_x0=num_inference_steps == 1 and decoder.model_output_type == "x0",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _iter_tiled_decode_coords(plan: LTX2VideoDiffusionTilePlan):
|
||||
for t_idx in range(len(plan.temporal_tiles)):
|
||||
for h_idx in range(len(plan.height_tiles)):
|
||||
for w_idx in range(len(plan.width_tiles)):
|
||||
yield t_idx, h_idx, w_idx
|
||||
|
||||
@staticmethod
|
||||
def _get_tiled_feature_slice(
|
||||
features: torch.Tensor,
|
||||
coord: tuple[int, int, int],
|
||||
plan: LTX2VideoDiffusionTilePlan,
|
||||
) -> tuple[torch.Tensor, bool, bool]:
|
||||
t_idx, h_idx, w_idx = coord
|
||||
t0, t1 = plan.temporal_tiles[t_idx]
|
||||
h0, h1 = plan.height_tiles[h_idx]
|
||||
w0, w1 = plan.width_tiles[w_idx]
|
||||
is_origin = t0 == 0
|
||||
is_trailing = t1 == plan.num_frames
|
||||
feature_t1 = features.shape[1] if is_trailing else t1
|
||||
return features[:, t0:feature_t1, h0:h1, w0:w1], is_origin, is_trailing
|
||||
|
||||
def _tiled_pixel_shape_from_features(
|
||||
self,
|
||||
feature_tile: torch.Tensor,
|
||||
*,
|
||||
drop_leading_frame: bool,
|
||||
crop_trailing_ghost: bool,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
"""Predict one output tile's shape without materializing stage 4."""
|
||||
decoder = self.decoder
|
||||
stride_t, stride_h, stride_w = decoder.upsamples[-1].stride
|
||||
num_frames = feature_tile.shape[1] * stride_t
|
||||
if stride_t == 2 and drop_leading_frame:
|
||||
num_frames -= 1
|
||||
if crop_trailing_ghost and decoder.trailing_pad_latent_frames > 0:
|
||||
content_frames = max(
|
||||
num_frames - decoder.trailing_pad_latent_frames * decoder.temporal_compression_ratio,
|
||||
1,
|
||||
)
|
||||
num_frames = min(num_frames, max(content_frames, decoder.stage5_kernel[0]))
|
||||
return (
|
||||
feature_tile.shape[0],
|
||||
decoder.out_channels,
|
||||
num_frames,
|
||||
feature_tile.shape[2] * stride_h * decoder.patch_size,
|
||||
feature_tile.shape[3] * stride_w * decoder.patch_size,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _slice_tiled_noise(
|
||||
x_t_full: torch.Tensor,
|
||||
coord: tuple[int, int, int],
|
||||
tile_pixel_shape: tuple[int, int, int, int, int],
|
||||
plan: LTX2VideoDiffusionTilePlan,
|
||||
) -> torch.Tensor:
|
||||
t_idx, h_idx, w_idx = coord
|
||||
t0, _ = plan.temporal_tiles[t_idx]
|
||||
h0, _ = plan.height_tiles[h_idx]
|
||||
w0, _ = plan.width_tiles[w_idx]
|
||||
pixel_t0 = t0 * plan.scale_t - (1 if t0 != 0 and plan.scale_t == 2 else 0)
|
||||
return x_t_full[
|
||||
:,
|
||||
:,
|
||||
pixel_t0 : pixel_t0 + tile_pixel_shape[2],
|
||||
h0 * plan.scale_h : h0 * plan.scale_h + tile_pixel_shape[3],
|
||||
w0 * plan.scale_w : w0 * plan.scale_w + tile_pixel_shape[4],
|
||||
]
|
||||
|
||||
def _merge_tiled_decode(
|
||||
self,
|
||||
tiles: dict[tuple[int, int, int], torch.Tensor],
|
||||
plan: LTX2VideoDiffusionTilePlan,
|
||||
) -> torch.Tensor:
|
||||
"""Blend distributed RGB tiles in Diffusers' serial order."""
|
||||
frame_groups = []
|
||||
for t_idx in range(len(plan.temporal_tiles)):
|
||||
rows = [
|
||||
[tiles[(t_idx, h_idx, w_idx)] for w_idx in range(len(plan.width_tiles))]
|
||||
for h_idx in range(len(plan.height_tiles))
|
||||
]
|
||||
result_rows = []
|
||||
for h_idx, row in enumerate(rows):
|
||||
result_row = []
|
||||
for w_idx, tile in enumerate(row):
|
||||
if h_idx > 0:
|
||||
tile = self.blend_v(rows[h_idx - 1][w_idx], tile, plan.blend_height)
|
||||
if w_idx > 0:
|
||||
tile = self.blend_h(row[w_idx - 1], tile, plan.blend_width)
|
||||
keep_height = plan.stride_h * plan.scale_h if h_idx < len(rows) - 1 else tile.shape[3]
|
||||
keep_width = plan.stride_w * plan.scale_w if w_idx < len(row) - 1 else tile.shape[4]
|
||||
result_row.append(tile[:, :, :, :keep_height, :keep_width])
|
||||
result_rows.append(torch.cat(result_row, dim=4))
|
||||
frame_groups.append(torch.cat(result_rows, dim=3))
|
||||
|
||||
result = []
|
||||
for t_idx, group in enumerate(frame_groups):
|
||||
if t_idx > 0:
|
||||
group = self.blend_t(frame_groups[t_idx - 1], group, plan.blend_frames)
|
||||
if t_idx < len(frame_groups) - 1:
|
||||
keep_frames = plan.stride_t * plan.scale_t - (1 if t_idx == 0 and plan.scale_t == 2 else 0)
|
||||
group = group[:, :, :keep_frames]
|
||||
result.append(group)
|
||||
return torch.cat(result, dim=2)
|
||||
|
||||
def _default_generator(self, device: torch.device) -> torch.Generator:
|
||||
if device.type == "cpu":
|
||||
return torch.default_generator
|
||||
device_module = getattr(torch, device.type, None)
|
||||
default_generators = getattr(device_module, "default_generators", None)
|
||||
if default_generators is None:
|
||||
raise ValueError(
|
||||
f"Distributed LTX-2.5 diffusion decode on {device.type!r} requires an explicit torch.Generator."
|
||||
)
|
||||
device_index = device.index
|
||||
if device_index is None:
|
||||
current_device = getattr(device_module, "current_device", None)
|
||||
device_index = current_device() if current_device is not None else 0
|
||||
return default_generators[device_index]
|
||||
|
||||
def _sync_generators(
|
||||
self,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
device: torch.device,
|
||||
) -> torch.Generator | list[torch.Generator]:
|
||||
"""Make every rank start tile-noise generation from rank 0's state."""
|
||||
generators = self._default_generator(device) if generator is None else generator
|
||||
generator_list = generators if isinstance(generators, list) else [generators]
|
||||
for item in generator_list:
|
||||
state_on_device = item.get_state().to(device=device)
|
||||
dist.broadcast(state_on_device, src=0, group=self.distributed_executor.group)
|
||||
item.set_state(state_on_device.cpu())
|
||||
return generators
|
||||
|
||||
@staticmethod
|
||||
def _clone_generator(generator: torch.Generator) -> torch.Generator:
|
||||
cloned = torch.Generator(device=generator.device)
|
||||
cloned.set_state(generator.get_state())
|
||||
return cloned
|
||||
|
||||
def _clone_generators(
|
||||
self,
|
||||
generators: torch.Generator | list[torch.Generator],
|
||||
) -> torch.Generator | list[torch.Generator]:
|
||||
if isinstance(generators, list):
|
||||
return [self._clone_generator(item) for item in generators]
|
||||
return self._clone_generator(generators)
|
||||
|
||||
def _distributed_tile_split(
|
||||
self,
|
||||
z: torch.Tensor,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
num_inference_steps: int,
|
||||
) -> tuple[list[LTX2VideoDiffusionTileTask], GridSpec]:
|
||||
decoder = self.decoder
|
||||
features = decoder.forward_stages_1_to_3(z)
|
||||
plan = self._build_tiled_decode_plan(features, num_inference_steps)
|
||||
generators = self._sync_generators(generator, z.device)
|
||||
|
||||
x_t_full = None
|
||||
if not plan.single_step_x0:
|
||||
pixel_frames = plan.num_frames * plan.scale_t - (1 if plan.scale_t == 2 else 0)
|
||||
x_t_full = randn_tensor(
|
||||
(
|
||||
z.shape[0],
|
||||
decoder.out_channels,
|
||||
pixel_frames,
|
||||
plan.height * plan.scale_h,
|
||||
plan.width * plan.scale_w,
|
||||
),
|
||||
generator=generators,
|
||||
device=z.device,
|
||||
dtype=z.dtype,
|
||||
)
|
||||
|
||||
tasks = []
|
||||
for tile_id, coord in enumerate(self._iter_tiled_decode_coords(plan)):
|
||||
feature_tile, is_origin, is_trailing = self._get_tiled_feature_slice(features, coord, plan)
|
||||
tile_pixel_shape = self._tiled_pixel_shape_from_features(
|
||||
feature_tile,
|
||||
drop_leading_frame=is_origin,
|
||||
crop_trailing_ghost=is_trailing,
|
||||
)
|
||||
tile_generator = None
|
||||
x_t = None
|
||||
if plan.single_step_x0:
|
||||
# Save the canonical serial state for this tile, then advance
|
||||
# the request generator exactly as the single-rank tiled path
|
||||
# does. Task placement therefore cannot change tile noise.
|
||||
tile_generator = self._clone_generators(generators)
|
||||
discarded_noise = randn_tensor(
|
||||
tile_pixel_shape,
|
||||
generator=generators,
|
||||
device=z.device,
|
||||
dtype=z.dtype,
|
||||
)
|
||||
del discarded_noise
|
||||
else:
|
||||
x_t = self._slice_tiled_noise(x_t_full, coord, tile_pixel_shape, plan)
|
||||
|
||||
tasks.append(
|
||||
LTX2VideoDiffusionTileTask(
|
||||
tile_id=tile_id,
|
||||
grid_coord=coord,
|
||||
tensor=feature_tile,
|
||||
workload=math.prod(tile_pixel_shape),
|
||||
drop_leading_frame=is_origin,
|
||||
crop_trailing_ghost=is_trailing,
|
||||
noise_generator=tile_generator,
|
||||
x_t=x_t,
|
||||
)
|
||||
)
|
||||
|
||||
return tasks, GridSpec(
|
||||
split_dims=(2, 3, 4),
|
||||
grid_shape=(len(plan.temporal_tiles), len(plan.height_tiles), len(plan.width_tiles)),
|
||||
tile_spec={"plan": plan},
|
||||
output_dtype=z.dtype,
|
||||
)
|
||||
|
||||
def _distributed_tile_exec(
|
||||
self,
|
||||
task: LTX2VideoDiffusionTileTask,
|
||||
num_inference_steps: int,
|
||||
) -> torch.Tensor:
|
||||
context = self.decoder.forward_stage_4(
|
||||
task.tensor,
|
||||
drop_leading_frame=task.drop_leading_frame,
|
||||
crop_trailing_ghost=task.crop_trailing_ghost,
|
||||
)
|
||||
tile_pixel_shape = (
|
||||
context.shape[0],
|
||||
self.decoder.out_channels,
|
||||
context.shape[1],
|
||||
context.shape[2] * self.decoder.patch_size,
|
||||
context.shape[3] * self.decoder.patch_size,
|
||||
)
|
||||
if task.x_t is None:
|
||||
x_t = randn_tensor(
|
||||
tile_pixel_shape,
|
||||
generator=task.noise_generator,
|
||||
device=context.device,
|
||||
dtype=context.dtype,
|
||||
)
|
||||
else:
|
||||
x_t = task.x_t
|
||||
return self.decoder.denoise(context, x_t, num_inference_steps)
|
||||
|
||||
def _distributed_tile_merge(
|
||||
self,
|
||||
tiles: dict[tuple[int, int, int], torch.Tensor],
|
||||
grid_spec: GridSpec,
|
||||
) -> torch.Tensor:
|
||||
plan = grid_spec.tile_spec["plan"]
|
||||
if not isinstance(plan, LTX2VideoDiffusionTilePlan):
|
||||
raise TypeError(f"Expected an LTX2VideoDiffusionTilePlan, got {type(plan)!r}.")
|
||||
return self._merge_tiled_decode(tiles, plan)
|
||||
|
||||
def tiled_decode(
|
||||
self,
|
||||
z: torch.Tensor,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
num_inference_steps: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
if not self.is_distributed_enabled():
|
||||
return super().tiled_decode(z, generator=generator, num_inference_steps=num_inference_steps)
|
||||
|
||||
logger.debug("LTX-2.5 diffusion decoder running with distributed stage-4/stage-5 tiles")
|
||||
num_inference_steps = num_inference_steps or self.decoder.default_num_inference_steps
|
||||
result = self.distributed_executor.execute(
|
||||
z,
|
||||
DistributedOperator(
|
||||
split=lambda tensor: self._distributed_tile_split(tensor, generator, num_inference_steps),
|
||||
exec=lambda task: self._distributed_tile_exec(task, num_inference_steps),
|
||||
merge=self._distributed_tile_merge,
|
||||
),
|
||||
broadcast_result=False,
|
||||
)
|
||||
if result.numel() == 0:
|
||||
# The base decode method crops five dimensions before the LTX
|
||||
# runtime discards non-output-rank results.
|
||||
return torch.empty((0, self.decoder.out_channels, 0, 0, 0), device=z.device, dtype=z.dtype)
|
||||
return result
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared latent layout and normalization primitives for LTX pipelines."""
|
||||
|
||||
@@ -69,6 +69,13 @@ def unpack_latents(
|
||||
return latents.permute(0, 4, 1, 5, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(2, 3)
|
||||
|
||||
|
||||
def resolve_video_latent_statistics(pipeline: Any) -> tuple[torch.Tensor, torch.Tensor, float]:
|
||||
"""Return statistics owned by the active video decoder."""
|
||||
decoder = pipeline.diffusion_decoder if getattr(pipeline, "use_diffusion_decoder", False) else pipeline.vae
|
||||
scaling_factor = getattr(getattr(decoder, "config", None), "scaling_factor", 1.0)
|
||||
return decoder.latents_mean, decoder.latents_std, scaling_factor
|
||||
|
||||
|
||||
def normalize_latents(
|
||||
latents: torch.Tensor,
|
||||
latents_mean: torch.Tensor,
|
||||
@@ -202,11 +209,12 @@ def prepare_video_latents(
|
||||
) -> torch.Tensor:
|
||||
if latents is not None:
|
||||
if latents.ndim == 5:
|
||||
latents_mean, latents_std, scaling_factor = resolve_video_latent_statistics(pipeline)
|
||||
latents = normalize_latents(
|
||||
latents,
|
||||
pipeline.vae.latents_mean,
|
||||
pipeline.vae.latents_std,
|
||||
pipeline.vae.config.scaling_factor,
|
||||
latents_mean,
|
||||
latents_std,
|
||||
scaling_factor,
|
||||
)
|
||||
latents = pack_latents(
|
||||
latents,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Construction and execution of a fixed LTX refinement-phase adapter.
|
||||
|
||||
@@ -420,6 +420,8 @@ def build_ltx_phase_adapter(pipeline: Any) -> LTXPhaseAdapterRuntime | None:
|
||||
pipeline.od_config.model,
|
||||
profile.artifact_repo_id,
|
||||
profile.distilled_lora_filename,
|
||||
model_revision=getattr(pipeline.od_config, "revision", None),
|
||||
artifact_revision=profile.artifact_revision,
|
||||
)
|
||||
|
||||
if layer_fused and pipeline.od_config.dtype is not torch.bfloat16:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Request normalization and validation shared by LTX pipeline variants."""
|
||||
|
||||
@@ -86,6 +86,7 @@ def validate_pipeline_request(
|
||||
vae_spatial_compression_ratio: int,
|
||||
vae_temporal_compression_ratio: int,
|
||||
pipeline_name: str,
|
||||
min_video_latent_spatial_size: tuple[int, int] | None = None,
|
||||
request_sigmas: list[float] | None = None,
|
||||
request_phase_sigmas: tuple[list[float] | None, ...] | None = None,
|
||||
) -> None:
|
||||
@@ -97,6 +98,18 @@ def validate_pipeline_request(
|
||||
f"{request_inputs.width}x{request_inputs.height}."
|
||||
)
|
||||
|
||||
if min_video_latent_spatial_size is not None:
|
||||
min_latent_height, min_latent_width = min_video_latent_spatial_size
|
||||
latent_height = request_inputs.height // vae_spatial_compression_ratio
|
||||
latent_width = request_inputs.width // vae_spatial_compression_ratio
|
||||
if latent_height < min_latent_height or latent_width < min_latent_width:
|
||||
raise ValueError(
|
||||
f"{pipeline_name} DiffVAE requires video latent spatial dimensions of at least "
|
||||
f"{min_latent_width}x{min_latent_height} for neighborhood attention, got "
|
||||
f"{latent_width}x{latent_height} from request resolution "
|
||||
f"{request_inputs.width}x{request_inputs.height}."
|
||||
)
|
||||
|
||||
if request_inputs.num_frames < 1 or (request_inputs.num_frames - 1) % vae_temporal_compression_ratio != 0:
|
||||
raise ValueError(
|
||||
f"{pipeline_name} `num_frames` must be {vae_temporal_compression_ratio} * k + 1, "
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared recipe-driven runtime for LTX pipeline variants."""
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm_omni.platforms import current_omni_platform
|
||||
from . import ltx2_latents as latent_ops
|
||||
from .ltx2_components import (
|
||||
LTXComponentProfile,
|
||||
_ltx2_use_diffusion_decoder,
|
||||
detect_ltx_model_version,
|
||||
initialize_pipeline_components,
|
||||
resolve_ltx_component_profile,
|
||||
@@ -168,6 +169,7 @@ class LTXRuntime(
|
||||
"Use the default ulysses_mode='strict' for LTX sequence parallelism."
|
||||
)
|
||||
self.model_version = detect_ltx_model_version(od_config.model, revision=getattr(od_config, "revision", None))
|
||||
self.use_diffusion_decoder = _ltx2_use_diffusion_decoder(od_config, self.model_version)
|
||||
self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version)
|
||||
self.pipeline_recipe = resolve_ltx_pipeline_recipe(self.pipeline_kind, self.model_version)
|
||||
if getattr(od_config, "cache_backend", "none") == "cache_dit" and not self.pipeline_recipe.supports_cache_dit:
|
||||
@@ -178,6 +180,8 @@ class LTXRuntime(
|
||||
self._dit_modules = list(self.component_profile.dit_modules)
|
||||
self._encoder_modules = list(self.component_profile.encoder_modules)
|
||||
self._vae_modules = list(self.component_profile.vae_modules)
|
||||
if self.use_diffusion_decoder:
|
||||
self._vae_modules.append("diffusion_decoder")
|
||||
self._resident_modules = list(self.component_profile.resident_modules)
|
||||
if self.model_version in ("2.3", "2.5"):
|
||||
self.preserve_sp_padded_audio_duration = True
|
||||
@@ -249,12 +253,17 @@ class LTXRuntime(
|
||||
raise ValueError(f"{self.__class__.__name__} does not support `image` input.")
|
||||
request_sigmas = self._resolve_request_sigmas(req, sigmas)
|
||||
request_phase_sigmas = self._resolve_request_phase_sigmas(req, stage_1_sigmas, stage_2_sigmas)
|
||||
min_video_latent_spatial_size = None
|
||||
if getattr(self, "use_diffusion_decoder", False):
|
||||
first_stage_kernel = self.diffusion_decoder.config.decoder_stage_kernels[0]
|
||||
min_video_latent_spatial_size = (int(first_stage_kernel[-2]), int(first_stage_kernel[-1]))
|
||||
validate_pipeline_request(
|
||||
request_inputs,
|
||||
pipeline_recipe=self.pipeline_recipe,
|
||||
vae_spatial_compression_ratio=self.vae_spatial_compression_ratio,
|
||||
vae_temporal_compression_ratio=self.vae_temporal_compression_ratio,
|
||||
pipeline_name=self.__class__.__name__,
|
||||
min_video_latent_spatial_size=min_video_latent_spatial_size,
|
||||
request_sigmas=request_sigmas,
|
||||
request_phase_sigmas=request_phase_sigmas,
|
||||
)
|
||||
@@ -542,7 +551,10 @@ class LTXRuntime(
|
||||
return self._make_output((latents, audio_latents))
|
||||
|
||||
latents = latents.to(connector_prompt_embeds.dtype)
|
||||
if not self.vae.config.timestep_conditioning:
|
||||
use_diffusion_decoder = getattr(self, "use_diffusion_decoder", False)
|
||||
if use_diffusion_decoder:
|
||||
timestep_decode = None
|
||||
elif not self.vae.config.timestep_conditioning:
|
||||
timestep_decode = None
|
||||
else:
|
||||
noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype)
|
||||
@@ -559,14 +571,22 @@ class LTXRuntime(
|
||||
dist_initialized = torch.distributed.is_initialized()
|
||||
is_output_rank = not dist_initialized or torch.distributed.get_rank() == 0
|
||||
vae_decode_needs_all_ranks = False
|
||||
is_distributed_vae_enabled = getattr(self.vae, "is_distributed_enabled", None)
|
||||
video_decoder = self.diffusion_decoder if use_diffusion_decoder else self.vae
|
||||
is_distributed_vae_enabled = getattr(video_decoder, "is_distributed_enabled", None)
|
||||
if self.distributed_video_decode and dist_initialized and callable(is_distributed_vae_enabled):
|
||||
# Distributed tiled decode is collective, so every rank must enter it.
|
||||
vae_decode_needs_all_ranks = bool(is_distributed_vae_enabled())
|
||||
|
||||
should_decode_video = not self.distributed_video_decode or is_output_rank or vae_decode_needs_all_ranks
|
||||
if should_decode_video:
|
||||
video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0]
|
||||
if use_diffusion_decoder:
|
||||
video = self.diffusion_decoder.decode(
|
||||
latents.to(self.diffusion_decoder.dtype),
|
||||
generator=generator,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
else:
|
||||
video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0]
|
||||
else:
|
||||
video = torch.empty(0, device=latents.device, dtype=latents.dtype)
|
||||
|
||||
@@ -800,11 +820,12 @@ class LTXRuntime(
|
||||
self.transformer_spatial_patch_size,
|
||||
self.transformer_temporal_patch_size,
|
||||
)
|
||||
latents_mean, latents_std, scaling_factor = latent_ops.resolve_video_latent_statistics(self)
|
||||
latents = latent_ops.denormalize_latents(
|
||||
latents,
|
||||
self.vae.latents_mean,
|
||||
self.vae.latents_std,
|
||||
self.vae.config.scaling_factor,
|
||||
latents_mean,
|
||||
latents_std,
|
||||
scaling_factor,
|
||||
)
|
||||
|
||||
audio_latents = latent_ops.unpad_audio_latents(audio_latents, forward_ctx.original_audio_num_frames)
|
||||
|
||||
Reference in new issue
Block a user