[Diffusion] Add LTX-2.5 Diffusion VAE decoder support (#6189)

Signed-off-by: mglyn <1203789601@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Mu GuanLinandHongsheng Liu authored and GitHub committed 2026-08-27 23:54:40 +08:00
1 parent 262ff14693
commit 5e4daaf197
22 files changed
+1832 -61

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Shared recipe-driven runtime for LTX pipeline variants."""
@@ -29,6 +29,7 @@ from vllm_omni.platforms import current_omni_platform
from . import ltx2_latents as latent_ops
from .ltx2_components import (
LTXComponentProfile,
_ltx2_use_diffusion_decoder,
detect_ltx_model_version,
initialize_pipeline_components,
resolve_ltx_component_profile,
@@ -168,6 +169,7 @@ class LTXRuntime(
"Use the default ulysses_mode='strict' for LTX sequence parallelism."
)
self.model_version = detect_ltx_model_version(od_config.model, revision=getattr(od_config, "revision", None))
self.use_diffusion_decoder = _ltx2_use_diffusion_decoder(od_config, self.model_version)
self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version)
self.pipeline_recipe = resolve_ltx_pipeline_recipe(self.pipeline_kind, self.model_version)
if getattr(od_config, "cache_backend", "none") == "cache_dit" and not self.pipeline_recipe.supports_cache_dit:
@@ -178,6 +180,8 @@ class LTXRuntime(
self._dit_modules = list(self.component_profile.dit_modules)
self._encoder_modules = list(self.component_profile.encoder_modules)
self._vae_modules = list(self.component_profile.vae_modules)
if self.use_diffusion_decoder:
self._vae_modules.append("diffusion_decoder")
self._resident_modules = list(self.component_profile.resident_modules)
if self.model_version in ("2.3", "2.5"):
self.preserve_sp_padded_audio_duration = True
@@ -249,12 +253,17 @@ class LTXRuntime(
raise ValueError(f"{self.__class__.__name__} does not support `image` input.")
request_sigmas = self._resolve_request_sigmas(req, sigmas)
request_phase_sigmas = self._resolve_request_phase_sigmas(req, stage_1_sigmas, stage_2_sigmas)
min_video_latent_spatial_size = None
if getattr(self, "use_diffusion_decoder", False):
first_stage_kernel = self.diffusion_decoder.config.decoder_stage_kernels[0]
min_video_latent_spatial_size = (int(first_stage_kernel[-2]), int(first_stage_kernel[-1]))
validate_pipeline_request(
request_inputs,
pipeline_recipe=self.pipeline_recipe,
vae_spatial_compression_ratio=self.vae_spatial_compression_ratio,
vae_temporal_compression_ratio=self.vae_temporal_compression_ratio,
pipeline_name=self.__class__.__name__,
min_video_latent_spatial_size=min_video_latent_spatial_size,
request_sigmas=request_sigmas,
request_phase_sigmas=request_phase_sigmas,
)
@@ -542,7 +551,10 @@ class LTXRuntime(
return self._make_output((latents, audio_latents))
latents = latents.to(connector_prompt_embeds.dtype)
if not self.vae.config.timestep_conditioning:
use_diffusion_decoder = getattr(self, "use_diffusion_decoder", False)
if use_diffusion_decoder:
timestep_decode = None
elif not self.vae.config.timestep_conditioning:
timestep_decode = None
else:
noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype)
@@ -559,14 +571,22 @@ class LTXRuntime(
dist_initialized = torch.distributed.is_initialized()
is_output_rank = not dist_initialized or torch.distributed.get_rank() == 0
vae_decode_needs_all_ranks = False
is_distributed_vae_enabled = getattr(self.vae, "is_distributed_enabled", None)
video_decoder = self.diffusion_decoder if use_diffusion_decoder else self.vae
is_distributed_vae_enabled = getattr(video_decoder, "is_distributed_enabled", None)
if self.distributed_video_decode and dist_initialized and callable(is_distributed_vae_enabled):
# Distributed tiled decode is collective, so every rank must enter it.
vae_decode_needs_all_ranks = bool(is_distributed_vae_enabled())
should_decode_video = not self.distributed_video_decode or is_output_rank or vae_decode_needs_all_ranks
if should_decode_video:
video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0]
if use_diffusion_decoder:
video = self.diffusion_decoder.decode(
latents.to(self.diffusion_decoder.dtype),
generator=generator,
return_dict=False,
)[0]
else:
video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0]
else:
video = torch.empty(0, device=latents.device, dtype=latents.dtype)
@@ -800,11 +820,12 @@ class LTXRuntime(
self.transformer_spatial_patch_size,
self.transformer_temporal_patch_size,
)
latents_mean, latents_std, scaling_factor = latent_ops.resolve_video_latent_statistics(self)
latents = latent_ops.denormalize_latents(
latents,
self.vae.latents_mean,
self.vae.latents_std,
self.vae.config.scaling_factor,
latents_mean,
latents_std,
scaling_factor,
)
audio_latents = latent_ops.unpad_audio_latents(audio_latents, forward_ctx.original_audio_num_frames)