[Diffusion] Add LTX-2.5 Diffusion VAE decoder support (#6189)
Signed-off-by: mglyn <1203789601@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
262ff14693
commit
5e4daaf197
22 files changed
+1832
-61
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared recipe-driven runtime for LTX pipeline variants."""
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm_omni.platforms import current_omni_platform
|
||||
from . import ltx2_latents as latent_ops
|
||||
from .ltx2_components import (
|
||||
LTXComponentProfile,
|
||||
_ltx2_use_diffusion_decoder,
|
||||
detect_ltx_model_version,
|
||||
initialize_pipeline_components,
|
||||
resolve_ltx_component_profile,
|
||||
@@ -168,6 +169,7 @@ class LTXRuntime(
|
||||
"Use the default ulysses_mode='strict' for LTX sequence parallelism."
|
||||
)
|
||||
self.model_version = detect_ltx_model_version(od_config.model, revision=getattr(od_config, "revision", None))
|
||||
self.use_diffusion_decoder = _ltx2_use_diffusion_decoder(od_config, self.model_version)
|
||||
self.component_profile = resolve_ltx_component_profile(self.pipeline_kind, self.model_version)
|
||||
self.pipeline_recipe = resolve_ltx_pipeline_recipe(self.pipeline_kind, self.model_version)
|
||||
if getattr(od_config, "cache_backend", "none") == "cache_dit" and not self.pipeline_recipe.supports_cache_dit:
|
||||
@@ -178,6 +180,8 @@ class LTXRuntime(
|
||||
self._dit_modules = list(self.component_profile.dit_modules)
|
||||
self._encoder_modules = list(self.component_profile.encoder_modules)
|
||||
self._vae_modules = list(self.component_profile.vae_modules)
|
||||
if self.use_diffusion_decoder:
|
||||
self._vae_modules.append("diffusion_decoder")
|
||||
self._resident_modules = list(self.component_profile.resident_modules)
|
||||
if self.model_version in ("2.3", "2.5"):
|
||||
self.preserve_sp_padded_audio_duration = True
|
||||
@@ -249,12 +253,17 @@ class LTXRuntime(
|
||||
raise ValueError(f"{self.__class__.__name__} does not support `image` input.")
|
||||
request_sigmas = self._resolve_request_sigmas(req, sigmas)
|
||||
request_phase_sigmas = self._resolve_request_phase_sigmas(req, stage_1_sigmas, stage_2_sigmas)
|
||||
min_video_latent_spatial_size = None
|
||||
if getattr(self, "use_diffusion_decoder", False):
|
||||
first_stage_kernel = self.diffusion_decoder.config.decoder_stage_kernels[0]
|
||||
min_video_latent_spatial_size = (int(first_stage_kernel[-2]), int(first_stage_kernel[-1]))
|
||||
validate_pipeline_request(
|
||||
request_inputs,
|
||||
pipeline_recipe=self.pipeline_recipe,
|
||||
vae_spatial_compression_ratio=self.vae_spatial_compression_ratio,
|
||||
vae_temporal_compression_ratio=self.vae_temporal_compression_ratio,
|
||||
pipeline_name=self.__class__.__name__,
|
||||
min_video_latent_spatial_size=min_video_latent_spatial_size,
|
||||
request_sigmas=request_sigmas,
|
||||
request_phase_sigmas=request_phase_sigmas,
|
||||
)
|
||||
@@ -542,7 +551,10 @@ class LTXRuntime(
|
||||
return self._make_output((latents, audio_latents))
|
||||
|
||||
latents = latents.to(connector_prompt_embeds.dtype)
|
||||
if not self.vae.config.timestep_conditioning:
|
||||
use_diffusion_decoder = getattr(self, "use_diffusion_decoder", False)
|
||||
if use_diffusion_decoder:
|
||||
timestep_decode = None
|
||||
elif not self.vae.config.timestep_conditioning:
|
||||
timestep_decode = None
|
||||
else:
|
||||
noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype)
|
||||
@@ -559,14 +571,22 @@ class LTXRuntime(
|
||||
dist_initialized = torch.distributed.is_initialized()
|
||||
is_output_rank = not dist_initialized or torch.distributed.get_rank() == 0
|
||||
vae_decode_needs_all_ranks = False
|
||||
is_distributed_vae_enabled = getattr(self.vae, "is_distributed_enabled", None)
|
||||
video_decoder = self.diffusion_decoder if use_diffusion_decoder else self.vae
|
||||
is_distributed_vae_enabled = getattr(video_decoder, "is_distributed_enabled", None)
|
||||
if self.distributed_video_decode and dist_initialized and callable(is_distributed_vae_enabled):
|
||||
# Distributed tiled decode is collective, so every rank must enter it.
|
||||
vae_decode_needs_all_ranks = bool(is_distributed_vae_enabled())
|
||||
|
||||
should_decode_video = not self.distributed_video_decode or is_output_rank or vae_decode_needs_all_ranks
|
||||
if should_decode_video:
|
||||
video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0]
|
||||
if use_diffusion_decoder:
|
||||
video = self.diffusion_decoder.decode(
|
||||
latents.to(self.diffusion_decoder.dtype),
|
||||
generator=generator,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
else:
|
||||
video = self.vae.decode(latents.to(self.vae.dtype), timestep_decode, return_dict=False)[0]
|
||||
else:
|
||||
video = torch.empty(0, device=latents.device, dtype=latents.dtype)
|
||||
|
||||
@@ -800,11 +820,12 @@ class LTXRuntime(
|
||||
self.transformer_spatial_patch_size,
|
||||
self.transformer_temporal_patch_size,
|
||||
)
|
||||
latents_mean, latents_std, scaling_factor = latent_ops.resolve_video_latent_statistics(self)
|
||||
latents = latent_ops.denormalize_latents(
|
||||
latents,
|
||||
self.vae.latents_mean,
|
||||
self.vae.latents_std,
|
||||
self.vae.config.scaling_factor,
|
||||
latents_mean,
|
||||
latents_std,
|
||||
scaling_factor,
|
||||
)
|
||||
|
||||
audio_latents = latent_ops.unpad_audio_latents(audio_latents, forward_ctx.original_audio_num_frames)
|
||||
|
||||
Reference in new issue
Block a user