[Diffusion] Add LTX-2.5 Diffusion VAE decoder support (#6189)
Signed-off-by: mglyn <1203789601@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
262ff14693
commit
5e4daaf197
22 files changed
+1832
-61
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared component construction helpers for the LTX model family."""
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from diffusers import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoVaeNeighborhoodNattenProcessor
|
||||
from diffusers.pipelines.ltx2 import LTX2TextConnectors
|
||||
from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
|
||||
from diffusers.pipelines.ltx2.vocoder import LTX2Vocoder
|
||||
@@ -35,6 +36,12 @@ from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
from .ltx2_diffusion_decoder import (
|
||||
LTX25_NATIVE_ARTIFACT_REVISION,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_FILENAME,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_REPO_ID,
|
||||
)
|
||||
from .ltx2_diffusion_decoder_distributed import DistributedLTX2VideoDiffusionDecoderModel
|
||||
from .ltx2_request import LTXCheckpointKind, validate_ltx_checkpoint
|
||||
from .ltx2_transformer import (
|
||||
LTX2VideoTransformer3DModel,
|
||||
@@ -69,6 +76,18 @@ _LTX_COMPONENT_SUBFOLDERS = (
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LTX2_CONV_VAE_EXTRA = "ltx2_use_conv_vae"
|
||||
_LTX2_DIFFUSION_DECODER_SUBFOLDER = "diffusion_decoder"
|
||||
|
||||
|
||||
def _ltx2_use_diffusion_decoder(od_config: Any, model_version: str) -> bool:
|
||||
"""Select DiffVAE by default for LTX-2.5, with an explicit ConvVAE opt-in."""
|
||||
extras = getattr(od_config, "extras", {}) or {}
|
||||
use_conv_vae = extras.get(_LTX2_CONV_VAE_EXTRA, False)
|
||||
if not isinstance(use_conv_vae, bool):
|
||||
raise TypeError(f"{_LTX2_CONV_VAE_EXTRA} must be a bool, got {type(use_conv_vae)!r}")
|
||||
return model_version == "2.5" and not use_conv_vae
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LTXComponentProfile:
|
||||
@@ -84,6 +103,7 @@ class LTXComponentProfile:
|
||||
text_encoder_cls: type | None = Gemma3ForConditionalGeneration
|
||||
vocoder_fallback_cls: type | None = None
|
||||
artifact_repo_id: str | None = None
|
||||
artifact_revision: str | None = None
|
||||
latent_upsampler_filename: str | None = None
|
||||
distilled_lora_filename: str | None = None
|
||||
transformer_subfolder: str = "transformer"
|
||||
@@ -202,6 +222,7 @@ LTX25_TWO_STAGE_COMPONENT_PROFILE = replace(
|
||||
name="ltx2_5_two_stage",
|
||||
resident_modules=(*LTX25_FULL_COMPONENT_PROFILE.resident_modules, "latent_upsampler"),
|
||||
artifact_repo_id="Lightricks/LTX-2.5",
|
||||
artifact_revision=LTX25_NATIVE_ARTIFACT_REVISION,
|
||||
latent_upsampler_filename=("latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors"),
|
||||
distilled_lora_filename="loras/ltx-2.5-22b-distilled-lora-450-bf16.safetensors",
|
||||
)
|
||||
@@ -239,14 +260,30 @@ def resolve_ltx_artifact(
|
||||
model: str,
|
||||
repo_id: str,
|
||||
filename: str,
|
||||
*,
|
||||
model_revision: str | None,
|
||||
artifact_revision: str | None,
|
||||
) -> str:
|
||||
"""Resolve an official LTX sidecar from the model root or its Hub repository."""
|
||||
"""Resolve an official LTX sidecar without crossing repository revisions.
|
||||
|
||||
A local model path selects where the primary components are loaded from;
|
||||
it does not imply that independently hosted sidecars must be offline. Hub
|
||||
offline behavior remains controlled by huggingface_hub itself.
|
||||
"""
|
||||
candidate = Path(model) / filename
|
||||
if candidate.is_file():
|
||||
return str(candidate)
|
||||
|
||||
# Hub revisions are repository-scoped. Reuse the model revision only when
|
||||
# the model and artifact are in the same repository; otherwise use the
|
||||
# independently pinned artifact revision.
|
||||
revision = model_revision if model == repo_id else artifact_revision
|
||||
try:
|
||||
return hf_hub_download(repo_id=repo_id, filename=filename)
|
||||
return hf_hub_download(
|
||||
repo_id=repo_id,
|
||||
filename=filename,
|
||||
revision=revision,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise FileNotFoundError(
|
||||
f"Unable to resolve LTX artifact {filename!r}. Searched {candidate}; "
|
||||
@@ -254,6 +291,18 @@ def resolve_ltx_artifact(
|
||||
) from exc
|
||||
|
||||
|
||||
def _create_ltx25_natten_processor() -> LTX2VideoVaeNeighborhoodNattenProcessor:
|
||||
"""Create the required production attention backend with an actionable error."""
|
||||
try:
|
||||
return LTX2VideoVaeNeighborhoodNattenProcessor()
|
||||
except (ImportError, OSError, RuntimeError, ValueError) as exc:
|
||||
raise RuntimeError(
|
||||
"LTX-2.5 DiffVAE requires the shi-labs/natten Hub kernel. "
|
||||
"Install kernels==0.14.1, use a supported GPU, leave "
|
||||
"DIFFUSERS_DISABLE_REMOTE_CODE unset, and allow Hub access during kernel initialization."
|
||||
) from exc
|
||||
|
||||
|
||||
def _load_ltx_latent_upsampler_single_file(path: str, dtype: torch.dtype) -> LTX2LatentUpsamplerModel:
|
||||
"""Load an official single-file upsampler into the Diffusers module."""
|
||||
with safe_open(path, framework="pt", device="cpu") as handle:
|
||||
@@ -519,18 +568,49 @@ def _load_component(
|
||||
local_files_only: bool,
|
||||
dtype: torch.dtype,
|
||||
revision: str | None,
|
||||
prefetch_list: tuple[str, ...] = _LTX_COMPONENT_SUBFOLDERS,
|
||||
) -> Any:
|
||||
return from_pretrained_with_prefetch(
|
||||
component_cls.from_pretrained,
|
||||
model,
|
||||
subfolder=subfolder,
|
||||
prefetch_list=_LTX_COMPONENT_SUBFOLDERS,
|
||||
prefetch_list=prefetch_list,
|
||||
local_files_only=local_files_only,
|
||||
revision=revision,
|
||||
torch_dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
def _load_ltx25_native_diffusion_decoder(
|
||||
model: str,
|
||||
*,
|
||||
local_files_only: bool,
|
||||
dtype: torch.dtype,
|
||||
revision: str | None,
|
||||
) -> DistributedLTX2VideoDiffusionDecoderModel:
|
||||
"""Load canonical Native weights into the local Diffusers-compatible class."""
|
||||
config = DistributedLTX2VideoDiffusionDecoderModel.load_config(
|
||||
model,
|
||||
subfolder=_LTX2_DIFFUSION_DECODER_SUBFOLDER,
|
||||
local_files_only=local_files_only,
|
||||
revision=revision,
|
||||
)
|
||||
checkpoint_path = resolve_ltx_artifact(
|
||||
model,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_REPO_ID,
|
||||
LTX25_NATIVE_DIFFUSION_DECODER_FILENAME,
|
||||
model_revision=revision,
|
||||
artifact_revision=LTX25_NATIVE_ARTIFACT_REVISION,
|
||||
)
|
||||
decoder = DistributedLTX2VideoDiffusionDecoderModel.from_ltx25_native_checkpoint(
|
||||
checkpoint_path,
|
||||
config,
|
||||
dtype,
|
||||
)
|
||||
decoder.init_distributed()
|
||||
return decoder
|
||||
|
||||
|
||||
def _place_aux_components(pipeline: Any) -> None:
|
||||
parallel_config = getattr(pipeline.od_config, "parallel_config", None)
|
||||
use_managed_placement = bool(
|
||||
@@ -555,6 +635,7 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
model = od_config.model
|
||||
revision = getattr(od_config, "revision", None)
|
||||
local_files_only = os.path.exists(model)
|
||||
use_diffusion_decoder = pipeline.use_diffusion_decoder
|
||||
|
||||
pipeline.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
@@ -604,6 +685,27 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
dtype=dtype,
|
||||
revision=revision,
|
||||
)
|
||||
if use_diffusion_decoder:
|
||||
pipeline.diffusion_decoder = _load_ltx25_native_diffusion_decoder(
|
||||
model,
|
||||
local_files_only=local_files_only,
|
||||
dtype=dtype,
|
||||
revision=revision,
|
||||
)
|
||||
# The canonical decoder is trained and validated with NATTEN. Its
|
||||
# portable FlexAttention fallback materializes an impractically large
|
||||
# block mask at production video sizes, so fail early if the matching
|
||||
# Hub kernel cannot be loaded instead of failing later during decode.
|
||||
pipeline.diffusion_decoder.set_attn_processor(_create_ltx25_natten_processor())
|
||||
vae_patch_parallel_size = int(
|
||||
getattr(getattr(od_config, "parallel_config", None), "vae_patch_parallel_size", 1)
|
||||
)
|
||||
if vae_patch_parallel_size > 1 or getattr(od_config, "vae_use_tiling", False):
|
||||
pipeline.diffusion_decoder.enable_tiling()
|
||||
pipeline.diffusion_decoder.set_parallel_size(
|
||||
vae_patch_parallel_size,
|
||||
mode=getattr(getattr(od_config, "parallel_config", None), "vae_parallel_mode", "tile"),
|
||||
)
|
||||
pipeline.audio_vae = _load_component(
|
||||
AutoencoderKLLTX2Audio,
|
||||
model,
|
||||
@@ -656,6 +758,8 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
model,
|
||||
profile.artifact_repo_id,
|
||||
profile.latent_upsampler_filename,
|
||||
model_revision=revision,
|
||||
artifact_revision=profile.artifact_revision,
|
||||
)
|
||||
pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype)
|
||||
else:
|
||||
@@ -665,6 +769,8 @@ def initialize_pipeline_components(pipeline: Any, od_config: Any) -> None:
|
||||
model,
|
||||
profile.artifact_repo_id,
|
||||
profile.latent_upsampler_filename,
|
||||
model_revision=revision,
|
||||
artifact_revision=profile.artifact_revision,
|
||||
)
|
||||
pipeline.latent_upsampler = _load_ltx_latent_upsampler_single_file(upsampler_path, dtype)
|
||||
|
||||
|
||||
Reference in new issue
Block a user