From 30c72afdda7d1cc4c3a40c699f3cd795e06b6f22 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Wed, 12 Aug 2026 11:58:58 +0800 Subject: [PATCH] [Model] Address LTX-2.5 review feedback Signed-off-by: Hongsheng Liu --- .../diffusers_pipeline_adapter.md | 12 ----- .../diffusers_pipeline_adapter/README.md | 12 ----- .../test_diffusers_backend.py | 4 +- .../pipeline_diffusers_adapter.py | 48 +++---------------- .../diffusers_adapter/pipeline_utils.py | 37 ++++++++++++++ 5 files changed, 45 insertions(+), 68 deletions(-) diff --git a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md index 476cfa158..890d831c7 100644 --- a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md +++ b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md @@ -156,18 +156,6 @@ vllm serve "Lightricks/LTX-2.5-Diffusers" \ '{"0":{"height":512,"width":768,"num_frames":121,"fps":24,"frame_rate":24.0,"output_type":"np"}}' ``` -Then request an MP4 with synchronized audio: - -```bash -curl -X POST http://localhost:8091/v1/videos/sync \ - -F 'prompt=A red fox walks through a snowy forest at dawn while birds sing in the distance.' \ - -F 'size=768x512' \ - -F 'num_frames=121' \ - -F 'fps=24' \ - -F 'seed=42' \ - -o ltx-2.5.mp4 -``` - The converted repository selects the distilled transformer by default. When a request supplies neither `sigmas`, `timesteps`, nor `num_inference_steps`, the adapter uses the official eight-sigma distilled schedule and disables CFG, diff --git a/examples/online_serving/diffusers_pipeline_adapter/README.md b/examples/online_serving/diffusers_pipeline_adapter/README.md index 98305667c..af13a96c9 100644 --- a/examples/online_serving/diffusers_pipeline_adapter/README.md +++ b/examples/online_serving/diffusers_pipeline_adapter/README.md @@ -158,18 +158,6 @@ vllm serve "Lightricks/LTX-2.5-Diffusers" \ '{"0":{"height":512,"width":768,"num_frames":121,"fps":24,"frame_rate":24.0,"output_type":"np"}}' ``` -Then request an MP4 with synchronized audio: - -```bash -curl -X POST http://localhost:8091/v1/videos/sync \ - -F 'prompt=A red fox walks through a snowy forest at dawn while birds sing in the distance.' \ - -F 'size=768x512' \ - -F 'num_frames=121' \ - -F 'fps=24' \ - -F 'seed=42' \ - -o ltx-2.5.mp4 -``` - The converted repository selects the distilled transformer by default. When a request supplies neither `sigmas`, `timesteps`, nor `num_inference_steps`, the adapter uses the official eight-sigma distilled schedule and disables CFG, diff --git a/tests/diffusion/diffusion_backend/test_diffusers_backend.py b/tests/diffusion/diffusion_backend/test_diffusers_backend.py index 13df38ca7..23d1630b9 100644 --- a/tests/diffusion/diffusion_backend/test_diffusers_backend.py +++ b/tests/diffusion/diffusion_backend/test_diffusers_backend.py @@ -194,7 +194,7 @@ class TestPipelineArgumentsHandling: def test_ltx25_requires_upstream_diffusers_feature(self, mocker): mocker.patch( - "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter._supports_ltx25_diffusers", + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils._supports_ltx25_diffusers", return_value=False, ) mock_from_pretrained = mocker.patch( @@ -720,7 +720,7 @@ class TestPipelineArgumentsHandling: loaded_pipeline.vae = SimpleNamespace(enable_tiling=enable_tiling) mocker.patch( - "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter._supports_ltx25_diffusers", + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils._supports_ltx25_diffusers", return_value=True, ) mocker.patch( diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py index e62c34d8f..78e71bafd 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py @@ -24,12 +24,9 @@ from torch import nn from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import ( - LTX25_DIFFUSERS_COMMIT, - LTX25_DIFFUSERS_MODEL_ID, - LTX25_ORIGINAL_MODEL_ID, BasePipelineUtils, get_pipeline_utils, - is_ltx25_diffusers_model, + validate_model_compatibility, ) from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import ( apply_diffusers_quantization_config, @@ -74,23 +71,6 @@ _DIFFUSERS_CONFIG_LOAD_KWARGS = { } -def _supports_ltx25_diffusers() -> bool: - """Check the standard-pipeline capabilities added with LTX-2.5.""" - try: - import diffusers - import transformers - - pipeline_class = diffusers.LTX2Pipeline - init_parameters = inspect.signature(pipeline_class.__init__).parameters - return ( - "duration_head" in init_parameters - and "prompt_enhancer" in init_parameters - and hasattr(transformers, "Gemma4UnifiedForConditionalGeneration") - ) - except (AttributeError, ImportError, RuntimeError): - return False - - class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): """Black-box adapter that delegates full pipeline execution to a diffusers pipeline. @@ -134,7 +114,11 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): model_id = self.od_config.model dtype = self.od_config.dtype - self._validate_model_compatibility() + validate_model_compatibility(self.od_config) + + pipeline_class = self.od_config.diffusers_pipeline_cls + pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None + self._pipeline_utils = get_pipeline_utils(pipeline_class_name) load_kwargs = { "torch_dtype": dtype, @@ -142,9 +126,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): } convert_diffusers_quantization_config(load_kwargs) - pipeline_class = self.od_config.diffusers_pipeline_cls - pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None - self._pipeline_utils = get_pipeline_utils(pipeline_class_name) self._pipeline_utils.update_load_kwargs(self.od_config, load_kwargs) component_names = ( self._load_diffusers_component_names(model_id, load_kwargs) @@ -289,23 +270,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): "quantization conversion." ) - def _validate_model_compatibility(self) -> None: - normalized_model = str(self.od_config.model).rstrip("/\\").replace("\\", "/").lower() - if normalized_model == LTX25_ORIGINAL_MODEL_ID.lower(): - raise ValueError( - f"{LTX25_ORIGINAL_MODEL_ID} is a component-weight repository without a Diffusers " - f"model_index.json. Use the official converted repository {LTX25_DIFFUSERS_MODEL_ID} " - "with --diffusion-load-format diffusers." - ) - - if is_ltx25_diffusers_model(self.od_config) and not _supports_ltx25_diffusers(): - raise ImportError( - f"{LTX25_DIFFUSERS_MODEL_ID} requires unreleased LTX-2.5 support from Diffusers and " - "Gemma 4 Unified support from Transformers. Install the source-preview dependencies with " - "`python -m pip install --upgrade 'transformers>=5.10.1' " - f"'diffusers @ git+https://github.com/huggingface/diffusers.git@{LTX25_DIFFUSERS_COMMIT}'`." - ) - def _load_diffusers_component_names( self, model_id: str, diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py index 23bd34162..1efbdc0ed 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import inspect import json import logging from pathlib import Path @@ -33,6 +34,23 @@ _LTX25_DISTILLED_GUIDANCE_DEFAULTS: dict[str, float] = { } +def _supports_ltx25_diffusers() -> bool: + """Check the standard-pipeline capabilities added with LTX-2.5.""" + try: + import diffusers + import transformers + + pipeline_class = diffusers.LTX2Pipeline + init_parameters = inspect.signature(pipeline_class.__init__).parameters + return ( + "duration_head" in init_parameters + and "prompt_enhancer" in init_parameters + and hasattr(transformers, "Gemma4UnifiedForConditionalGeneration") + ) + except (AttributeError, ImportError, RuntimeError): + return False + + def is_ltx25_diffusers_model(od_config: OmniDiffusionConfig) -> bool: """Return whether config targets the official converted LTX-2.5 repo.""" normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower() @@ -83,6 +101,25 @@ class BasePipelineUtils: pass +def validate_model_compatibility(od_config: OmniDiffusionConfig) -> None: + """Reject known model/dependency combinations before loading weights.""" + normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower() + if normalized_model == LTX25_ORIGINAL_MODEL_ID.lower(): + raise ValueError( + f"{LTX25_ORIGINAL_MODEL_ID} is a component-weight repository without a Diffusers " + f"model_index.json. Use the official converted repository {LTX25_DIFFUSERS_MODEL_ID} " + "with --diffusion-load-format diffusers." + ) + + if is_ltx25_diffusers_model(od_config) and not _supports_ltx25_diffusers(): + raise ImportError( + f"{LTX25_DIFFUSERS_MODEL_ID} requires unreleased LTX-2.5 support from Diffusers and " + "Gemma 4 Unified support from Transformers. Install the source-preview dependencies with " + "`python -m pip install --upgrade 'transformers>=5.10.1' " + f"'diffusers @ git+https://github.com/huggingface/diffusers.git@{LTX25_DIFFUSERS_COMMIT}'`." + ) + + class LTX2PipelineUtils(BasePipelineUtils): """Diffusers call policy for the official default LTX-2.5 checkpoint."""