diff --git a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md index 7edf7c61d..476cfa158 100644 --- a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md +++ b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md @@ -29,6 +29,10 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b - Tongyi-MAI/Z-Image-Turbo - Wan2.2-I2V-A14B-Diffusers +LTX-2.5 is also supported through a source-preview dependency path described +below. The adapter behavior is unit tested, but the gated full checkpoint is +not part of the regular CI matrix. + If you find that a model not listed above also produces different outputs from running diffusers model directly. Please consider file an issue. @@ -116,6 +120,69 @@ The model loading and inference strictly follows the diffusers library, and they Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview). Some particular examples are below. +#### LTX-2.5 (source preview) + +The repository linked from the LTX-2.5 release, +[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5), contains +component weights and does not have the `model_index.json` required by +`DiffusionPipeline.from_pretrained()`. Use Lightricks' official converted +repository, [`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers), +with this backend. Both repositories are gated; accept the model terms on +Hugging Face and authenticate with a read token before starting the server. + +LTX-2.5 support landed after Diffusers 0.39.0 and is not in a stable Diffusers +release yet. After installing vLLM-Omni, install the reviewed upstream commit +and Gemma 4-capable Transformers version: + +```bash +python -m pip install --upgrade 'transformers>=5.10.1' \ + 'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20' +hf auth login +``` + +Serve the default distilled text-to-video pipeline as follows. The model is +large; CPU offload and VAE tiling reduce GPU memory pressure at the cost of +latency and host memory. + +```bash +vllm serve "Lightricks/LTX-2.5-Diffusers" \ + --omni \ + --port 8091 \ + --diffusion-load-format diffusers \ + --dtype bfloat16 \ + --enable-cpu-offload \ + --vae-use-tiling \ + --default-sampling-params \ + '{"0":{"height":512,"width":768,"num_frames":121,"fps":24,"frame_rate":24.0,"output_type":"np"}}' +``` + +Then request an MP4 with synchronized audio: + +```bash +curl -X POST http://localhost:8091/v1/videos/sync \ + -F 'prompt=A red fox walks through a snowy forest at dawn while birds sing in the distance.' \ + -F 'size=768x512' \ + -F 'num_frames=121' \ + -F 'fps=24' \ + -F 'seed=42' \ + -o ltx-2.5.mp4 +``` + +The converted repository selects the distilled transformer by default. When a +request supplies neither `sigmas`, `timesteps`, nor `num_inference_steps`, the +adapter uses the official eight-sigma distilled schedule and disables CFG, +spatio-temporal guidance, and modality guidance. An explicit schedule or step +count takes precedence; a step count without `sigmas` uses Diffusers' generic +schedule and may reduce quality. The adapter preserves both `frames` and the +pipeline's singular `audio` output and reads the audio sample rate from the +loaded vocoder (48 kHz for this checkpoint). + +This path currently covers the default one-stage distilled T2V pipeline. +Loading `transformer_full`, two-stage upsampling, image/video conditioning, and +the diffusion video decoder require additional pipeline assembly and are not +enabled by this example. Native vLLM-Omni acceleration features listed under +[Limitations](#limitations) remain unavailable with the Diffusers backend. + #### Wan Series The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference. diff --git a/examples/online_serving/diffusers_pipeline_adapter/README.md b/examples/online_serving/diffusers_pipeline_adapter/README.md index 37bf48aa0..98305667c 100644 --- a/examples/online_serving/diffusers_pipeline_adapter/README.md +++ b/examples/online_serving/diffusers_pipeline_adapter/README.md @@ -26,6 +26,10 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b - Tongyi-MAI/Z-Image-Turbo - Wan2.2-I2V-A14B-Diffusers +LTX-2.5 is also supported through a source-preview dependency path described +below. The adapter behavior is unit tested, but the gated full checkpoint is +not part of the regular CI matrix. + If you find that a model not listed above also produces different outputs from running diffusers model directly. Please consider file an issue or submit a PR to fix. @@ -118,6 +122,69 @@ The model loading and inference strictly follows the diffusers library, and they Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview). Some particular examples are below. +#### LTX-2.5 (source preview) + +The repository linked from the LTX-2.5 release, +[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5), contains +component weights and does not have the `model_index.json` required by +`DiffusionPipeline.from_pretrained()`. Use Lightricks' official converted +repository, [`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers), +with this backend. Both repositories are gated; accept the model terms on +Hugging Face and authenticate with a read token before starting the server. + +LTX-2.5 support landed after Diffusers 0.39.0 and is not in a stable Diffusers +release yet. After installing vLLM-Omni, install the reviewed upstream commit +and Gemma 4-capable Transformers version: + +```bash +python -m pip install --upgrade 'transformers>=5.10.1' \ + 'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20' +hf auth login +``` + +Serve the default distilled text-to-video pipeline as follows. The model is +large; CPU offload and VAE tiling reduce GPU memory pressure at the cost of +latency and host memory. + +```bash +vllm serve "Lightricks/LTX-2.5-Diffusers" \ + --omni \ + --port 8091 \ + --diffusion-load-format diffusers \ + --dtype bfloat16 \ + --enable-cpu-offload \ + --vae-use-tiling \ + --default-sampling-params \ + '{"0":{"height":512,"width":768,"num_frames":121,"fps":24,"frame_rate":24.0,"output_type":"np"}}' +``` + +Then request an MP4 with synchronized audio: + +```bash +curl -X POST http://localhost:8091/v1/videos/sync \ + -F 'prompt=A red fox walks through a snowy forest at dawn while birds sing in the distance.' \ + -F 'size=768x512' \ + -F 'num_frames=121' \ + -F 'fps=24' \ + -F 'seed=42' \ + -o ltx-2.5.mp4 +``` + +The converted repository selects the distilled transformer by default. When a +request supplies neither `sigmas`, `timesteps`, nor `num_inference_steps`, the +adapter uses the official eight-sigma distilled schedule and disables CFG, +spatio-temporal guidance, and modality guidance. An explicit schedule or step +count takes precedence; a step count without `sigmas` uses Diffusers' generic +schedule and may reduce quality. The adapter preserves both `frames` and the +pipeline's singular `audio` output and reads the audio sample rate from the +loaded vocoder (48 kHz for this checkpoint). + +This path currently covers the default one-stage distilled T2V pipeline. +Loading `transformer_full`, two-stage upsampling, image/video conditioning, and +the diffusion video decoder require additional pipeline assembly and are not +enabled by this example. Native vLLM-Omni acceleration features listed under +[Limitations](#limitations) remain unavailable with the Diffusers backend. + #### Wan Series The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference. diff --git a/tests/diffusion/diffusion_backend/test_diffusers_backend.py b/tests/diffusion/diffusion_backend/test_diffusers_backend.py index babc23b4a..13df38ca7 100644 --- a/tests/diffusion/diffusion_backend/test_diffusers_backend.py +++ b/tests/diffusion/diffusion_backend/test_diffusers_backend.py @@ -17,6 +17,7 @@ from vllm_omni.diffusion.data import ( OmniDiffusionConfig, ) from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline +from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import LTX25_DISTILLED_SIGMAS from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -180,6 +181,32 @@ class TestPipelineArgumentsHandling: with pytest.raises(NotImplementedError): DiffusersAdapterPipeline(od_config=od_config) + def test_ltx25_original_repo_rejected_before_loading(self, mocker): + mock_from_pretrained = mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained" + ) + adapter = DiffusersAdapterPipeline(od_config=_make_od_config(model="Lightricks/LTX-2.5")) + + with pytest.raises(ValueError, match="Lightricks/LTX-2.5-Diffusers"): + adapter.load_weights() + + mock_from_pretrained.assert_not_called() + + def test_ltx25_requires_upstream_diffusers_feature(self, mocker): + mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter._supports_ltx25_diffusers", + return_value=False, + ) + mock_from_pretrained = mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained" + ) + adapter = DiffusersAdapterPipeline(od_config=_make_od_config(model="Lightricks/LTX-2.5-Diffusers")) + + with pytest.raises(ImportError, match="7564fb016dabda0c943416190fc92398c50b1b20"): + adapter.load_weights() + + mock_from_pretrained.assert_not_called() + def test_adapter_load_weights_injects_supported_quantization_config(self, mocker): ( FakePipelineQuantizationConfig, @@ -374,6 +401,22 @@ class TestPipelineArgumentsHandling: assert isinstance(output, DiffusionOutput) assert output.output == raw_output + def test_adapter_preserves_video_and_singular_audio_output(self): + frames = torch.zeros(1, 2, 3, 4, 3) + audio = torch.zeros(1, 48000) + adapter = DiffusersAdapterPipeline(od_config=_make_od_config()) + adapter._pipeline = SimpleNamespace(vocoder=SimpleNamespace(config=SimpleNamespace(output_sampling_rate=48000))) + + output = adapter._wrap_output(SimpleNamespace(frames=frames, audio=audio)) + + assert isinstance(output.output, dict) + assert output.output["payload"]["video"] is frames + assert output.output["payload"]["audio"] is audio + assert output.output["metadata"] == {"audio": {"sample_rate": 48000}} + + frames_only = adapter._wrap_output(SimpleNamespace(frames=frames, audio=None)) + assert frames_only.output is frames + def test_adapter_build_call_kwargs(self, mocker): class MockPipeline: def __call__( @@ -385,6 +428,7 @@ class TestPipelineArgumentsHandling: height=None, width=None, num_frames=None, + stg_scale=None, num_images_per_prompt=None, num_videos_per_prompt=None, output_type=None, @@ -406,6 +450,7 @@ class TestPipelineArgumentsHandling: diffusers_call_kwargs={ "guidance_scale": 1.25, "eta": 0.3, + "stg_scale": 0.25, "output_type": "np", } ) @@ -426,6 +471,12 @@ class TestPipelineArgumentsHandling: num_outputs_per_prompt=2, seed=123, output_type="pil", + extra_args={ + "num_inference_steps": 99, + "stg_scale": 0.75, + "unsupported_argument": True, + "extra_args": {"must_not": "leak"}, + }, ), ) @@ -438,6 +489,9 @@ class TestPipelineArgumentsHandling: assert kwargs["height"] == 320 assert kwargs["width"] == 640 assert kwargs["num_frames"] == 8 + assert kwargs["stg_scale"] == 0.75 + assert "unsupported_argument" not in kwargs + assert "extra_args" not in kwargs assert kwargs["num_images_per_prompt"] == 2 assert kwargs["num_videos_per_prompt"] == 2 assert kwargs["output_type"] == "pil" @@ -629,6 +683,97 @@ class TestPipelineArgumentsHandling: with pytest.raises(ValueError): pipeline.forward(DiffusionRequestBatch(requests=[problematic_request])) + def test_ltx25_uses_distilled_sampling_defaults_from_renamed_local_directory(self, mocker, tmp_path): + class LTX2Pipeline: + def __call__( + self, + prompt=None, + num_inference_steps=None, + sigmas=None, + timesteps=None, + guidance_scale=None, + audio_guidance_scale=None, + stg_scale=None, + audio_stg_scale=None, + modality_scale=None, + audio_modality_scale=None, + guidance_rescale=None, + audio_guidance_rescale=None, + num_frames=None, + frame_rate=None, + output_type=None, + generator=None, + ): + return None + + def to(self, device): + return self + + # Local model directories do not retain their Hub repository name. + # Detect 2.5 from its pipeline component metadata instead. + (tmp_path / "model_index.json").write_text( + '{"_class_name":"LTX2Pipeline","duration_head":["diffusers","LTX2DurationHead"]}', + encoding="utf-8", + ) + enable_tiling = mocker.Mock() + loaded_pipeline = LTX2Pipeline() + loaded_pipeline.vae = SimpleNamespace(enable_tiling=enable_tiling) + + mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter._supports_ltx25_diffusers", + return_value=True, + ) + mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained", + return_value=loaded_pipeline, + ) + adapter = DiffusersAdapterPipeline( + od_config=_make_od_config( + model=str(tmp_path), + vae_use_tiling=True, + ) + ) + adapter.load_weights() + enable_tiling.assert_called_once_with() + + sampling = OmniDiffusionSamplingParams( + num_frames=121, + frame_rate=24.0, + seed=42, + generator_device="cpu", + ) + kwargs = adapter._build_call_kwargs(DiffusionRequestBatch(requests=[_make_request(sampling_params=sampling)])) + + assert kwargs["sigmas"] == LTX25_DISTILLED_SIGMAS + assert kwargs["num_inference_steps"] == 8 + assert kwargs["guidance_scale"] == 1.0 + assert kwargs["audio_guidance_scale"] == 1.0 + assert kwargs["stg_scale"] == 0.0 + assert kwargs["audio_stg_scale"] == 0.0 + assert kwargs["modality_scale"] == 1.0 + assert kwargs["audio_modality_scale"] == 1.0 + assert kwargs["guidance_rescale"] == 0.0 + assert kwargs["audio_guidance_rescale"] == 0.0 + + explicit_steps = OmniDiffusionSamplingParams(num_inference_steps=12, generator_device="cpu") + step_kwargs = adapter._build_call_kwargs( + DiffusionRequestBatch(requests=[_make_request(sampling_params=explicit_steps)]) + ) + assert step_kwargs["num_inference_steps"] == 12 + assert "sigmas" not in step_kwargs + + custom_sigmas = [1.0, 0.5] + explicit_schedule = OmniDiffusionSamplingParams( + num_inference_steps=2, + sigmas=custom_sigmas, + generator_device="cpu", + ) + schedule_kwargs = adapter._build_call_kwargs( + DiffusionRequestBatch(requests=[_make_request(sampling_params=explicit_schedule)]) + ) + assert schedule_kwargs["num_inference_steps"] == 2 + assert schedule_kwargs["sigmas"] is custom_sigmas + @pytest.mark.advanced_model @hardware_test(res={"cuda": "L4"}, num_cards=1) diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py index a4a9bc0dd..e62c34d8f 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py @@ -23,7 +23,14 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline from torch import nn from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import BasePipelineUtils, get_pipeline_utils +from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import ( + LTX25_DIFFUSERS_COMMIT, + LTX25_DIFFUSERS_MODEL_ID, + LTX25_ORIGINAL_MODEL_ID, + BasePipelineUtils, + get_pipeline_utils, + is_ltx25_diffusers_model, +) from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import ( apply_diffusers_quantization_config, convert_diffusers_quantization_config, @@ -67,6 +74,23 @@ _DIFFUSERS_CONFIG_LOAD_KWARGS = { } +def _supports_ltx25_diffusers() -> bool: + """Check the standard-pipeline capabilities added with LTX-2.5.""" + try: + import diffusers + import transformers + + pipeline_class = diffusers.LTX2Pipeline + init_parameters = inspect.signature(pipeline_class.__init__).parameters + return ( + "duration_head" in init_parameters + and "prompt_enhancer" in init_parameters + and hasattr(transformers, "Gemma4UnifiedForConditionalGeneration") + ) + except (AttributeError, ImportError, RuntimeError): + return False + + class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): """Black-box adapter that delegates full pipeline execution to a diffusers pipeline. @@ -110,6 +134,7 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): model_id = self.od_config.model dtype = self.od_config.dtype + self._validate_model_compatibility() load_kwargs = { "torch_dtype": dtype, @@ -130,6 +155,11 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}") self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs) + # Gated repositories may be loadable with an explicit token in + # diffusers_load_kwargs even when config enrichment could not resolve + # their pipeline class. In that case the loaded class is authoritative. + if pipeline_class_name is None: + self._pipeline_utils = get_pipeline_utils(self._pipeline.__class__.__name__) self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config) self._pipeline.to(self.device) @@ -146,14 +176,14 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): # VAE slicing and tiling: try-catch because not all models have VAE if self.od_config.vae_use_slicing: try: - self._pipeline.enable_vae_slicing() + self._enable_vae_optimization("slicing") except Exception as e: logger.warning( f"Failed to enable VAE slicing for diffusers pipeline {self._pipeline.__class__.__name__}: {e}" ) if self.od_config.vae_use_tiling: try: - self._pipeline.enable_vae_tiling() + self._enable_vae_optimization("tiling") except Exception as e: logger.warning( f"Failed to enable VAE tiling for diffusers pipeline {self._pipeline.__class__.__name__}: {e}" @@ -162,6 +192,18 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): # Attention backend self._set_attention_backend() + def _enable_vae_optimization(self, optimization: str) -> None: + """Enable a VAE optimization through the pipeline or VAE component API.""" + pipeline_method = getattr(self._pipeline, f"enable_vae_{optimization}", None) + if callable(pipeline_method): + pipeline_method() + return + + vae_method = getattr(getattr(self._pipeline, "vae", None), f"enable_{optimization}", None) + if not callable(vae_method): + raise AttributeError(f"Neither the pipeline nor its VAE supports {optimization}") + vae_method() + # ------------------------------------------------------------------ # Step-wise execution — explicitly rejected # ------------------------------------------------------------------ @@ -247,6 +289,23 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): "quantization conversion." ) + def _validate_model_compatibility(self) -> None: + normalized_model = str(self.od_config.model).rstrip("/\\").replace("\\", "/").lower() + if normalized_model == LTX25_ORIGINAL_MODEL_ID.lower(): + raise ValueError( + f"{LTX25_ORIGINAL_MODEL_ID} is a component-weight repository without a Diffusers " + f"model_index.json. Use the official converted repository {LTX25_DIFFUSERS_MODEL_ID} " + "with --diffusion-load-format diffusers." + ) + + if is_ltx25_diffusers_model(self.od_config) and not _supports_ltx25_diffusers(): + raise ImportError( + f"{LTX25_DIFFUSERS_MODEL_ID} requires unreleased LTX-2.5 support from Diffusers and " + "Gemma 4 Unified support from Transformers. Install the source-preview dependencies with " + "`python -m pip install --upgrade 'transformers>=5.10.1' " + f"'diffusers @ git+https://github.com/huggingface/diffusers.git@{LTX25_DIFFUSERS_COMMIT}'`." + ) + def _load_diffusers_component_names( self, model_id: str, @@ -370,11 +429,22 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): ) # Request-time sampling params - for key, value in sampling.__dict__.items(): + for key, value in sampling.extra_args.items(): if value is None: continue if self._accept_call_kwargs is None or key in self._accept_call_kwargs: kwargs[key] = value + else: + logger.warning( + f"Skipping unsupported diffusers pipeline __call__ argument `{key}` from extra_args. " + f"Check out the documentation of {self._pipeline.__class__.__name__}." + ) + + for key, value in sampling.__dict__.items(): + if key == "extra_args" or value is None: + continue + if self._accept_call_kwargs is None or key in self._accept_call_kwargs: + kwargs[key] = value # Special format fields in sampling params if output_type := sampling.output_type or self.od_config.output_type: @@ -393,6 +463,8 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): else: kwargs["generator"] = torch.Generator(device=sampling.generator_device) + self._pipeline_utils.update_call_kwargs(self.od_config, kwargs) + logger.info( "Calling diffusers pipeline with kwargs: %s", DiffusersAdapterPipeline._summarize_call_kwargs_value(kwargs) ) @@ -502,6 +574,21 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): return DiffusionOutput(output=output.images) if hasattr(output, "frames"): + audio = getattr(output, "audio", None) + if audio is not None: + metadata: dict[str, Any] = {} + if (sample_rate := self._get_output_audio_sample_rate()) is not None: + metadata["audio"] = {"sample_rate": sample_rate} + envelope: dict[str, Any] = { + "payload": { + "video": output.frames, + "audio": audio, + } + } + if metadata: + envelope["metadata"] = metadata + return DiffusionOutput(output=envelope) + # Preserve diffusers video format (`output_type`) return DiffusionOutput(output=output.frames) @@ -510,6 +597,24 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): return DiffusionOutput(output=output) + def _get_output_audio_sample_rate(self) -> int | None: + vocoder = getattr(self._pipeline, "vocoder", None) + config = getattr(vocoder, "config", None) + if config is None: + return None + + if hasattr(config, "get"): + value = config.get("output_sampling_rate") + else: + value = getattr(config, "output_sampling_rate", None) + if isinstance(value, bool): + return None + try: + sample_rate = int(value) + except (TypeError, ValueError): + return None + return sample_rate if sample_rate > 0 else None + @staticmethod def _summarize_call_kwargs_value(value: Any) -> Any: """Return a sanitized summary of diffusers call kwargs for logging.""" diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py index 4c810bb68..23bd34162 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json +import logging +from pathlib import Path from typing import Any from diffusers.pipelines.pipeline_utils import DiffusionPipeline @@ -8,6 +11,57 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.inputs.data import OmniDiffusionSamplingParams +logger = logging.getLogger(__name__) + +LTX25_ORIGINAL_MODEL_ID = "Lightricks/LTX-2.5" +LTX25_DIFFUSERS_MODEL_ID = "Lightricks/LTX-2.5-Diffusers" +LTX25_DIFFUSERS_COMMIT = "7564fb016dabda0c943416190fc92398c50b1b20" + +# Diffusers' public LTX-2.5 schedule intentionally excludes the scheduler's +# terminal zero. FlowMatchEulerDiscreteScheduler appends that internally. +LTX25_DISTILLED_SIGMAS = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875] + +_LTX25_DISTILLED_GUIDANCE_DEFAULTS: dict[str, float] = { + "guidance_scale": 1.0, + "audio_guidance_scale": 1.0, + "stg_scale": 0.0, + "audio_stg_scale": 0.0, + "modality_scale": 1.0, + "audio_modality_scale": 1.0, + "guidance_rescale": 0.0, + "audio_guidance_rescale": 0.0, +} + + +def is_ltx25_diffusers_model(od_config: OmniDiffusionConfig) -> bool: + """Return whether config targets the official converted LTX-2.5 repo.""" + normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower() + if normalized_model == LTX25_DIFFUSERS_MODEL_ID.lower(): + return True + + # A downloaded checkpoint may be served from any directory name. The + # duration-head component is the pipeline-level marker introduced for + # LTX-2.5; older LTX-2.0/2.3 indexes do not contain it. + model_path = Path(str(od_config.model)).expanduser() + subfolder = od_config.diffusers_load_kwargs.get("subfolder") + if subfolder is not None: + model_path /= str(subfolder) + try: + with (model_path / "model_index.json").open(encoding="utf-8") as model_index_file: + model_index = json.load(model_index_file) + except (OSError, TypeError, ValueError): + return False + if not isinstance(model_index, dict): + return False + + duration_head = model_index.get("duration_head") + return ( + model_index.get("_class_name") == "LTX2Pipeline" + and isinstance(duration_head, list) + and bool(duration_head) + and duration_head[0] is not None + ) + class BasePipelineUtils: """No-op hooks for pipeline-specific diffusers adapter behavior.""" @@ -21,6 +75,43 @@ class BasePipelineUtils: def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None: pass + def update_call_kwargs( + self, + od_config: OmniDiffusionConfig, + call_kwargs: dict[str, Any], + ) -> None: + pass + + +class LTX2PipelineUtils(BasePipelineUtils): + """Diffusers call policy for the official default LTX-2.5 checkpoint.""" + + def update_call_kwargs( + self, + od_config: OmniDiffusionConfig, + call_kwargs: dict[str, Any], + ) -> None: + if not is_ltx25_diffusers_model(od_config): + return + + has_sigmas = call_kwargs.get("sigmas") is not None + has_timesteps = call_kwargs.get("timesteps") is not None + has_step_count = call_kwargs.get("num_inference_steps") is not None + if not has_sigmas and not has_timesteps: + if has_step_count: + logger.warning( + "LTX-2.5 distilled inference received num_inference_steps without sigmas. " + "Diffusers will use a generic schedule; omit num_inference_steps to use the " + "official distilled schedule." + ) + else: + call_kwargs["sigmas"] = list(LTX25_DISTILLED_SIGMAS) + call_kwargs["num_inference_steps"] = len(LTX25_DISTILLED_SIGMAS) + + for key, value in _LTX25_DISTILLED_GUIDANCE_DEFAULTS.items(): + if call_kwargs.get(key) is None: + call_kwargs[key] = value + class WanPipelineUtils(BasePipelineUtils): def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None: @@ -49,6 +140,7 @@ class WanPipelineUtils(BasePipelineUtils): PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = { + "LTX2Pipeline": LTX2PipelineUtils, "WanPipeline": WanPipelineUtils, "WanImageToVideoPipeline": WanPipelineUtils, "WanVACEPipeline": WanPipelineUtils,