From 40284854bb26cebd67dffc1c3d8de4753ec707d1 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Wed, 12 Aug 2026 12:33:36 +0800 Subject: [PATCH] [Model] Minimize LTX-2.5 adapter changes Signed-off-by: Hongsheng Liu --- .../diffusers_pipeline_adapter.md | 54 ++++---------- .../diffusers_pipeline_adapter/README.md | 54 ++++---------- .../test_diffusers_backend.py | 26 +++---- .../pipeline_diffusers_adapter.py | 74 ++----------------- .../diffusers_adapter/pipeline_utils.py | 47 ++++++++++++ 5 files changed, 97 insertions(+), 158 deletions(-) diff --git a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md index 890d831c7..fd911cd58 100644 --- a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md +++ b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md @@ -28,10 +28,7 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b - Qwen/Qwen-Image - Tongyi-MAI/Z-Image-Turbo - Wan2.2-I2V-A14B-Diffusers - -LTX-2.5 is also supported through a source-preview dependency path described -below. The adapter behavior is unit tested, but the gated full checkpoint is -not part of the regular CI matrix. +- Lightricks/LTX-2.5-Diffusers (source-preview dependencies) If you find that a model not listed above also produces different outputs from running diffusers model directly. Please consider file an issue. @@ -122,54 +119,33 @@ Some particular examples are below. #### LTX-2.5 (source preview) -The repository linked from the LTX-2.5 release, -[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5), contains -component weights and does not have the `model_index.json` required by -`DiffusionPipeline.from_pretrained()`. Use Lightricks' official converted -repository, [`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers), -with this backend. Both repositories are gated; accept the model terms on -Hugging Face and authenticate with a read token before starting the server. - -LTX-2.5 support landed after Diffusers 0.39.0 and is not in a stable Diffusers -release yet. After installing vLLM-Omni, install the reviewed upstream commit -and Gemma 4-capable Transformers version: +Use the gated Diffusers checkpoint +[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers); +[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) contains +component weights and is not directly loadable by this backend. Accept the +model terms, authenticate, and install the source-preview dependencies: ```bash +hf auth login python -m pip install --upgrade 'transformers>=5.10.1' \ 'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20' -hf auth login ``` -Serve the default distilled text-to-video pipeline as follows. The model is -large; CPU offload and VAE tiling reduce GPU memory pressure at the cost of -latency and host memory. - ```bash -vllm serve "Lightricks/LTX-2.5-Diffusers" \ +vllm serve Lightricks/LTX-2.5-Diffusers \ --omni \ - --port 8091 \ --diffusion-load-format diffusers \ --dtype bfloat16 \ --enable-cpu-offload \ - --vae-use-tiling \ - --default-sampling-params \ - '{"0":{"height":512,"width":768,"num_frames":121,"fps":24,"frame_rate":24.0,"output_type":"np"}}' + --vae-use-tiling ``` -The converted repository selects the distilled transformer by default. When a -request supplies neither `sigmas`, `timesteps`, nor `num_inference_steps`, the -adapter uses the official eight-sigma distilled schedule and disables CFG, -spatio-temporal guidance, and modality guidance. An explicit schedule or step -count takes precedence; a step count without `sigmas` uses Diffusers' generic -schedule and may reduce quality. The adapter preserves both `frames` and the -pipeline's singular `audio` output and reads the audio sample rate from the -loaded vocoder (48 kHz for this checkpoint). - -This path currently covers the default one-stage distilled T2V pipeline. -Loading `transformer_full`, two-stage upsampling, image/video conditioning, and -the diffusion video decoder require additional pipeline assembly and are not -enabled by this example. Native vLLM-Omni acceleration features listed under -[Limitations](#limitations) remain unavailable with the Diffusers backend. +The converted repository loads the distilled transformer. When no `sigmas`, +`timesteps`, or `num_inference_steps` is supplied, the adapter applies the +official eight-sigma unguided schedule and returns synchronized 48 kHz audio. +Support is limited to one-stage distilled T2V; `transformer_full`, two-stage +upsampling, image/video conditioning, and the diffusion video decoder are not +supported. The general Diffusers backend [limitations](#limitations) apply. #### Wan Series diff --git a/examples/online_serving/diffusers_pipeline_adapter/README.md b/examples/online_serving/diffusers_pipeline_adapter/README.md index af13a96c9..98af462e8 100644 --- a/examples/online_serving/diffusers_pipeline_adapter/README.md +++ b/examples/online_serving/diffusers_pipeline_adapter/README.md @@ -25,10 +25,7 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b - Qwen/Qwen-Image - Tongyi-MAI/Z-Image-Turbo - Wan2.2-I2V-A14B-Diffusers - -LTX-2.5 is also supported through a source-preview dependency path described -below. The adapter behavior is unit tested, but the gated full checkpoint is -not part of the regular CI matrix. +- Lightricks/LTX-2.5-Diffusers (source-preview dependencies) If you find that a model not listed above also produces different outputs from running diffusers model directly. Please consider file an issue or submit a PR to fix. @@ -124,54 +121,33 @@ Some particular examples are below. #### LTX-2.5 (source preview) -The repository linked from the LTX-2.5 release, -[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5), contains -component weights and does not have the `model_index.json` required by -`DiffusionPipeline.from_pretrained()`. Use Lightricks' official converted -repository, [`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers), -with this backend. Both repositories are gated; accept the model terms on -Hugging Face and authenticate with a read token before starting the server. - -LTX-2.5 support landed after Diffusers 0.39.0 and is not in a stable Diffusers -release yet. After installing vLLM-Omni, install the reviewed upstream commit -and Gemma 4-capable Transformers version: +Use the gated Diffusers checkpoint +[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers); +[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) contains +component weights and is not directly loadable by this backend. Accept the +model terms, authenticate, and install the source-preview dependencies: ```bash +hf auth login python -m pip install --upgrade 'transformers>=5.10.1' \ 'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20' -hf auth login ``` -Serve the default distilled text-to-video pipeline as follows. The model is -large; CPU offload and VAE tiling reduce GPU memory pressure at the cost of -latency and host memory. - ```bash -vllm serve "Lightricks/LTX-2.5-Diffusers" \ +vllm serve Lightricks/LTX-2.5-Diffusers \ --omni \ - --port 8091 \ --diffusion-load-format diffusers \ --dtype bfloat16 \ --enable-cpu-offload \ - --vae-use-tiling \ - --default-sampling-params \ - '{"0":{"height":512,"width":768,"num_frames":121,"fps":24,"frame_rate":24.0,"output_type":"np"}}' + --vae-use-tiling ``` -The converted repository selects the distilled transformer by default. When a -request supplies neither `sigmas`, `timesteps`, nor `num_inference_steps`, the -adapter uses the official eight-sigma distilled schedule and disables CFG, -spatio-temporal guidance, and modality guidance. An explicit schedule or step -count takes precedence; a step count without `sigmas` uses Diffusers' generic -schedule and may reduce quality. The adapter preserves both `frames` and the -pipeline's singular `audio` output and reads the audio sample rate from the -loaded vocoder (48 kHz for this checkpoint). - -This path currently covers the default one-stage distilled T2V pipeline. -Loading `transformer_full`, two-stage upsampling, image/video conditioning, and -the diffusion video decoder require additional pipeline assembly and are not -enabled by this example. Native vLLM-Omni acceleration features listed under -[Limitations](#limitations) remain unavailable with the Diffusers backend. +The converted repository loads the distilled transformer. When no `sigmas`, +`timesteps`, or `num_inference_steps` is supplied, the adapter applies the +official eight-sigma unguided schedule and returns synchronized 48 kHz audio. +Support is limited to one-stage distilled T2V; `transformer_full`, two-stage +upsampling, image/video conditioning, and the diffusion video decoder are not +supported. The general Diffusers backend [limitations](#limitations) apply. #### Wan Series diff --git a/tests/diffusion/diffusion_backend/test_diffusers_backend.py b/tests/diffusion/diffusion_backend/test_diffusers_backend.py index 23d1630b9..a9f40f771 100644 --- a/tests/diffusion/diffusion_backend/test_diffusers_backend.py +++ b/tests/diffusion/diffusion_backend/test_diffusers_backend.py @@ -17,7 +17,10 @@ from vllm_omni.diffusion.data import ( OmniDiffusionConfig, ) from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline -from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import LTX25_DISTILLED_SIGMAS +from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import ( + LTX25_DISTILLED_SIGMAS, + get_pipeline_utils, +) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams @@ -401,11 +404,17 @@ class TestPipelineArgumentsHandling: assert isinstance(output, DiffusionOutput) assert output.output == raw_output - def test_adapter_preserves_video_and_singular_audio_output(self): + def test_ltx25_preserves_video_and_singular_audio_output(self, tmp_path): frames = torch.zeros(1, 2, 3, 4, 3) audio = torch.zeros(1, 48000) - adapter = DiffusersAdapterPipeline(od_config=_make_od_config()) + (tmp_path / "model_index.json").write_text( + '{"_class_name":"LTX2Pipeline","duration_head":["diffusers","LTX2DurationHead"]}', + encoding="utf-8", + ) + od_config = _make_od_config(model=str(tmp_path)) + adapter = DiffusersAdapterPipeline(od_config=od_config) adapter._pipeline = SimpleNamespace(vocoder=SimpleNamespace(config=SimpleNamespace(output_sampling_rate=48000))) + adapter._pipeline_utils = get_pipeline_utils("LTX2Pipeline") output = adapter._wrap_output(SimpleNamespace(frames=frames, audio=audio)) @@ -428,7 +437,6 @@ class TestPipelineArgumentsHandling: height=None, width=None, num_frames=None, - stg_scale=None, num_images_per_prompt=None, num_videos_per_prompt=None, output_type=None, @@ -450,7 +458,6 @@ class TestPipelineArgumentsHandling: diffusers_call_kwargs={ "guidance_scale": 1.25, "eta": 0.3, - "stg_scale": 0.25, "output_type": "np", } ) @@ -471,12 +478,6 @@ class TestPipelineArgumentsHandling: num_outputs_per_prompt=2, seed=123, output_type="pil", - extra_args={ - "num_inference_steps": 99, - "stg_scale": 0.75, - "unsupported_argument": True, - "extra_args": {"must_not": "leak"}, - }, ), ) @@ -489,9 +490,6 @@ class TestPipelineArgumentsHandling: assert kwargs["height"] == 320 assert kwargs["width"] == 640 assert kwargs["num_frames"] == 8 - assert kwargs["stg_scale"] == 0.75 - assert "unsupported_argument" not in kwargs - assert "extra_args" not in kwargs assert kwargs["num_images_per_prompt"] == 2 assert kwargs["num_videos_per_prompt"] == 2 assert kwargs["output_type"] == "pil" diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py index 78e71bafd..6c68eb5d1 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py @@ -116,16 +116,15 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): dtype = self.od_config.dtype validate_model_compatibility(self.od_config) - pipeline_class = self.od_config.diffusers_pipeline_cls - pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None - self._pipeline_utils = get_pipeline_utils(pipeline_class_name) - load_kwargs = { "torch_dtype": dtype, **self.od_config.diffusers_load_kwargs, } convert_diffusers_quantization_config(load_kwargs) + pipeline_class = self.od_config.diffusers_pipeline_cls + pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None + self._pipeline_utils = get_pipeline_utils(pipeline_class_name) self._pipeline_utils.update_load_kwargs(self.od_config, load_kwargs) component_names = ( self._load_diffusers_component_names(model_id, load_kwargs) @@ -136,9 +135,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}") self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs) - # Gated repositories may be loadable with an explicit token in - # diffusers_load_kwargs even when config enrichment could not resolve - # their pipeline class. In that case the loaded class is authoritative. if pipeline_class_name is None: self._pipeline_utils = get_pipeline_utils(self._pipeline.__class__.__name__) self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config) @@ -157,14 +153,14 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): # VAE slicing and tiling: try-catch because not all models have VAE if self.od_config.vae_use_slicing: try: - self._enable_vae_optimization("slicing") + self._pipeline_utils.enable_vae_optimization(self._pipeline, "slicing") except Exception as e: logger.warning( f"Failed to enable VAE slicing for diffusers pipeline {self._pipeline.__class__.__name__}: {e}" ) if self.od_config.vae_use_tiling: try: - self._enable_vae_optimization("tiling") + self._pipeline_utils.enable_vae_optimization(self._pipeline, "tiling") except Exception as e: logger.warning( f"Failed to enable VAE tiling for diffusers pipeline {self._pipeline.__class__.__name__}: {e}" @@ -173,18 +169,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): # Attention backend self._set_attention_backend() - def _enable_vae_optimization(self, optimization: str) -> None: - """Enable a VAE optimization through the pipeline or VAE component API.""" - pipeline_method = getattr(self._pipeline, f"enable_vae_{optimization}", None) - if callable(pipeline_method): - pipeline_method() - return - - vae_method = getattr(getattr(self._pipeline, "vae", None), f"enable_{optimization}", None) - if not callable(vae_method): - raise AttributeError(f"Neither the pipeline nor its VAE supports {optimization}") - vae_method() - # ------------------------------------------------------------------ # Step-wise execution — explicitly rejected # ------------------------------------------------------------------ @@ -393,19 +377,8 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): ) # Request-time sampling params - for key, value in sampling.extra_args.items(): - if value is None: - continue - if self._accept_call_kwargs is None or key in self._accept_call_kwargs: - kwargs[key] = value - else: - logger.warning( - f"Skipping unsupported diffusers pipeline __call__ argument `{key}` from extra_args. " - f"Check out the documentation of {self._pipeline.__class__.__name__}." - ) - for key, value in sampling.__dict__.items(): - if key == "extra_args" or value is None: + if value is None: continue if self._accept_call_kwargs is None or key in self._accept_call_kwargs: kwargs[key] = value @@ -533,26 +506,13 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): """ from vllm_omni.diffusion.data import DiffusionOutput + output = self._pipeline_utils.normalize_output(self._pipeline, self.od_config, output) + if hasattr(output, "images"): # Preserve diffusers image format (`output_type`) return DiffusionOutput(output=output.images) if hasattr(output, "frames"): - audio = getattr(output, "audio", None) - if audio is not None: - metadata: dict[str, Any] = {} - if (sample_rate := self._get_output_audio_sample_rate()) is not None: - metadata["audio"] = {"sample_rate": sample_rate} - envelope: dict[str, Any] = { - "payload": { - "video": output.frames, - "audio": audio, - } - } - if metadata: - envelope["metadata"] = metadata - return DiffusionOutput(output=envelope) - # Preserve diffusers video format (`output_type`) return DiffusionOutput(output=output.frames) @@ -561,24 +521,6 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): return DiffusionOutput(output=output) - def _get_output_audio_sample_rate(self) -> int | None: - vocoder = getattr(self._pipeline, "vocoder", None) - config = getattr(vocoder, "config", None) - if config is None: - return None - - if hasattr(config, "get"): - value = config.get("output_sampling_rate") - else: - value = getattr(config, "output_sampling_rate", None) - if isinstance(value, bool): - return None - try: - sample_rate = int(value) - except (TypeError, ValueError): - return None - return sample_rate if sample_rate > 0 else None - @staticmethod def _summarize_call_kwargs_value(value: Any) -> Any: """Return a sanitized summary of diffusers call kwargs for logging.""" diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py index 1efbdc0ed..4321365e9 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py @@ -90,6 +90,9 @@ class BasePipelineUtils: def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None: pass + def enable_vae_optimization(self, pipeline: DiffusionPipeline, optimization: str) -> None: + getattr(pipeline, f"enable_vae_{optimization}")() + def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None: pass @@ -100,6 +103,14 @@ class BasePipelineUtils: ) -> None: pass + def normalize_output( + self, + pipeline: DiffusionPipeline, + od_config: OmniDiffusionConfig, + output: Any, + ) -> Any: + return output + def validate_model_compatibility(od_config: OmniDiffusionConfig) -> None: """Reject known model/dependency combinations before loading weights.""" @@ -123,6 +134,13 @@ def validate_model_compatibility(od_config: OmniDiffusionConfig) -> None: class LTX2PipelineUtils(BasePipelineUtils): """Diffusers call policy for the official default LTX-2.5 checkpoint.""" + def enable_vae_optimization(self, pipeline: DiffusionPipeline, optimization: str) -> None: + pipeline_method = getattr(pipeline, f"enable_vae_{optimization}", None) + if callable(pipeline_method): + pipeline_method() + else: + getattr(pipeline.vae, f"enable_{optimization}")() + def update_call_kwargs( self, od_config: OmniDiffusionConfig, @@ -149,6 +167,35 @@ class LTX2PipelineUtils(BasePipelineUtils): if call_kwargs.get(key) is None: call_kwargs[key] = value + def normalize_output( + self, + pipeline: DiffusionPipeline, + od_config: OmniDiffusionConfig, + output: Any, + ) -> Any: + if not is_ltx25_diffusers_model(od_config): + return output + + frames = getattr(output, "frames", None) + audio = getattr(output, "audio", None) + if frames is None or audio is None: + return output + + envelope: dict[str, Any] = {"payload": {"video": frames, "audio": audio}} + config = getattr(getattr(pipeline, "vocoder", None), "config", None) + value = ( + config.get("output_sampling_rate") + if hasattr(config, "get") + else getattr(config, "output_sampling_rate", None) + ) + try: + sample_rate = None if isinstance(value, bool) else int(value) + except (TypeError, ValueError): + sample_rate = None + if sample_rate is not None and sample_rate > 0: + envelope["metadata"] = {"audio": {"sample_rate": sample_rate}} + return envelope + class WanPipelineUtils(BasePipelineUtils): def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None: