From f19891e4f1bf1f4b29dec941e722a74069a82c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Tue, 5 May 2026 20:54:39 +0800 Subject: [PATCH] [bugfix][CI] Diffusers backend update (#3096) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .buildkite/test-nightly.yml | 2 + docs/.nav.yml | 2 +- .../diffusers_pipeline_adapter.md | 115 +++-- .../diffusers_pipeline_adapter/README.md | 117 +++-- .../stage_config.yaml | 31 -- .../test_diffusers_backend.py | 449 ++++++++++++++++++ tests/diffusion/test_diffusers_adapter.py | 186 -------- tests/diffusion/test_diffusion_scheduler.py | 2 +- tests/e2e/accuracy/conftest.py | 26 +- tests/e2e/accuracy/ftfy_mock/sitecustomize.py | 90 ++++ tests/e2e/accuracy/helpers.py | 33 ++ .../test_diffusers_backend_similarity.py | 356 ++++++++++++++ .../online_serving/test_diffusers_adapter.py | 56 --- tests/helpers/runtime.py | 11 +- vllm_omni/diffusion/data.py | 19 + vllm_omni/diffusion/diffusion_engine.py | 42 +- .../pipeline_diffusers_adapter.py | 216 +++++++-- .../diffusers_adapter/pipeline_utils.py | 64 +++ 18 files changed, 1366 insertions(+), 451 deletions(-) delete mode 100644 examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml create mode 100644 tests/diffusion/diffusion_backend/test_diffusers_backend.py delete mode 100644 tests/diffusion/test_diffusers_adapter.py create mode 100644 tests/e2e/accuracy/ftfy_mock/sitecustomize.py create mode 100644 tests/e2e/accuracy/test_diffusers_backend_similarity.py delete mode 100644 tests/e2e/online_serving/test_diffusers_adapter.py create mode 100644 vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 675ce4e6b..f3c8027ef 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -512,6 +512,7 @@ steps: timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model agents: queue: "mithril-h100-pool" plugins: @@ -710,6 +711,7 @@ steps: commands: - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m advanced_model --run-level advanced_model + - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model agents: queue: "mithril-h100-pool" plugins: diff --git a/docs/.nav.yml b/docs/.nav.yml index 3ad99b5d4..ddd2b0308 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -34,7 +34,7 @@ nav: - Online Serving: - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md - vLLM-Omni Helm Chart: user_guide/examples/online_serving/chart-helm.md - - Diffusers Backend Adapter Example: user_guide/examples/online_serving/diffusers_pipeline_adapter.md + - Diffusers Backend Adapter: user_guide/examples/online_serving/diffusers_pipeline_adapter.md - GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md - Image-To-Image: user_guide/examples/online_serving/image_to_image.md - Image-To-Video: user_guide/examples/online_serving/image_to_video.md diff --git a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md index ac88071d5..1adc61193 100644 --- a/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md +++ b/docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md @@ -1,18 +1,14 @@ -# Diffusers Backend Adapter Example +# Diffusers Backend Adapter Source . -This example demonstrates how to serve any 🤗 Diffusers pipeline through vLLM-Omni -using the `diffusers` load format. - -## Supported Models - -Any model loadable via `DiffusionPipeline.from_pretrained()` should be supported, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio. +vLLM-Omni supports running diffusion models with the diffusers backend, directly serving any 🤗 Diffusers pipeline online without implementing them natively. ## Limitations -The diffusers backend is a black-box adapter. The following features are NOT yet supported. +The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online. +Currently, the following features are NOT yet supported. It is not guaranteed whether they will be supported in the future. - CFG parallel execution @@ -22,72 +18,87 @@ It is not guaranteed whether they will be supported in the future. For these features, it is recommended to use natively supported pipelines instead. -## Usage +## Model Support -### Option 1: CLI arguments +Any model loadable via `DiffusionPipeline.from_pretrained()` should run, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio. + +However, as we strive to ensure output similarity between vLLM-Omni's diffuser backend and plain diffusers library, the following models are particularly verified: + +- Qwen/Qwen-Image +- Tongyi-MAI/Z-Image-Turbo +- Wan2.2-I2V-A14B-Diffusers + +If you find that a model not listed above also produces different outputs from running diffusers model directly. +Please consider file an issue. + +## Usage ```bash vllm serve "stable-diffusion-v1-5/stable-diffusion-v1-5" \ --omni \ - --diffusion-load-format diffusers \ - --diffusers-load-kwargs '{"use_safetensors": true}' \ - --diffusers-call-kwargs '{"num_inference_steps": 30, "guidance_scale": 7.5}' + --diffusion-load-format diffusers ``` -`--diffusers-load-kwargs` and `--diffusers-call-kwargs` are only valid together with `--diffusion-load-format diffusers`. +Users turn on the diffusers backend primarily through `--diffusion-load-format diffusers` argument. +There are two more optional arguments, `--diffusers-load-kwargs` and `--diffusers-call-kwargs`, +which are only valid together with `--diffusion-load-format diffusers`. -### Option 2: Stage config YAML - -```bash -vllm serve stable-diffusion-v1-5/stable-diffusion-v1-5 --stage-configs-path examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml --omni -``` - -The particular fields of interest are `model`, `diffusion_load_format`, `diffusers_load_kwargs`, and `diffusers_call_kwargs` under `engine_args`. They are the same as the CLI arguments. - -## Send a Request - -```bash -curl http://localhost:8000/v1/images/generations \ - -H "Content-Type: application/json" \ - -d '{ - "model": "stable-diffusion-v1-5/stable-diffusion-v1-5", - "prompt": "a photo of an astronaut riding a horse on mars", - "n": 1, - "size": "512x512" - }' -``` - -Or refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`. +After launching the model, users send a request as usual. Refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`. ## Configuration Reference -For the diffusers adapter, set options under **`engine_args`**: +### `--diffusers-load-kwargs` -### `diffusion_load_format: "diffusers"` +Passed as-is to `DiffusionPipeline.from_pretrained()`. -This field selects the Hugging Face diffusers adapter path (see `DiffusersPipelineLoader`). - -### `diffusers_load_kwargs` - -Passed to `DiffusionPipeline.from_pretrained()`. - -This is suitable for model-specific configurations not available through the vLLM-Omni interface (such as `Omni.__init__()`, `vllm serve` CLI arguments, and stage config YAML fields outside `diffusers_load_kwargs`). +This is suitable for model-specific configurations not available through the vLLM-Omni interface. +For example: `--diffusers-load-kwargs '{"use_safetensors": true}'`. When a parameter is available in the vLLM-Omni interface, it will be adapted here. But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_load_kwargs`, the **latter** will take precedence. -### `diffusers_call_kwargs` +### `--diffusers-call-kwargs` Passed to `pipeline.__call__()`. -This is suitable for sampling parameters not available through the vLLM-Omni interface (such as `Omni.generate()` and online serving payloads). +This is suitable for sampling parameters not available through the vLLM-Omni interface (such as online serving payloads). When a parameter is available in the vLLM-Omni interface, it will be adapted here. But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time). -## Example materials +### Attention Backends -??? abstract "stage_config.yaml" - ``````yaml - --8<-- "examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml" - `````` +The diffusers backend converts +[vLLM-Omni standard of attention backend setting](../../../docs/user_guide/diffusion/attention_backends.md) +to [diffusers standard](https://huggingface.co/docs/diffusers/optimization/attention_backends#available-backends). + +Specifically for `FLASH_ATTN`, it will first attempt to use FlashAttention-3 and then FlashAttention-2. + +For each attempted version of `FLASH_ATTN` and `SAGE_ATTN`, it will first try to load the attention backend from HuggingFace `kernels` library, then without. + +For unsuccessful attention selection or `TORCH_SDPA`, it will use the PyTorch's default attention backend. + +The loaded attention backend and the failed attempts (if any) are logged to console. + +### Model Specific Settings + +The model loading and inference strictly follows the diffusers library, and they may be different from vLLM-Omni's native interface for some specific models. +Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview). +Some particular examples are below. + +#### Wan Series + +The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference. + +Since our `OmniDiffusionConfig` contains these two values ([source](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/diffusion/data.py)), we can directly pass `--boundary-ratio` and `--flow-shift` arguments to `vllm serve` command. + +```bash +vllm serve "Wan2.2-T2V-A14B-Diffusers" \ + --omni \ + --boundary-ratio 0.875 \ + --flow-shift 3 \ + --diffusion-load-format diffusers +``` + +These extra CLI args will be attempted to pass as-is to the `OmniDiffusionConfig` dataclass and being accessible during model loading time. +Special routines inside the pipeline adapter ensures that they are set properly. diff --git a/examples/online_serving/diffusers_pipeline_adapter/README.md b/examples/online_serving/diffusers_pipeline_adapter/README.md index 8dbf9369a..d6f41f632 100644 --- a/examples/online_serving/diffusers_pipeline_adapter/README.md +++ b/examples/online_serving/diffusers_pipeline_adapter/README.md @@ -1,15 +1,11 @@ -# Diffusers Backend Adapter Example +# Diffusers Backend Adapter -This example demonstrates how to serve any 🤗 Diffusers pipeline through vLLM-Omni -using the `diffusers` load format. - -## Supported Models - -Any model loadable via `DiffusionPipeline.from_pretrained()` should be supported, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio. +vLLM-Omni supports running diffusion models with the diffusers backend, directly serving any 🤗 Diffusers pipeline online without implementing them natively. ## Limitations -The diffusers backend is a black-box adapter. The following features are NOT yet supported. +The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online. +Currently, the following features are NOT yet supported. It is not guaranteed whether they will be supported in the future. - CFG parallel execution @@ -19,65 +15,92 @@ It is not guaranteed whether they will be supported in the future. For these features, it is recommended to use natively supported pipelines instead. -## Usage +## Model Support -### Option 1: CLI arguments +Any model loadable via `DiffusionPipeline.from_pretrained()` should run, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio. + +However, as we strive to ensure output similarity between vLLM-Omni's diffuser backend and plain diffusers library, the following models are particularly verified: + +- Qwen/Qwen-Image +- Tongyi-MAI/Z-Image-Turbo +- Wan2.2-I2V-A14B-Diffusers + +If you find that a model not listed above also produces different outputs from running diffusers model directly. +Please consider file an issue or submit a PR to fix. + +## Usage ```bash vllm serve "stable-diffusion-v1-5/stable-diffusion-v1-5" \ --omni \ - --diffusion-load-format diffusers \ - --diffusers-load-kwargs '{"use_safetensors": true}' \ - --diffusers-call-kwargs '{"num_inference_steps": 30, "guidance_scale": 7.5}' + --diffusion-load-format diffusers ``` -`--diffusers-load-kwargs` and `--diffusers-call-kwargs` are only valid together with `--diffusion-load-format diffusers`. +Users turn on the diffusers backend primarily through `--diffusion-load-format diffusers` argument. +There are two more optional arguments, `--diffusers-load-kwargs` and `--diffusers-call-kwargs`, +which are only valid together with `--diffusion-load-format diffusers`. -### Option 2: Stage config YAML - -```bash -vllm serve stable-diffusion-v1-5/stable-diffusion-v1-5 --stage-configs-path examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml --omni -``` - -The particular fields of interest are `model`, `diffusion_load_format`, `diffusers_load_kwargs`, and `diffusers_call_kwargs` under `engine_args`. They are the same as the CLI arguments. - -## Send a Request - -```bash -curl http://localhost:8000/v1/images/generations \ - -H "Content-Type: application/json" \ - -d '{ - "model": "stable-diffusion-v1-5/stable-diffusion-v1-5", - "prompt": "a photo of an astronaut riding a horse on mars", - "n": 1, - "size": "512x512" - }' -``` - -Or refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`. +After launching the model, users send a request as usual. Refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`. ## Configuration Reference -For the diffusers adapter, set options under **`engine_args`**: +### `--diffusers-load-kwargs` -### `diffusion_load_format: "diffusers"` +Passed as-is to `DiffusionPipeline.from_pretrained()`. -This field selects the Hugging Face diffusers adapter path (see `DiffusersPipelineLoader`). - -### `diffusers_load_kwargs` - -Passed to `DiffusionPipeline.from_pretrained()`. - -This is suitable for model-specific configurations not available through the vLLM-Omni interface (such as `Omni.__init__()`, `vllm serve` CLI arguments, and stage config YAML fields outside `diffusers_load_kwargs`). +This is suitable for model-specific configurations not available through the vLLM-Omni interface. +For example: `--diffusers-load-kwargs '{"use_safetensors": true}'`. When a parameter is available in the vLLM-Omni interface, it will be adapted here. But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_load_kwargs`, the **latter** will take precedence. -### `diffusers_call_kwargs` +### `--diffusers-call-kwargs` Passed to `pipeline.__call__()`. -This is suitable for sampling parameters not available through the vLLM-Omni interface (such as `Omni.generate()` and online serving payloads). +This is suitable for sampling parameters not available through the vLLM-Omni interface (such as online serving payloads). When a parameter is available in the vLLM-Omni interface, it will be adapted here. But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time). + +!!! note + In vLLM-Omni, the default values for some sampling parameters may be different from diffusers. + Consider referring to [`OmniDiffusionSamplingParams`](../../../vllm_omni/inputs/data.py) for the default sampling parameters in the vLLM-Omni interface, + and the corresponding [diffusers pipeline](https://huggingface.co/docs/diffusers/main/en/api/pipelines/overview)'s `__call__` function documentation. + +### Attention Backends + +The diffusers backend converts +[vLLM-Omni standard of attention backend setting](../../../docs/user_guide/diffusion/attention_backends.md) +to [diffusers standard](https://huggingface.co/docs/diffusers/optimization/attention_backends#available-backends). + +Specifically for `FLASH_ATTN`, it will first attempt to use FlashAttention-3 and then FlashAttention-2. + +For each attempted version of `FLASH_ATTN` and `SAGE_ATTN`, it will first try to load the attention backend from HuggingFace `kernels` library, then without. + +For unsuccessful attention selection or `TORCH_SDPA`, it will use the PyTorch's default attention backend. + +The loaded attention backend and the failed attempts (if any) are logged to console. + +### Model Specific Settings + +The model loading and inference strictly follows the diffusers library, and they may be different from vLLM-Omni's native interface for some specific models. +Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview). +Some particular examples are below. + +#### Wan Series + +The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference. + +Since our `OmniDiffusionConfig` contains these two values ([source](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/diffusion/data.py)), we can directly pass `--boundary-ratio` and `--flow-shift` arguments to `vllm serve` command. + +```bash +vllm serve "Wan2.2-T2V-A14B-Diffusers" \ + --omni \ + --boundary-ratio 0.875 \ + --flow-shift 3 \ + --diffusion-load-format diffusers +``` + +These extra CLI args will be attempted to pass as-is to the `OmniDiffusionConfig` dataclass and being accessible during model loading time. +Special routines inside the pipeline adapter ensures that they are set properly. diff --git a/examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml b/examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml deleted file mode 100644 index 7c96eb6c1..000000000 --- a/examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Example stage config for diffusers backend -# This config demonstrates serving Stable Diffusion 1.5 via the diffusers adapter. -# Users should copy and modify this for their own models. - -model_type: diffusion - -stage_args: - - stage_id: 0 - stage_type: diffusion - engine_args: - model_stage: diffusion - model: "stable-diffusion-v1-5/stable-diffusion-v1-5" - distributed_executor_backend: "mp" - # gpu_memory_utilization: 0.9 - engine_output_type: image - # Select the HF diffusers adapter - diffusion_load_format: "diffusers" - # model_class_name: "DiffusersAdapterPipeline" # default when diffusion_load_format is diffusers - diffusers_load_kwargs: - # Passed to DiffusionPipeline.from_pretrained(). - # Good for model-specific loading parameters not covered by OmniDiffusionConfig. - # During model load time, parameters here override their counterparts in the vLLM-Omni interface. - use_safetensors: true - diffusers_call_kwargs: - # Passed to pipeline.__call__(). - # Good for model-specific sampling parameters not covered by OmniDiffusionSamplingParams. - # During request time, parameters here are overridden by the counterparts in OmniDiffusionSamplingParams. - num_inference_steps: 30 - guidance_scale: 7.5 - final_output: true - final_output_type: image diff --git a/tests/diffusion/diffusion_backend/test_diffusers_backend.py b/tests/diffusion/diffusion_backend/test_diffusers_backend.py new file mode 100644 index 000000000..92dd73358 --- /dev/null +++ b/tests/diffusion/diffusion_backend/test_diffusers_backend.py @@ -0,0 +1,449 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections import namedtuple +from types import SimpleNamespace + +import pytest +import torch +from diffusers import DiffusionPipeline # pyright: ignore[reportPrivateImportUsage] +from PIL import Image + +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data +from vllm_omni.diffusion.data import ( + DiffusionOutput, + DiffusionParallelConfig, + OmniDiffusionConfig, +) +from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +pytestmark = [pytest.mark.diffusion] + + +def _make_od_config(**overrides) -> OmniDiffusionConfig: + od_config = OmniDiffusionConfig( + model="test/model", + model_class_name="DiffusersAdapterPipeline", + dtype=torch.float16, + diffusion_load_format="diffusers", + diffusers_load_kwargs={}, + diffusers_call_kwargs={}, + output_type="pil", + parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, sequence_parallel_size=1), + cache_backend="none", + ) + for key, value in overrides.items(): + setattr(od_config, key, value) + return od_config + + +def _make_request(**overrides) -> OmniDiffusionRequest: + prompt = overrides.pop("prompt", "a test prompt") + negative_prompt = overrides.pop("negative_prompt", None) + prompt_obj: dict[str, str] = {"prompt": prompt} + if negative_prompt is not None: + prompt_obj["negative_prompt"] = negative_prompt + + defaults = { + "prompts": [prompt_obj], + "sampling_params": OmniDiffusionSamplingParams( + num_inference_steps=20, + guidance_scale=7.5, + height=16, + width=16, + num_frames=1, + num_outputs_per_prompt=1, + seed=42, + output_type="pil", + generator_device="cpu", + ), + } + defaults.update(overrides) + return OmniDiffusionRequest(**defaults) + + +@pytest.mark.core_model +@pytest.mark.cpu +class TestPipelineArgumentsHandling: + def test_adapter_forward_returns_output(self, mocker): + od_config = _make_od_config() + request = _make_request() + stub_image = Image.new("RGB", (request.sampling_params.width, request.sampling_params.height)) # pyright: ignore[reportArgumentType] + + adapter = DiffusersAdapterPipeline(od_config=od_config) + MockPipelineOutput = namedtuple("MockPipelineOutput", ["image"]) + MockPipeline = type("MockPipeline", (DiffusionPipeline,), {}) + adapter._pipeline = MockPipeline() + + mocker.patch.object( + MockPipeline, + "__call__", + return_value=MockPipelineOutput(image=stub_image), + ) + output = adapter.forward(request) + + assert isinstance(output, DiffusionOutput) + assert isinstance(output.output, MockPipelineOutput) + assert output.output.image is stub_image + + @pytest.mark.parametrize( + "feature_id", + ["cfg_parallel", "ulysses", "ring", "teacache", "cache_dit", "enforce_eager", "quantization"], + ) + def test_adapter_guard_unsupported_feature(self, feature_id): + if feature_id == "cfg_parallel": + od_config = _make_od_config( + parallel_config=DiffusionParallelConfig(cfg_parallel_size=2, sequence_parallel_size=1), + cache_backend="none", + ) + elif feature_id == "ulysses": + od_config = _make_od_config( + parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, ulysses_degree=2), + cache_backend="none", + ) + elif feature_id == "ring": + od_config = _make_od_config( + parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, ring_degree=2), + cache_backend="none", + ) + elif feature_id == "teacache": + od_config = _make_od_config( + parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, sequence_parallel_size=1), + cache_backend="tea_cache", + ) + elif feature_id == "cache_dit": + od_config = _make_od_config( + parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, sequence_parallel_size=1), + cache_backend="cache_dit", + ) + elif feature_id == "enforce_eager": + od_config = _make_od_config(enforce_eager=True) + elif feature_id == "quantization": + od_config = _make_od_config(quantization_config=SimpleNamespace(quant_method="fp8")) + else: + raise ValueError(f"Unknown feature ID: {feature_id}") + + with pytest.raises(NotImplementedError): + DiffusersAdapterPipeline(od_config=od_config) + + def test_adapter_guard_unknown_output_type(self, mocker): + """Test that the adapter wraps an unknown output type as-is. + This is useful when `return_dict=True` and the diffusers pipeline returns an OrderedDict subclass.""" + + adapter = DiffusersAdapterPipeline(od_config=_make_od_config()) + raw_output = {"unexpected": "dict-output"} + + MockPipeline = type("MockPipeline", (DiffusionPipeline,), {}) + adapter._pipeline = MockPipeline() + + mocker.patch.object( + MockPipeline, + "__call__", + return_value=raw_output, + ) + output = adapter.forward(_make_request()) + + assert isinstance(output, DiffusionOutput) + assert output.output == raw_output + + def test_adapter_build_call_kwargs(self, mocker): + class MockPipeline: + def __call__( + self, + prompt=None, + negative_prompt=None, + num_inference_steps=None, + # guidance_scale=None, # deliberately not included + height=None, + width=None, + num_frames=None, + num_images_per_prompt=None, + num_videos_per_prompt=None, + output_type=None, + generator=None, + ): + """Need to make the signature match the actual DiffusionPipeline.__call__, because inspect.signature() is used""" + return None + + def to(self, device): + return self + + mock_from_pretrained = mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained", + return_value=MockPipeline(), + ) + + adapter = DiffusersAdapterPipeline( + od_config=_make_od_config( + diffusers_call_kwargs={ + "guidance_scale": 1.25, + "eta": 0.3, + "output_type": "np", + } + ) + ) + + adapter.load_weights() + mock_from_pretrained.assert_called_once() + + req = _make_request( + prompt="a cat on mars", + negative_prompt="low quality", + sampling_params=OmniDiffusionSamplingParams( + num_inference_steps=9, + guidance_scale=8.0, + height=320, + width=640, + num_frames=8, + num_outputs_per_prompt=2, + seed=123, + output_type="pil", + ), + ) + + kwargs = adapter._build_call_kwargs(req) + + assert kwargs["prompt"] == "a cat on mars" + assert kwargs["negative_prompt"] == "low quality" + assert kwargs["num_inference_steps"] == 9 + assert "guidance_scale" not in kwargs + assert kwargs["height"] == 320 + assert kwargs["width"] == 640 + assert kwargs["num_frames"] == 8 + assert kwargs["num_images_per_prompt"] == 2 + assert kwargs["num_videos_per_prompt"] == 2 + assert kwargs["output_type"] == "pil" + assert isinstance(kwargs["generator"], torch.Generator) + assert kwargs["generator"].device.type == "cpu" + assert kwargs["generator"].initial_seed() == 123 + + def test_adapter_extract_input_reads_negative_prompt_fallback(self): + # 1. test with diffusers_call_kwargs.negative_prompt is a correct list[str] + adapter = DiffusersAdapterPipeline( + od_config=_make_od_config( + diffusers_call_kwargs={ + "negative_prompt": [ + "fallback negative prompt 0", + "fallback negative prompt 1", + "fallback negative prompt 2", + ], + } + ) + ) + input_kwargs = adapter._extract_input( + [ + "a prompt from a string", + {"prompt": "a prompt from a dict"}, + { + "prompt": "a prompt with its own negative prompt", + "negative_prompt": "request negative prompt", + }, + ] + ) + assert input_kwargs["prompt"] == [ + "a prompt from a string", + "a prompt from a dict", + "a prompt with its own negative prompt", + ] + assert input_kwargs["negative_prompt"] == [ + "fallback negative prompt 0", + "fallback negative prompt 1", + "request negative prompt", + ] + + # 2. test with diffusers_call_kwargs.negative_prompt is a single str + adapter = DiffusersAdapterPipeline( + od_config=_make_od_config(diffusers_call_kwargs={"negative_prompt": "fallback negative prompt"}) + ) + input_kwargs = adapter._extract_input( + [ + "a prompt from a string", + {"prompt": "a prompt from a dict"}, + { + "prompt": "a prompt with its own negative prompt", + "negative_prompt": "request negative prompt", + }, + ] + ) + assert input_kwargs["prompt"] == [ + "a prompt from a string", + "a prompt from a dict", + "a prompt with its own negative prompt", + ] + assert input_kwargs["negative_prompt"] == [ + "fallback negative prompt", + "fallback negative prompt", + "request negative prompt", + ] + + # 3. test with diffusers_call_kwargs.negative_prompt is None + adapter = DiffusersAdapterPipeline(od_config=_make_od_config(diffusers_call_kwargs={})) + input_kwargs = adapter._extract_input( + [ + "a prompt from a string", + {"prompt": "a prompt from a dict"}, + { + "prompt": "a prompt with its own negative prompt", + "negative_prompt": "request negative prompt", + }, + ] + ) + assert input_kwargs["prompt"] == [ + "a prompt from a string", + "a prompt from a dict", + "a prompt with its own negative prompt", + ] + assert input_kwargs["negative_prompt"] == ["", "", "request negative prompt"] + + # 4. test with diffusers_call_kwargs.negative_prompt is a list[str] but its length is less than the number of prompts + adapter = DiffusersAdapterPipeline( + od_config=_make_od_config( + diffusers_call_kwargs={ + "negative_prompt": [ + "fallback negative prompt 0", + ] + } + ) + ) + with pytest.raises(ValueError): + adapter._extract_input( + [ + "a prompt from a string", + {"prompt": "a prompt from a dict"}, + { + "prompt": "a prompt with its own negative prompt", + "negative_prompt": "request negative prompt", + }, + ] + ) + + # 5. test with diffusers_call_kwargs.negative_prompt is a list[str] but its length is greater than the number of prompts + adapter = DiffusersAdapterPipeline( + od_config=_make_od_config( + diffusers_call_kwargs={ + "negative_prompt": [ + "fallback negative prompt 0", + "fallback negative prompt 1", + "fallback negative prompt 2", + "fallback negative prompt 3", + ] + } + ) + ) + input_kwargs = adapter._extract_input( + [ + "a prompt from a string", + {"prompt": "a prompt from a dict"}, + { + "prompt": "a prompt with its own negative prompt", + "negative_prompt": "request negative prompt", + }, + ] + ) + assert input_kwargs["prompt"] == [ + "a prompt from a string", + "a prompt from a dict", + "a prompt with its own negative prompt", + ] + assert input_kwargs["negative_prompt"] == [ + "fallback negative prompt 0", + "fallback negative prompt 1", + "request negative prompt", + ] + + # 6. test when no negative prompt is provided + adapter = DiffusersAdapterPipeline(od_config=_make_od_config()) + input_kwargs = adapter._extract_input( + [ + "a prompt from a string", + {"prompt": "a prompt from a dict"}, + ] + ) + assert input_kwargs["prompt"] == [ + "a prompt from a string", + "a prompt from a dict", + ] + assert "negative_prompt" not in input_kwargs + + def test_adapter_load_weights_uses_registered_pipeline_utils(self, mocker): + class WanImageToVideoPipeline: + pass + + class MockPipeline: + def __call__(self, prompt=None): + return None + + def to(self, device): + return self + + od_config = _make_od_config( + diffusers_pipeline_cls=WanImageToVideoPipeline, + boundary_ratio=0.875, + ) + mock_from_pretrained = mocker.patch( + "vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained", + return_value=MockPipeline(), + ) + + pipeline = DiffusersAdapterPipeline(od_config=od_config) + pipeline.load_weights() + + mock_from_pretrained.assert_called_once() + _, kwargs = mock_from_pretrained.call_args + assert kwargs["boundary_ratio"] == 0.875 + + problematic_request = _make_request( + prompt="a prompt from a string", + sampling_params=OmniDiffusionSamplingParams( + boundary_ratio=0.875, + ), + ) + with pytest.raises(ValueError): + pipeline.forward(problematic_request) + + +@pytest.mark.advanced_model +@hardware_test(res={"cuda": "L4"}, num_cards=1) +class TestDiffusersBackendEndToEndExecution: + @pytest.mark.parametrize( + "omni_server", + [ + OmniServerParams( + model="tiny-random/Qwen-Image", + server_args=[ + "--diffusion-load-format", + "diffusers", + "--diffusers-call-kwargs", + '{"height": 512, "width": 0}', # deliberately weird width to be overridden + ], + ), + ], + indirect=True, + ) + def test_t2i_random_weights( + self, + omni_server: OmniServer, + openai_client: OpenAIClientHandler, + ): + messages = dummy_messages_from_mix_data(content_text="a photo of an astronaut riding a horse on mars") + + request_config = { + "model": omni_server.model, + "messages": messages, + "extra_body": { + "width": 512, + "num_inference_steps": 2, + "negative_prompt": "blurry", + "true_cfg_scale": 4.0, + "seed": 42, + }, + } + + response = openai_client.send_diffusion_request(request_config) + image: Image.Image = response[0].images[0] # pyright: ignore[reportOptionalSubscript] + + # Request config has incomplete width/height, so internal assertion in `send_diffusion_request` is incomplete. + assert image.size == (512, 512) diff --git a/tests/diffusion/test_diffusers_adapter.py b/tests/diffusion/test_diffusers_adapter.py deleted file mode 100644 index ac2ec2e3f..000000000 --- a/tests/diffusion/test_diffusers_adapter.py +++ /dev/null @@ -1,186 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections import namedtuple -from types import SimpleNamespace - -import pytest -import torch -from diffusers import DiffusionPipeline -from PIL import Image - -from vllm_omni.diffusion.data import ( - DiffusionOutput, - DiffusionParallelConfig, - OmniDiffusionConfig, -) -from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline -from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.inputs.data import OmniDiffusionSamplingParams - -pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] - - -def _make_od_config(**overrides) -> OmniDiffusionConfig: - od_config = OmniDiffusionConfig( - model="test/model", - model_class_name="DiffusersAdapterPipeline", - dtype=torch.float16, - diffusion_load_format="diffusers", - diffusers_load_kwargs={}, - diffusers_call_kwargs={}, - output_type="pil", - parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, sequence_parallel_size=1), - cache_backend="none", - ) - for key, value in overrides.items(): - setattr(od_config, key, value) - return od_config - - -def _make_request(**overrides) -> OmniDiffusionRequest: - prompt = overrides.pop("prompt", "a test prompt") - negative_prompt = overrides.pop("negative_prompt", None) - prompt_obj: dict[str, str] = {"prompt": prompt} - if negative_prompt is not None: - prompt_obj["negative_prompt"] = negative_prompt - - defaults = { - "prompts": [prompt_obj], - "sampling_params": OmniDiffusionSamplingParams( - num_inference_steps=20, - guidance_scale=7.5, - height=16, - width=16, - num_frames=1, - num_outputs_per_prompt=1, - seed=42, - output_type="pil", - generator_device="cpu", - ), - } - defaults.update(overrides) - return OmniDiffusionRequest(**defaults) - - -class TestDiffusersAdapterPipeline: - def test_adapter_forward_returns_output(self, mocker): - od_config = _make_od_config() - request = _make_request() - stub_image = Image.new("RGB", (request.sampling_params.width, request.sampling_params.height)) # pyright: ignore[reportArgumentType] - - adapter = DiffusersAdapterPipeline(od_config=od_config) - MockPipelineOutput = namedtuple("MockPipelineOutput", ["image"]) - MockPipeline = type("MockPipeline", (DiffusionPipeline,), {}) - adapter._pipeline = MockPipeline() - - mocker.patch.object( - MockPipeline, - "__call__", - return_value=MockPipelineOutput(image=stub_image), - ) - output = adapter.forward(request) - - assert isinstance(output, DiffusionOutput) - assert isinstance(output.output, MockPipelineOutput) - assert output.output.image is stub_image - - @pytest.mark.parametrize( - "feature_id", - ["cfg_parallel", "ulysses", "ring", "teacache", "cache_dit", "enforce_eager", "quantization"], - ) - def test_adapter_guard_unsupported_feature(self, feature_id): - if feature_id == "cfg_parallel": - od_config = _make_od_config( - parallel_config=DiffusionParallelConfig(cfg_parallel_size=2, sequence_parallel_size=1), - cache_backend="none", - ) - elif feature_id == "ulysses": - od_config = _make_od_config( - parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, ulysses_degree=2), - cache_backend="none", - ) - elif feature_id == "ring": - od_config = _make_od_config( - parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, ring_degree=2), - cache_backend="none", - ) - elif feature_id == "teacache": - od_config = _make_od_config( - parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, sequence_parallel_size=1), - cache_backend="tea_cache", - ) - elif feature_id == "cache_dit": - od_config = _make_od_config( - parallel_config=DiffusionParallelConfig(cfg_parallel_size=1, sequence_parallel_size=1), - cache_backend="cache_dit", - ) - elif feature_id == "enforce_eager": - od_config = _make_od_config(enforce_eager=True) - elif feature_id == "quantization": - od_config = _make_od_config(quantization_config=SimpleNamespace(quant_method="fp8")) - else: - raise ValueError(f"Unknown feature ID: {feature_id}") - - with pytest.raises(NotImplementedError): - DiffusersAdapterPipeline(od_config=od_config) - - def test_adapter_guard_unknown_output_type(self, mocker): - """Test that the adapter wraps an unknown output type as-is. - This is useful when `return_dict=True` and the diffusers pipeline returns an OrderedDict subclass.""" - - adapter = DiffusersAdapterPipeline(od_config=_make_od_config()) - raw_output = {"unexpected": "dict-output"} - - MockPipeline = type("MockPipeline", (DiffusionPipeline,), {}) - adapter._pipeline = MockPipeline() - - mocker.patch.object( - MockPipeline, - "__call__", - return_value=raw_output, - ) - output = adapter.forward(_make_request()) - - assert isinstance(output, DiffusionOutput) - assert output.output == raw_output - - def test_adapter_build_call_kwargs(self): - adapter = DiffusersAdapterPipeline( - od_config=_make_od_config( - diffusers_call_kwargs={ - "guidance_scale": 1.25, - "eta": 0.3, - "output_type": "np", - } - ) - ) - req = _make_request( - prompt="a cat on mars", - negative_prompt="low quality", - sampling_params=OmniDiffusionSamplingParams( - num_inference_steps=9, - guidance_scale=8.0, - height=320, - width=640, - num_frames=8, - num_outputs_per_prompt=2, - seed=123, - output_type="pil", - ), - ) - - kwargs = adapter._build_call_kwargs(req) - - assert kwargs["prompt"] == "a cat on mars" - assert kwargs["negative_prompt"] == "low quality" - assert kwargs["num_inference_steps"] == 9 - assert kwargs["guidance_scale"] == 8.0 - assert kwargs["height"] == 320 - assert kwargs["width"] == 640 - assert kwargs["num_frames"] == 8 - assert kwargs["num_images_per_prompt"] == 2 - assert kwargs["output_type"] == "pil" - assert isinstance(kwargs["generator"], torch.Generator) - assert kwargs["generator"].device.type == "cpu" - assert kwargs["generator"].initial_seed() == 123 diff --git a/tests/diffusion/test_diffusion_scheduler.py b/tests/diffusion/test_diffusion_scheduler.py index a64d9920e..6c76d0a46 100644 --- a/tests/diffusion/test_diffusion_scheduler.py +++ b/tests/diffusion/test_diffusion_scheduler.py @@ -414,7 +414,7 @@ class TestDiffusionEngine: def test_dummy_run_raises_on_output_error(self, mocker: MockerFixture) -> None: engine = DiffusionEngine.__new__(DiffusionEngine) - engine.od_config = SimpleNamespace(model_class_name="mock_model") + engine.od_config = SimpleNamespace(model_class_name="mock_model", diffusion_load_format="default") engine.pre_process_func = None engine.add_req_and_wait_for_response = mocker.Mock(return_value=DiffusionOutput(error="boom")) diff --git a/tests/e2e/accuracy/conftest.py b/tests/e2e/accuracy/conftest.py index 709fdf345..3ff3bcc34 100644 --- a/tests/e2e/accuracy/conftest.py +++ b/tests/e2e/accuracy/conftest.py @@ -186,25 +186,39 @@ def accuracy_artifact_root() -> Path: @pytest.fixture(scope="session") -def qwen_bear_image(accuracy_artifact_root: Path) -> Image.Image: +def qwen_bear_image(accuracy_artifact_root: Path): """Download the Qwen bear image from the URL and save it to the accuracy artifact root.""" QWEN_BEAR_IMAGE_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/qwen-bear.png" + image_path = accuracy_artifact_root / "qwen_bear.png" + if image_path.exists(): + image = Image.open(image_path).convert("RGB") + yield image + image.close() + return response = requests.get(QWEN_BEAR_IMAGE_URL, timeout=60) response.raise_for_status() image = Image.open(BytesIO(response.content)).convert("RGB") - image.save(accuracy_artifact_root / "qwen_bear.png") - return image + image.save(image_path) + yield image + image.close() @pytest.fixture(scope="session") -def rabbit_image(accuracy_artifact_root: Path) -> Image.Image: +def rabbit_image(accuracy_artifact_root: Path): """Download the rabbit image from the URL and save it to the accuracy artifact root.""" RABBIT_IMAGE_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/rabbit.png" + image_path = accuracy_artifact_root / "rabbit.png" + if image_path.exists(): + image = Image.open(image_path).convert("RGB") + yield image + image.close() + return response = requests.get(RABBIT_IMAGE_URL, timeout=60) response.raise_for_status() image = Image.open(BytesIO(response.content)).convert("RGB") - image.save(accuracy_artifact_root / "rabbit.png") - return image + image.save(image_path) + yield image + image.close() def _build_accuracy_server_config( diff --git a/tests/e2e/accuracy/ftfy_mock/sitecustomize.py b/tests/e2e/accuracy/ftfy_mock/sitecustomize.py new file mode 100644 index 000000000..1b3a6d39c --- /dev/null +++ b/tests/e2e/accuracy/ftfy_mock/sitecustomize.py @@ -0,0 +1,90 @@ +""" +This sitecustomize.py is used to ensure the presence of an ftfy implementation. +ftfy is a text encoding sanitizer and it is implicitly required by diffusers' WanImageToVideoPipeline. +And this pipeline is used in several accuracy tests. + +A sitecustomize.py is a Python mechanism to inject code into the interpreter at startup. +To use this sitecustomize.py, include its directory in the PYTHONPATH environment variable. + +The sitecustomize approach must be used because diffusers' Wan pipelines may be imported in a subprocess. +For example, when launching with vLLM-Omni's diffusers backend in online serving mode. +`subprocess.Popen` (used by OmniServer test helper) does not inherit any mocks from the parent process. +This is the only way to inject mocks into the subprocess. + +.. note:: + If installing the real ftfy library, the relevant tests may fail the similarity assertion. + Because vLLM-Omni doesn't use ftfy to preprocess the text input. + Hence, we must hack diffusers' Wan pipelines to not use ftfy either. +""" + +import importlib.abc +import sys +from types import ModuleType +from typing import Any + +_TARGET_MODULE = "diffusers.pipelines.wan.pipeline_wan_i2v" + + +class _IdentityFtfy: + @staticmethod + def fix_text(text: str) -> str: + return text + + +def _patch_wan_i2v_module(module: ModuleType) -> None: + # If real ftfy was imported by diffusers, preserve it. + if not hasattr(module, "ftfy"): + module.ftfy = _IdentityFtfy() # type: ignore[attr-defined] + print("ftfy (text encoding sanitizer) is not installed. Using mock ftfy implementation (identity function)") + else: + print("ftfy (text encoding sanitizer) is installed. Using actual ftfy implementation.") + + +class _PatchWanI2VLoader(importlib.abc.Loader): + def __init__(self, wrapped_loader: importlib.abc.Loader) -> None: + self._wrapped_loader = wrapped_loader + + def create_module(self, spec: Any) -> ModuleType | None: + create_module = getattr(self._wrapped_loader, "create_module", None) + if create_module is None: + return None + return create_module(spec) + + def exec_module(self, module: ModuleType) -> None: + self._wrapped_loader.exec_module(module) + _patch_wan_i2v_module(module) + + +class _PatchWanI2VFinder(importlib.abc.MetaPathFinder): + def find_spec(self, fullname: str, path: Any, target: Any = None) -> Any: + if fullname != _TARGET_MODULE: + return None + # Ask the remaining finders for the real module spec. Skip this finder + # to avoid infinite recursion. + for finder in sys.meta_path: + if finder is self: + continue + find_spec = getattr(finder, "find_spec", None) + if find_spec is None: + continue + spec = find_spec(fullname, path, target) + if spec is None: + continue + if spec.loader is not None and not isinstance(spec.loader, _PatchWanI2VLoader): + spec.loader = _PatchWanI2VLoader(spec.loader) + return spec + return None + + +def _install_hook() -> None: + existing_module = sys.modules.get(_TARGET_MODULE) + if existing_module is not None: + _patch_wan_i2v_module(existing_module) + return + if not any(isinstance(finder, _PatchWanI2VFinder) for finder in sys.meta_path): + sys.meta_path.insert(0, _PatchWanI2VFinder()) + + +print("Installing ftfy mock hook", flush=True) +_install_hook() +print("ftfy mock hook installed", flush=True) diff --git a/tests/e2e/accuracy/helpers.py b/tests/e2e/accuracy/helpers.py index 382d3ea9b..62d3e0a69 100644 --- a/tests/e2e/accuracy/helpers.py +++ b/tests/e2e/accuracy/helpers.py @@ -1,4 +1,6 @@ +import os from pathlib import Path +from typing import Any import numpy as np import pytest @@ -6,6 +8,37 @@ import torch from PIL import Image from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure +FTFY_SITECUSTOMIZE_MOCK_DIR = Path(__file__).with_name("ftfy_mock") + + +class IdentityFtfy: + @staticmethod + def fix_text(text: str) -> str: + return text + + +def apply_ftfy_mock(*, wan_i2v_module: Any | None = None) -> None: + """Mock ftfy library for test cases in the main process""" + if wan_i2v_module is None: + from diffusers.pipelines.wan import pipeline_wan_i2v as wan_i2v_module + + if not hasattr(wan_i2v_module, "ftfy"): + wan_i2v_module.ftfy = IdentityFtfy() + print("ftfy (text encoding sanitizer) is not installed. Using mock ftfy implementation (identity function)") + else: + print("ftfy (text encoding sanitizer) is installed. Using actual ftfy implementation.") + + +def env_to_apply_ftfy_mock_in_subproc(env: dict[str, str] | None = None) -> dict[str, str]: + """Mock ftfy library for test cases in subprocesses, by prepending the sitecustomize.py directory to PYTHONPATH""" + env_dict = dict(env or {}) + pythonpath = env_dict.get("PYTHONPATH", os.environ.get("PYTHONPATH", "")) + path_entries = [str(FTFY_SITECUSTOMIZE_MOCK_DIR)] + if pythonpath: + path_entries.append(pythonpath) + env_dict["PYTHONPATH"] = os.pathsep.join(path_entries) + return env_dict + def reset_artifact_dir(path: Path) -> Path: import shutil diff --git a/tests/e2e/accuracy/test_diffusers_backend_similarity.py b/tests/e2e/accuracy/test_diffusers_backend_similarity.py new file mode 100644 index 000000000..b9d452bdc --- /dev/null +++ b/tests/e2e/accuracy/test_diffusers_backend_similarity.py @@ -0,0 +1,356 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Run vLLM-Omni with diffusers backend, then run diffusers directly. Compare the output similarity. +Also, do a quick performance check on the end-to-end latency. +The latency check is not put under /tests/dfx/perf/ because this is not a complete performance check to be monitored. +""" + +import base64 +import gc +import io +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np +import pytest +import requests +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import export_to_video # pyright: ignore[reportPrivateImportUsage] +from PIL import Image + +from benchmarks.accuracy.common import pil_to_base64 +from tests.e2e.accuracy.helpers import ( + apply_ftfy_mock, + assert_similarity, + env_to_apply_ftfy_mock_in_subproc, + model_output_dir, +) +from tests.e2e.accuracy.wan22_i2v.test_wan22_i2v_video_similarity import ( + _parse_psnr_score, + _parse_ssim_score, + _run_ffmpeg_similarity, +) +from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServer, OpenAIClientHandler + +pytestmark = [pytest.mark.full_model, pytest.mark.diffusion] + + +PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style." +NEGATIVE_PROMPT = "blurry, low quality" +WIDTH = 512 +HEIGHT = 512 +NUM_INFERENCE_STEPS = 20 +TRUE_CFG_SCALE = 4.0 +SEED = 42 +SSIM_THRESHOLD = 0.97 +PSNR_THRESHOLD = 30.0 + +VIDEO_PROMPT = "The bear in the image dances happily" +VIDEO_WIDTH = 832 +VIDEO_HEIGHT = 480 +VIDEO_NUM_INFERENCE_STEPS = 10 +NUM_FRAMES = 9 +FPS = 4 +GUIDANCE_SCALE = 4.0 +GUIDANCE_SCALE_2 = 1.0 +BOUNDARY_RATIO = 0.875 +FLOW_SHIFT = 12.0 +VIDEO_SSIM_THRESHOLD = 0.95 +VIDEO_PSNR_THRESHOLD = 30.0 + + +def _run_vllm_omni_wan22_i2v( + *, + model: str, + output_path: Path, + conditioning_image: Image.Image, +) -> float: + server_args = [ + "--num-gpus", + "1", + "--stage-init-timeout", + "400", + "--init-timeout", + "900", + "--diffusion-load-format", + "diffusers", + "--boundary-ratio", + str(BOUNDARY_RATIO), + "--flow-shift", + str(FLOW_SHIFT), + ] + form_data = { + "prompt": VIDEO_PROMPT, + "negative_prompt": NEGATIVE_PROMPT, + "height": VIDEO_HEIGHT, + "width": VIDEO_WIDTH, + "num_inference_steps": VIDEO_NUM_INFERENCE_STEPS, + "num_frames": NUM_FRAMES, + "fps": FPS, + "guidance_scale": GUIDANCE_SCALE, + "guidance_scale_2": GUIDANCE_SCALE_2, + "seed": SEED, + } + with OmniServer(model, server_args, env_dict=env_to_apply_ftfy_mock_in_subproc(), use_omni=True) as omni_server: + client = OpenAIClientHandler( + host=omni_server.host, + port=omni_server.port, + run_level="full_model", + ) + request_config = { + "model": omni_server.model, + "form_data": form_data, + "image_reference": f"data:image/png;base64,{pil_to_base64(conditioning_image, 'png')}", + } + result = client.send_video_diffusion_request(request_config)[0] + video_bytes = result.videos[0] # pyright: ignore[reportOptionalSubscript] # Guaranteed not None + output_path.write_bytes(video_bytes) + return result.e2e_latency # pyright: ignore[reportReturnType] # Guaranteed not None + + +def _run_diffusers_wan22_i2v(*, model: str, output_path: Path, conditioning_image: Image.Image) -> float: + from diffusers import WanImageToVideoPipeline # pyright: ignore[reportPrivateImportUsage] + from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler + + run_pre_test_cleanup(enable_force=True) + apply_ftfy_mock() + pipe: WanImageToVideoPipeline | None = None + try: + pipe = WanImageToVideoPipeline.from_pretrained( + model, + torch_dtype=torch.bfloat16, + trust_remote_code=True, + boundary_ratio=BOUNDARY_RATIO, + ) + pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=FLOW_SHIFT) + pipe.to("cuda") + + _diffusers_dummy_run(pipe) + + generator = torch.Generator(device="cuda").manual_seed(SEED) + with torch.inference_mode(): + start_time = time.perf_counter() + result = pipe( + image=conditioning_image, + prompt=VIDEO_PROMPT, + negative_prompt=NEGATIVE_PROMPT, + height=VIDEO_HEIGHT, + width=VIDEO_WIDTH, + num_inference_steps=VIDEO_NUM_INFERENCE_STEPS, + num_frames=NUM_FRAMES, + guidance_scale=GUIDANCE_SCALE, + guidance_scale_2=GUIDANCE_SCALE_2, + generator=generator, + ) + end_time = time.perf_counter() + e2e_latency = end_time - start_time + frames = result.frames[0] # pyright: ignore[reportAttributeAccessIssue] + export_to_video(frames, str(output_path), fps=FPS) # pyright: ignore[reportArgumentType] + return e2e_latency + finally: + if pipe is not None and hasattr(pipe, "maybe_free_model_hooks"): + pipe.maybe_free_model_hooks() + del pipe + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + run_post_test_cleanup(enable_force=True) + + +def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]: + server_args = [ + "--num-gpus", + "1", + "--stage-init-timeout", + "400", + "--init-timeout", + "900", + "--diffusion-load-format", + "diffusers", + # "--enable-diffusion-pipeline-profiler", + ] + with OmniServer(model, server_args, use_omni=True) as omni_server: + start_time = time.perf_counter() + response = requests.post( + f"http://{omni_server.host}:{omni_server.port}/v1/images/generations", + json={ + "model": omni_server.model, + "prompt": PROMPT, + "size": f"{WIDTH}x{HEIGHT}", + "n": 1, + "response_format": "b64_json", + "negative_prompt": NEGATIVE_PROMPT, + "num_inference_steps": NUM_INFERENCE_STEPS, + "true_cfg_scale": TRUE_CFG_SCALE, + "seed": SEED, + }, + timeout=600, + ) + end_time = time.perf_counter() + e2e_latency = end_time - start_time + response.raise_for_status() + payload = response.json() + assert len(payload["data"]) == 1 + image_bytes = base64.b64decode(payload["data"][0]["b64_json"]) + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + image.load() + image.save(output_path) + return image, e2e_latency + + +def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.Image, float]: + run_pre_test_cleanup(enable_force=True) + pipe: DiffusionPipeline | None = None + try: + pipe = DiffusionPipeline.from_pretrained( + model, + torch_dtype=torch.bfloat16, + trust_remote_code=True, + ).to("cuda") + + _diffusers_dummy_run(pipe) + + generator = torch.Generator(device="cuda").manual_seed(SEED) + with torch.inference_mode(): + start_time = time.perf_counter() + result = pipe( # pyright: ignore[reportCallIssue] + prompt=PROMPT, + negative_prompt=NEGATIVE_PROMPT, + width=WIDTH, + height=HEIGHT, + num_inference_steps=NUM_INFERENCE_STEPS, + true_cfg_scale=TRUE_CFG_SCALE, + generator=generator, + ) + end_time = time.perf_counter() + e2e_latency = end_time - start_time + output_image = result.images[0].convert("RGB") + output_image.save(output_path) + return output_image, e2e_latency + finally: + if pipe is not None and hasattr(pipe, "maybe_free_model_hooks"): + pipe.maybe_free_model_hooks() + del pipe + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + run_post_test_cleanup(enable_force=True) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("model_id", ["Qwen/Qwen-Image"]) +def test_diffusers_backend_t2i_matches_diffusers(model_id: str, accuracy_artifact_root: Path) -> None: + output_dir = model_output_dir(accuracy_artifact_root, model_id + "-diffusers-backend") + + vllm_output, vllm_latency = _run_vllm_omni_qwen_image(model=model_id, output_path=output_dir / "vllm_omni.png") + diffusers_output, diffusers_latency = _run_diffusers_qwen_image( + model=model_id, output_path=output_dir / "diffusers.png" + ) + vllm_latency = vllm_latency * 1000 + diffusers_latency = diffusers_latency * 1000 + latency_threshold_factor = 0.2 + latency_threshold = diffusers_latency * (1 + latency_threshold_factor) + print(f"{model_id} latency metrics:") + print( + f" Latency={vllm_latency:.2f}ms, threshold<={latency_threshold:.2f}ms, diffusers latency={diffusers_latency:.2f}ms, lower is better" + ) + + assert_similarity( + model_name=model_id, + vllm_image=vllm_output, + diffusers_image=diffusers_output, + width=WIDTH, + height=HEIGHT, + ssim_threshold=SSIM_THRESHOLD, + psnr_threshold=PSNR_THRESHOLD, + ) + assert vllm_latency <= latency_threshold, ( + f"VLLM latency ({vllm_latency:.2f}ms) is greater than {latency_threshold_factor * 100}% more than Diffusers latency ({diffusers_latency:.2f}ms)." + ) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("model_id", ["Wan-AI/Wan2.2-I2V-A14B-Diffusers"]) +def test_diffusers_backend_i2v_matches_diffusers( + model_id: str, + accuracy_artifact_root: Path, + qwen_bear_image: Image.Image, +) -> None: + output_dir = model_output_dir(accuracy_artifact_root, model_id + "-diffusers-backend") + + resized_image = qwen_bear_image.resize((VIDEO_WIDTH, VIDEO_HEIGHT), Image.Resampling.LANCZOS) + + vllm_path = output_dir / "vllm_omni.mp4" + vllm_latency = _run_vllm_omni_wan22_i2v(model=model_id, output_path=vllm_path, conditioning_image=resized_image) + vllm_latency = vllm_latency * 1000 + + diffusers_path = output_dir / "diffusers.mp4" + diffusers_latency = _run_diffusers_wan22_i2v( + model=model_id, output_path=diffusers_path, conditioning_image=resized_image + ) + diffusers_latency = diffusers_latency * 1000 + latency_threshold_factor = 0.3 + latency_threshold = diffusers_latency * (1 + latency_threshold_factor) + + ssim_output = _run_ffmpeg_similarity("ssim", vllm_path, diffusers_path) + psnr_output = _run_ffmpeg_similarity("psnr", vllm_path, diffusers_path) + ssim_score = _parse_ssim_score(ssim_output) + psnr_score = _parse_psnr_score(psnr_output) + print(f"{model_id} latency metrics:") + print( + f" Latency={vllm_latency:.2f}ms, threshold<={latency_threshold:.2f}ms, diffusers latency={diffusers_latency:.2f}ms, lower is better" + ) + print(f"{model_id} similarity metrics:") + print(f" SSIM: value={ssim_score:.6f}, threshold>={VIDEO_SSIM_THRESHOLD:.6f}, range=[-1, 1], higher is better") + print( + f" PSNR: value={psnr_score:.6f} dB, threshold>={VIDEO_PSNR_THRESHOLD:.6f} dB, range=[0, +inf), higher is better" + ) + + assert ssim_score >= VIDEO_SSIM_THRESHOLD, ( + f"SSIM below threshold for {model_id}: got {ssim_score:.6f}, expected >= {VIDEO_SSIM_THRESHOLD:.6f}." + ) + assert psnr_score >= VIDEO_PSNR_THRESHOLD, ( + f"PSNR below threshold for {model_id}: got {psnr_score:.6f}, expected >= {VIDEO_PSNR_THRESHOLD:.6f}." + ) + assert vllm_latency <= latency_threshold, ( + f"VLLM latency ({vllm_latency:.2f}ms) is greater than {latency_threshold_factor * 100}% more than Diffusers latency ({diffusers_latency:.2f}ms)." + ) + + +def _diffusers_dummy_run(pipe: DiffusionPipeline) -> None: + from vllm_omni.diffusion.diffusion_engine import supports_multimodal_input + + supports_image_input, supports_audio_input = supports_multimodal_input( + SimpleNamespace( + diffusion_load_format="diffusers", + diffusers_pipeline_cls=pipe.__class__, + model_class_name="DiffusersAdapterPipeline", + ), # pyright: ignore[reportArgumentType] + ) + height = 512 + width = 512 + kwargs: dict[str, Any] = { + "prompt": "dummy run", + "height": height, + "width": width, + "num_inference_steps": 1, + } + if supports_image_input: + # Provide a dummy image input if the model supports it + dummy_image = Image.new("RGB", (width, height)) + kwargs["image"] = dummy_image + if supports_audio_input: + audio_sr = 16000 + dummy_audio = np.random.randn(audio_sr * 2).astype(np.float32) + kwargs["audio"] = dummy_audio + with torch.inference_mode(): + pipe(**kwargs) # pyright: ignore[reportCallIssue] diff --git a/tests/e2e/online_serving/test_diffusers_adapter.py b/tests/e2e/online_serving/test_diffusers_adapter.py deleted file mode 100644 index 8b41db13a..000000000 --- a/tests/e2e/online_serving/test_diffusers_adapter.py +++ /dev/null @@ -1,56 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""End-to-end tests for DiffusersAdapterPipeline. - -It tests the full user flow of launching a diffusers-backed model and running inference. -""" - -import pytest -from PIL import Image - -from tests.helpers.mark import hardware_test -from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler, dummy_messages_from_mix_data - -pytestmark = [pytest.mark.diffusion, pytest.mark.core_model] - - -@pytest.mark.parametrize( - "omni_server", - [ - OmniServerParams( - model="tiny-random/Qwen-Image", - server_args=[ - "--diffusion-load-format", - "diffusers", - "--diffusers-call-kwargs", - '{"height": 512, "width": 0}', # deliberately weird width to be overridden - ], - ), - ], - indirect=True, -) -@hardware_test(res={"cuda": "L4"}, num_cards=1) -def test_t2i_with_diffusers_adapter( - omni_server: OmniServer, - openai_client: OpenAIClientHandler, -): - messages = dummy_messages_from_mix_data(content_text="a photo of an astronaut riding a horse on mars") - - request_config = { - "model": omni_server.model, - "messages": messages, - "extra_body": { - "width": 512, - "num_inference_steps": 2, - "negative_prompt": "blurry", - "true_cfg_scale": 4.0, - "seed": 42, - }, - } - - response = openai_client.send_diffusion_request(request_config) - image: Image.Image = response[0].images[0] # pyright: ignore[reportOptionalSubscript] - - # Request config has incomplete width/height, so internal assertion in `send_diffusion_request` is incomplete. - assert image.size == (512, 512) diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 79ca9ee5c..79f6af791 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -625,7 +625,7 @@ class DiffusionResponse: text_content: str | None = None images: list[Image.Image] | None = None audios: list[Any] | None = None - videos: list[Any] | None = None + videos: list[bytes] | None = None e2e_latency: float | None = None success: bool = False error_message: str | None = None @@ -1054,7 +1054,9 @@ class OpenAIClientHandler: responses.append(response) return responses - def send_video_diffusion_request(self, request_config: dict[str, Any], request_num: int = 1) -> list[OmniResponse]: + def send_video_diffusion_request( + self, request_config: dict[str, Any], request_num: int = 1 + ) -> list[DiffusionResponse]: """ Send native /v1/videos requests. """ @@ -1077,7 +1079,6 @@ class OpenAIClientHandler: normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference}) result = DiffusionResponse() - start_time = time.perf_counter() create_url = self._build_url("/v1/videos") response = requests.post( create_url, @@ -1086,14 +1087,16 @@ class OpenAIClientHandler: headers={"Accept": "application/json"}, timeout=60, ) + start_time = time.perf_counter() response.raise_for_status() job_data = response.json() video_id = job_data["id"] self._wait_until_video_completed(video_id) + end_time = time.perf_counter() video_content = self._download_video_content(video_id) result.success = True result.videos = [video_content] - result.e2e_latency = time.perf_counter() - start_time + result.e2e_latency = end_time - start_time assert_diffusion_response(result, request_config, run_level=self.run_level) return [result] diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 676fcd9c7..18f627bc8 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -8,6 +8,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field, fields from typing import TYPE_CHECKING, Any +import diffusers import torch from PIL import Image from pydantic import model_validator @@ -460,6 +461,8 @@ class OmniDiffusionConfig: diffusers_load_kwargs: dict[str, Any] = field(default_factory=dict) # kwargs forwarded to pipeline.__call__() diffusers_call_kwargs: dict[str, Any] = field(default_factory=dict) + # Actual diffusers pipeline object (to determine inputs of the dummy run) + diffusers_pipeline_cls: type[diffusers.DiffusionPipeline] | None = None # pyright: ignore[reportPrivateImportUsage] # http server endpoint config, would be ignored in local mode host: str | None = None @@ -714,6 +717,15 @@ class OmniDiffusionConfig: # (non-DiT models don't have a separate transformer folder/config) if self.diffusion_load_format == "diffusers": self.tf_model_config = TransformerConfig() + try: + diffusers_pipeline_cls_name = config_dict["_class_name"] + self.diffusers_pipeline_cls = getattr(diffusers, diffusers_pipeline_cls_name) + except (KeyError, AttributeError) as exc: + logger.warning( + f"Could not find valid _class_name for diffusers pipeline in model_index.json: {exc}. " + "Without knowing the underlying diffusers pipeline class, the dummy run will input only " + "text prompt, which may cause errors for pipelines that require additional inputs." + ) else: tf_config_dict = get_hf_file_to_dict("transformer/config.json", self.model) self.tf_model_config = TransformerConfig.from_dict(tf_config_dict) @@ -724,6 +736,13 @@ class OmniDiffusionConfig: # (non-DiT models don't have a separate transformer folder/config) if self.diffusion_load_format == "diffusers": self.tf_model_config = TransformerConfig() + logger.warning( + "Could not find valid model_index.json per diffusers format. " + "This model is suspectedly unsupported by the diffusers backend. " + "Also, without knowing the underlying diffusers pipeline class from model_index.json, " + "the dummy run will input only text prompt, which may cause errors for pipelines " + "that require additional inputs." + ) else: cfg = get_hf_file_to_dict("config.json", self.model) if cfg is None: diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index db5a85cfb..c4c62dd59 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -37,18 +37,23 @@ from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) -def supports_image_input(model_class_name: str) -> bool: - model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name) - if model_cls is None: - return False - return bool(getattr(model_cls, "support_image_input", False)) +def supports_multimodal_input(od_config: OmniDiffusionConfig) -> tuple[bool, bool]: + if od_config.diffusion_load_format == "diffusers" and (pipe_cls := od_config.diffusers_pipeline_cls) is not None: + signature = inspect.signature(pipe_cls.__call__) + support_image_input = "image" in signature.parameters + support_audio_input = ( + "audio" in signature.parameters or "audio_latents" in signature.parameters + ) # ref. LTX-2 format + return support_image_input, support_audio_input + supports_image_input = False + supports_audio_input = False -def supports_audio_input(model_class_name: str) -> bool: - model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name) - if model_cls is None: - return False - return bool(getattr(model_cls, "support_audio_input", False)) + model_cls = DiffusionModelRegistry._try_load_model_cls(od_config.model_class_name) + if model_cls is not None: + supports_image_input = bool(getattr(model_cls, "support_image_input", False)) + supports_audio_input = bool(getattr(model_cls, "support_audio_input", False)) + return supports_image_input, supports_audio_input def image_color_format(model_class_name: str) -> str: @@ -416,23 +421,20 @@ class DiffusionEngine: num_inference_steps = 1 height = 512 width = 512 - if supports_image_input(self.od_config.model_class_name): + prompt: OmniTextPrompt = {"prompt": "dummy run"} + + supports_image_input, supports_audio_input = supports_multimodal_input(self.od_config) + if supports_image_input: # Provide a dummy image input if the model supports it color_format = image_color_format(self.od_config.model_class_name) dummy_image = PIL.Image.new(color_format, (width, height)) - else: - dummy_image = None + prompt.setdefault("multi_modal_data", {})["image"] = dummy_image - if supports_audio_input(self.od_config.model_class_name): + if supports_audio_input: audio_sr = 16000 dummy_audio = np.random.randn(audio_sr * 2).astype(np.float32) - else: - dummy_audio = None + prompt.setdefault("multi_modal_data", {})["audio"] = dummy_audio - prompt: OmniTextPrompt = { - "prompt": "dummy run", - "multi_modal_data": {"image": dummy_image, "audio": dummy_audio}, - } req = OmniDiffusionRequest( prompts=[prompt], request_ids=["dummy_req_id"], diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py index 8a1fdfc08..81733aa8b 100644 --- a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py @@ -13,18 +13,21 @@ It does NOT support: - Step-wise execution (continuous batching) """ +import inspect import logging import os -from typing import Any +import re +from typing import Any, cast import torch from diffusers.pipelines.pipeline_utils import DiffusionPipeline from torch import nn from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import BasePipelineUtils, get_pipeline_utils from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.inputs.data import OmniPromptType +from vllm_omni.inputs.data import OmniPromptType, OmniTextPrompt from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) @@ -49,9 +52,11 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): def __init__(self, *, od_config: OmniDiffusionConfig, device: torch.device | None = None): super().__init__() self._pipeline: DiffusionPipeline + self._accept_call_kwargs: set[str] | None = None # None to accept all kwargs self.od_config = od_config self.device = device self._capabilities: dict[str, Any] = {} + self._pipeline_utils: BasePipelineUtils = BasePipelineUtils() self._raise_unsupported_features() self.setup_diffusion_pipeline_profiler( @@ -77,10 +82,18 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): } logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}") - self._pipeline = DiffusionPipeline.from_pretrained( - model_id, - **load_kwargs, - ).to(self.device) + pipeline_class = self.od_config.diffusers_pipeline_cls + pipeline_class_name = pipeline_class.__name__ if pipeline_class is not None else None + self._pipeline_utils = get_pipeline_utils(pipeline_class_name) + self._pipeline_utils.update_load_kwargs(self.od_config, load_kwargs) + + self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs) + self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config) + + self._pipeline.to(self.device) + + # Cache __call__kwargs signature introspection for later input validation + self._accept_call_kwargs = set(inspect.signature(self._pipeline.__call__).parameters.keys()) # CPU offloading if self.od_config.enable_layerwise_offload: @@ -270,30 +283,50 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): def _build_call_kwargs(self, req: OmniDiffusionRequest) -> dict[str, Any]: """Translate ``OmniDiffusionRequest`` into diffusers ``__call__`` kwargs.""" sampling = req.sampling_params - prompt, neg_prompt = self._extract_prompt(req.prompts) + input_kwargs = self._extract_input(req.prompts) + + self._pipeline_utils.validate_runtime_sampling_params(sampling) # Merge user-provided call kwargs from stage/CLI defaults. - # Request-time parameters take precedence over stage-config defaults - call_kwargs = self.od_config.diffusers_call_kwargs - kwargs: dict[str, Any] = { - **call_kwargs, - "prompt": prompt, - "num_inference_steps": sampling.num_inference_steps, - "guidance_scale": sampling.guidance_scale, - "output_type": sampling.output_type or self.od_config.output_type, - } + # Load time defaults -> input kwargs (prompts, neg prompts, images...) -> request-time sampling params + kwargs: dict[str, Any] = {} - if sampling.height is not None: - kwargs["height"] = sampling.height - if sampling.width is not None: - kwargs["width"] = sampling.width - if sampling.num_frames is not None and sampling.num_frames > 1: - kwargs["num_frames"] = sampling.num_frames - if sampling.num_outputs_per_prompt is not None and sampling.num_outputs_per_prompt > 1: - kwargs["num_images_per_prompt"] = sampling.num_outputs_per_prompt + # Load time defaults + for key, value in self.od_config.diffusers_call_kwargs.items(): + if self._accept_call_kwargs is None or key in self._accept_call_kwargs: + kwargs[key] = value + else: + logger.warning( + f"Skipping unsupported diffusers pipeline __call__ argument `{key}` from " + f"diffusers_call_kwargs. Check out the documentation of {self._pipeline.__class__.__name__}." + ) - if neg_prompt is not None: - kwargs["negative_prompt"] = neg_prompt + # Input kwargs + for key, value in input_kwargs.items(): + if self._accept_call_kwargs is None or key in self._accept_call_kwargs: + kwargs[key] = value + else: + logger.warning( + f"Skipping unsupported diffusers pipeline __call__ argument `{key}` from prompt input." + f"Check out the documentation of {self._pipeline.__class__.__name__}." + ) + + # Request-time sampling params + for key, value in sampling.__dict__.items(): + if value is None: + continue + if self._accept_call_kwargs is None or key in self._accept_call_kwargs: + kwargs[key] = value + + # Special format fields in sampling params + if output_type := sampling.output_type or self.od_config.output_type: + kwargs["output_type"] = output_type + + if (num_outputs_per_prompt := sampling.num_outputs_per_prompt) > 0: + # In diffusers, they are num_images_per_prompt, num_videos_per_prompt, etc. + for key in self._accept_call_kwargs or (): + if re.match(r"num_[a-z]+_per_prompt", key): + kwargs[key] = num_outputs_per_prompt if sampling.generator is not None: kwargs["generator"] = sampling.generator @@ -302,38 +335,100 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): else: kwargs["generator"] = torch.Generator(device=sampling.generator_device) - if sampling.latents is not None: - kwargs["latents"] = sampling.latents + logger.info( + "Calling diffusers pipeline with kwargs: %s", DiffusersAdapterPipeline._summarize_call_kwargs_value(kwargs) + ) return kwargs - @staticmethod - def _extract_prompt(prompt_obj: list[OmniPromptType]) -> tuple[str | list[str], str | list[str] | None]: + def _extract_input(self, prompt_obj: list[OmniPromptType]) -> dict[str, Any]: """Extract the text prompts and negative prompts from a list of prompt objects.""" if len(prompt_obj) == 1: if isinstance(prompt_obj[0], str): - return prompt_obj[0], None + return {"prompt": prompt_obj[0]} else: - return prompt_obj[0].get("prompt", ""), prompt_obj[0].get("negative_prompt", None) + obj = cast(OmniTextPrompt, prompt_obj[0]) + negative_prompt = obj.get("negative_prompt") + multi_modal_data = obj.get("multi_modal_data") or {} + kwargs = { + "prompt": obj.get("prompt", ""), + **multi_modal_data, + } + if negative_prompt is not None: + kwargs["negative_prompt"] = negative_prompt + return kwargs + + # Check the first element for the presence of multimodal data. + # The following elements should have the same multimodal data fields, or none has multimodal data. + multi_modal_data_fields: list[str] = [] + if isinstance(prompt_obj[0], dict) and (multi_modal_data := prompt_obj[0].get("multi_modal_data")): + multi_modal_data_fields = list(multi_modal_data.keys()) + if multi_modal_data_fields: + for i, prompt in enumerate(prompt_obj): + assert isinstance(prompt, dict) and (multi_modal_data := prompt.get("multi_modal_data")) is not None, ( + "When there are multiple prompts and the first prompt has multimodal data, " + f"each prompt should also contain the same multimodal data fields, but prompt {i} does not." + ) + for key in multi_modal_data_fields: + assert key in multi_modal_data, ( + "When there are multiple prompts and the first prompt has multimodal data, each prompt should " + f"also contain the same multimodal data fields, but prompt {i} does not contain {key}." + ) + assert not isinstance(multi_modal_data.get(key), list), ( + f"When there are multiple prompts and each prompt has multiple {key} data, this input pattern " + "is ambiguous as diffusers accepts flattened lists of text prompts and multimodal data. " + f"To use multiple {key} data, please use one single prompt instead." + ) + + input_kwargs = {"prompt": [], **{key: [] for key in multi_modal_data_fields}} + + # Negative prompt rule: + # - If any OmniTextPrompt has a negative prompt, or diffusers_call_kwargs has `negative_prompt`, + # enforce a negative prompt input (list[str]) -> `kwargs_should_contain_negative_prompt=true` + # (Because the negative prompt must be str, list[str], or None. It cannot be list[str|None]) + # - Further in this case, try: + # - 1. negative prompt in this OmniTextPrompt (typed dict) + # - 2. fallback negative prompt from diffusers_call_kwargs (single str or the i-th item in list[str]) + # - 3. empty string "" + # - Otherwise, `kwargs_should_contain_negative_prompt=False`. Do not add "negative_prompt" key to input_kwargs. + has_negative_prompt = any(isinstance(p, dict) and p.get("negative_prompt") is not None for p in prompt_obj) + fallback_negative_prompt = self.od_config.diffusers_call_kwargs.get("negative_prompt") + kwargs_should_contain_negative_prompt = has_negative_prompt or fallback_negative_prompt is not None + if kwargs_should_contain_negative_prompt: + input_kwargs["negative_prompt"] = [] + + for i, prompt in enumerate(prompt_obj): + this_fallback_negative_prompt = "" + if isinstance(fallback_negative_prompt, str): + this_fallback_negative_prompt = fallback_negative_prompt + elif isinstance(fallback_negative_prompt, list): + try: + this_fallback_negative_prompt = fallback_negative_prompt[i] + except IndexError: + raise ValueError( + "The fallback negative_prompt in diffusers_call_kwargs is a list, but its length " + f"({len(fallback_negative_prompt)}) is less than the number of prompts ({len(prompt_obj)}). " + "Please provide a list with the same length as the number of prompts." + ) - prompts = [] - negative_prompts: list[str] | None = [] - for prompt in prompt_obj: if isinstance(prompt, str): - prompts.append(prompt) + input_kwargs["prompt"].append(prompt) + if kwargs_should_contain_negative_prompt: + input_kwargs["negative_prompt"].append(this_fallback_negative_prompt) + for key in multi_modal_data_fields: + input_kwargs[key].append(None) else: - prompts.append(prompt.get("prompt", "")) - negative_prompts.append(prompt.get("negative_prompt", "")) - if all(not np for np in negative_prompts): - negative_prompts = None - return prompts, negative_prompts + obj = cast(OmniTextPrompt, prompt) + input_kwargs["prompt"].append(obj.get("prompt", "")) - @staticmethod - def _extract_negative_prompt(prompt_obj: Any) -> str | None: - """Extract the negative prompt from a prompt object, if present.""" - if isinstance(prompt_obj, dict): - return prompt_obj.get("negative_prompt") - return getattr(prompt_obj, "negative_prompt", None) + if kwargs_should_contain_negative_prompt: + negative_prompt: str = obj.get("negative_prompt", this_fallback_negative_prompt) + input_kwargs["negative_prompt"].append(negative_prompt) + + multi_modal_data = obj.get("multi_modal_data") or {} + for key in multi_modal_data_fields: + input_kwargs[key].append(multi_modal_data.get(key)) + return input_kwargs def _wrap_output(self, output: Any) -> DiffusionOutput: """Convert diffusers pipeline output to ``DiffusionOutput``. @@ -356,3 +451,30 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin): return DiffusionOutput(output=output.audios) return DiffusionOutput(output=output) + + @staticmethod + def _summarize_call_kwargs_value(value: Any) -> Any: + """Return a sanitized summary of diffusers call kwargs for logging.""" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, str): + if len(value) < 20: + return value + return f"{value[:10]}...{value[-10:]}" + if isinstance(value, torch.Tensor): + return f"Tensor with shape {tuple(value.shape)}, dtype {value.dtype}, device {value.device}" + if isinstance(value, torch.Generator): + return f"Generator with seed {value.initial_seed()} on device {value.device}" + if isinstance(value, dict): + return {k: DiffusersAdapterPipeline._summarize_call_kwargs_value(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set)): + if len(value) > 10: + return f"{value.__class__.__name__} of length {len(value)}" + return value.__class__([DiffusersAdapterPipeline._summarize_call_kwargs_value(v) for v in value]) + shape = getattr(value, "shape", None) + size = getattr(value, "size", None) + if shape is not None: + return {"type": type(value).__name__, "shape": tuple(shape)} + if size is not None and not callable(size): + return {"type": type(value).__name__, "size": size} + return f"<{type(value).__name__}>" diff --git a/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py new file mode 100644 index 000000000..4c810bb68 --- /dev/null +++ b/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +from diffusers.pipelines.pipeline_utils import DiffusionPipeline + +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + + +class BasePipelineUtils: + """No-op hooks for pipeline-specific diffusers adapter behavior.""" + + def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None: + pass + + def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None: + pass + + def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None: + pass + + +class WanPipelineUtils(BasePipelineUtils): + def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None: + if od_config.boundary_ratio is not None: + load_kwargs["boundary_ratio"] = od_config.boundary_ratio + + def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None: + if od_config.flow_shift is not None: + from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler + + pipeline.scheduler = UniPCMultistepScheduler.from_config( + pipeline.scheduler.config, flow_shift=od_config.flow_shift + ) + + def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None: + if sampling.boundary_ratio is not None: + raise ValueError( + "Boundary ratio is not supported at runtime with the diffusers backend for Wan models. Please set " + "it at model loading time using the `boundary_ratio` kwarg or `--diffusers-load-kwargs` JSON." + ) + if sampling.extra_args.get("flow_shift") is not None: + raise ValueError( + "Flow shift is not supported at runtime with the diffusers backend for Wan models. Please set " + "it at model loading time using the `flow_shift` kwarg." + ) + + +PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = { + "WanPipeline": WanPipelineUtils, + "WanImageToVideoPipeline": WanPipelineUtils, + "WanVACEPipeline": WanPipelineUtils, + "WanVideoToVideoPipeline": WanPipelineUtils, + "WanAnimatePipeline": WanPipelineUtils, +} + + +def get_pipeline_utils(pipeline_class_name: str | None) -> BasePipelineUtils: + if pipeline_class_name is None: + return BasePipelineUtils() + pipeline_utils_cls = PIPELINE_UTILS_REGISTRY.get(pipeline_class_name, BasePipelineUtils) + return pipeline_utils_cls()