From 17e27c80e31c7d37e44e9d7de8f066b67b987b13 Mon Sep 17 00:00:00 2001 From: Hyoseop Song Date: Wed, 8 Jul 2026 12:20:30 +0900 Subject: [PATCH] [Test] Add image-to-video offline example docs and tests (#3573) Signed-off-by: loveysuby Signed-off-by: Hyoseop Song Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: loveysuby Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com> --- .buildkite/test-nightly.yml | 38 ++++ .../image_to_video/README.md | 206 +++++++++++++++++- .../offline_inference/test_image_to_video.py | 81 +++++++ 3 files changed, 318 insertions(+), 7 deletions(-) create mode 100644 tests/examples/offline_inference/test_image_to_video.py diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 5eede7ae0..85f078533 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -1287,6 +1287,44 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2V · Doc Test" + timeout_in_minutes: 60 + commands: + - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2V · Other Function Test" timeout_in_minutes: 90 commands: diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index 17977bb7a..c161022a5 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -3,14 +3,140 @@ This shared example generates videos from images with VACE, Wan2.2, LTX-2/LTX-2.3, HunyuanVideo-1.5, Cosmos3, and other compatible pipelines. -## Local CLI Usage +- `image_to_video.py`: command-line script for single video generation with advanced options. -Download the example image: +## Table of Contents + +- [Overview](#overview) +- [Prerequisites](#prerequisites) +- [Quick Start](#quick-start) +- [Key Arguments](#key-arguments) +- [More CLI Examples](#more-cli-examples) +- [Advanced Features](#advanced-features) +- [FAQ](#faq) + +## Overview + +This folder provides a unified CLI script for image-to-video generation using vLLM-Omni diffusion/video pipelines. The script selects practical defaults for supported model families while still exposing common sampling, memory, and parallelism options. + +### Supported Models + +| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM Notes | +| ----- | ------------------ | -------------- | ------------- | -------- | ---------- | +| `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480 x 832 | 81 | 50 | 5.0 | Around 60 GiB BF16 for basic single-card usage | +| `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 480 x 832 | 81 | 50 | 4.0 | Around 20–25 GiB BF16, smallest I2V model | +| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v` | 480 x 832 | 121 | 50 | 6.0 | Around 100 GiB at default settings; the example enables `--enable-cpu-offload` + VAE tiling/slicing to fit an 80 GiB card | +| LTX2 (local path + `--model-class-name LTX2ImageToVideoPipeline`) | 512 x 768 | 121 | 40 | 4.0 | Memory use depends on frame count and tensor parallelism | + +!!! info + Peak VRAM: based on basic single-card usage, batch size = 1, without any acceleration/optimization features. Some model weights cannot fit into one card with 80 GiB VRAM, which may need to use CPU offloading. + +Default model: `Wan-AI/Wan2.2-I2V-A14B-Diffusers`. + +## Prerequisites + +Download the example image used in the snippets below: ```bash -wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg +curl -L -o cherry_blossom.jpg https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg ``` +## Quick Start + +### Python API + +Single-prompt generation using TI2V-5B (lightweight): + +```python +import PIL.Image +import torch + +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +if __name__ == "__main__": + image = PIL.Image.open("cherry_blossom.jpg").convert("RGB") + image = image.resize((576, 320)) + + omni = Omni( + model="Wan-AI/Wan2.2-TI2V-5B-Diffusers", + flow_shift=12.0, + ) + outputs = omni.generate( + { + "prompt": "Cherry blossoms swaying gently in the breeze, petals falling", + "multi_modal_data": {"image": image}, + }, + OmniDiffusionSamplingParams( + height=320, + width=576, + num_frames=17, + num_inference_steps=20, + guidance_scale=4.0, + generator=torch.Generator(device="cuda").manual_seed(42), + ), + ) + from diffusers.utils import export_to_video + frames = outputs[0].request_output.images + export_to_video(frames, "quick_test_i2v.mp4", fps=16) +``` + +### Local CLI Usage + +Quick test using TI2V-5B with a small resolution and few frames: + +```bash +python image_to_video.py \ + --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \ + --image cherry_blossom.jpg \ + --prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \ + --height 320 \ + --width 576 \ + --num-frames 17 \ + --guidance-scale 4.0 \ + --num-inference-steps 20 \ + --flow-shift 12.0 \ + --fps 16 \ + --output quick_test_i2v.mp4 +``` + +## Key Arguments + +| Argument | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `--model` | str | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Diffusers I2V model ID or local path | +| `--model-class-name` | str | `None` | Override model class name (e.g., `LTX2ImageToVideoPipeline`) | +| `--image` | str | (required) | Path to input image | +| `--prompt` | str | `""` | Text description of desired motion/animation | +| `--negative-prompt` | str | `""` | Optional list of artifacts to suppress | +| `--seed` | int | `42` | Random seed for deterministic sampling | +| `--guidance-scale` | float | `5.0` | CFG scale | +| `--guidance-scale-high` | float | `None` | Separate CFG for high-noise stage (MoE only) | +| `--height` | int | auto | Video height (auto-calculated from image if not set). Multiples of 16 | +| `--width` | int | auto | Video width (auto-calculated from image if not set). Multiples of 16 | +| `--num-frames` | int | `81` | Number of frames | +| `--num-inference-steps` | int | `50` | Number of denoising steps | +| `--boundary-ratio` | float | `0.875` | Boundary split ratio for two-stage MoE models | +| `--flow-shift` | float | `5.0` | Scheduler flow shift (5.0 for 720p, 12.0 for 480p) | +| `--sample-solver` | str | `unipc` | Wan2.2 sampling solver (`unipc` or `euler` for Lightning/Distill) | +| `--fps` | int | `None` | Frames per second for the saved MP4 | +| `--frame-rate` | float | `None` | Generation frame rate for pipelines that require it (e.g., LTX2) | +| `--output` | str | `i2v_output.mp4` | Path to save the generated video | +| `--vae-use-slicing` | flag | off | Enable VAE slicing for memory optimization | +| `--vae-use-tiling` | flag | off | Enable VAE tiling for memory optimization | +| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models | +| `--enable-layerwise-offload` | flag | off | Enable layerwise offloading on DiT modules | +| `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel | +| `--tensor-parallel-size` | int | `1` | Tensor parallel size (effective for models that support TP, e.g. LTX2) | +| `--ulysses-degree` | int | `1` | Ulysses sequence parallel degree | +| `--ring-degree` | int | `1` | Ring sequence parallel degree | +| `--cache-backend` | str | `None` | Cache backend: `cache_dit` or `tea_cache` | +| `--use-hsdp` | flag | off | Enable Hybrid Sharded Data Parallel | +| `--hsdp-shard-size` | int | `-1` | GPUs per shard group (-1 auto-calculates) | +| `--hsdp-replicate-size` | int | `1` | Number of replica groups for HSDP | + +## More CLI Examples + ### Wan2.2-I2V-A14B-Diffusers (MoE) ```bash @@ -18,7 +144,7 @@ python image_to_video.py \ --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \ --image cherry_blossom.jpg \ --prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \ - --negative-prompt "" \ + --negative-prompt "low quality, blurry" \ --height 480 \ --width 832 \ --num-frames 48 \ @@ -28,7 +154,7 @@ python image_to_video.py \ --boundary-ratio 0.875 \ --flow-shift 12.0 \ --fps 16 \ - --output i2v_output.mp4 + --output i2v_wan_moe.mp4 ``` ### Wan2.2-TI2V-5B-Diffusers (Unified) @@ -38,7 +164,7 @@ python image_to_video.py \ --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \ --image cherry_blossom.jpg \ --prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \ - --negative-prompt "" \ + --negative-prompt "low quality, blurry" \ --height 480 \ --width 832 \ --num-frames 48 \ @@ -46,7 +172,65 @@ python image_to_video.py \ --num-inference-steps 40 \ --flow-shift 12.0 \ --fps 16 \ - --output i2v_output.mp4 + --output i2v_wan_ti2v.mp4 +``` + +### HunyuanVideo-1.5 I2V (480p) + +```bash +python image_to_video.py \ + --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v \ + --image cherry_blossom.jpg \ + --prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \ + --height 480 \ + --width 832 \ + --num-frames 121 \ + --guidance-scale 6.0 \ + --flow-shift 5.0 \ + --num-inference-steps 50 \ + --fps 24 \ + --enable-cpu-offload \ + --vae-use-tiling \ + --vae-use-slicing \ + --output hunyuan_i2v.mp4 +``` + +### LTX2 Image-to-Video + +```bash +python image_to_video.py \ + --model /path/to/LTX-2 \ + --model-class-name LTX2ImageToVideoPipeline \ + --image cherry_blossom.jpg \ + --prompt "A cinematic dolly shot of cherry blossoms" \ + --height 512 \ + --width 768 \ + --num-frames 121 \ + --num-inference-steps 40 \ + --guidance-scale 4.0 \ + --frame-rate 24 \ + --fps 24 \ + --output ltx2_i2v.mp4 +``` + +## Advanced Features + +### CFG Parallel + +Set `--cfg-parallel-size 2` to enable CFG Parallel for faster inference on multi-GPU setups. +See more examples in the [cfg_parallel user guide](../../../docs/user_guide/diffusion/parallelism/cfg_parallel.md). + +### Cache Acceleration + +Use `--cache-backend cache_dit` for Cache-DiT acceleration or `--cache-backend tea_cache` for Timestep Embedding Aware Cache: + +```bash +python image_to_video.py \ + --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \ + --image cherry_blossom.jpg \ + --prompt "Cherry blossoms swaying gently in the breeze" \ + --cache-backend cache_dit \ + --output i2v_cached.mp4 ``` ### LTX-2.3 @@ -220,3 +404,11 @@ The VACE T2V command is documented in the shared For Wan2.2 LightX2V-converted local Diffusers directories and related LoRA assets, see the [LoRA guide](../../../docs/user_guide/diffusion/lora.md#wan22-lightx2v-offline-assembly). + +## FAQ + +**OOM errors**: Try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage. For very large models, add `--enable-cpu-offload` or `--enable-layerwise-offload`. + +**Auto-calculated resolution**: If `--height` and `--width` are not provided, the script calculates output dimensions from the input image while maintaining aspect ratio and targeting 480 x 832 area (or 512 x 768 for LTX2). + +**Wan2.2 MoE vs unified**: `I2V-A14B` is a larger Mixture-of-Experts model with separate low/high-noise DiT stages (use `--guidance-scale-high` and `--boundary-ratio`). `TI2V-5B` is a smaller unified T2V+I2V model that does not need these extra arguments. diff --git a/tests/examples/offline_inference/test_image_to_video.py b/tests/examples/offline_inference/test_image_to_video.py new file mode 100644 index 000000000..b53648cb3 --- /dev/null +++ b/tests/examples/offline_inference/test_image_to_video.py @@ -0,0 +1,81 @@ +""" +Offline inference tests: image-to-video. +See examples/offline_inference/image_to_video/README.md +""" + +import shutil +from pathlib import Path + +import pytest +import requests + +from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet +from tests.helpers.assertions import assert_video_valid +from tests.helpers.mark import hardware_marks + +pytestmark = [ + pytest.mark.usefixtures("clean_gpu_memory_between_tests"), + pytest.mark.full_model, + pytest.mark.example, + *hardware_marks(res={"cuda": "H100"}), +] + +I2V_SCRIPT = EXAMPLES / "offline_inference" / "image_to_video" / "image_to_video.py" +README_PATH = I2V_SCRIPT.with_name("README.md") +EXAMPLE_OUTPUT_SUBFOLDER = "example_offline_i2v" + +_IMAGE_URL = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg" +_IMAGE_NAME = "cherry_blossom.jpg" + +_SKIP_SECTIONS = { + "Prerequisites", + "Advanced Features", + "FAQ", + # VACE conditional tasks are covered by test_vace_video_generation.py, which + # runs the same README snippets with synthetic assets and smoke settings. The + # prep block here downloads inputs via wget (not portable, not collected), so + # exercising this section from the shared runner would only duplicate and fail. + "Wan2.1 VACE Conditional Tasks", +} + + +def _ensure_test_image(run_dir: Path) -> None: + """Download or copy the example image into run_dir so CLI snippets can find it.""" + dest = run_dir / _IMAGE_NAME + if dest.exists(): + return + src = I2V_SCRIPT.parent / _IMAGE_NAME + if src.exists(): + shutil.copy2(src, dest) + return + response = requests.get(_IMAGE_URL, timeout=60) + response.raise_for_status() + dest.write_bytes(response.content) + + +def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool, str]: + if h2_title in _SKIP_SECTIONS: + return True, f"README section '{h2_title}' is intentionally excluded for examples tests" + if language == "python": + return True, "Python API snippets produce video files that ExampleRunner does not auto-collect" + if "/path/to/" in code: + return True, "Snippet references a placeholder local model path" + return False, "" + + +README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet) + + +@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id) +def test_image_to_video(snippet: ReadmeSnippet, example_runner: ExampleRunner): + should_skip, reason = snippet.skip + if should_skip: + pytest.skip(reason) + + run_dir = example_runner.output_root / EXAMPLE_OUTPUT_SUBFOLDER / snippet.test_id + run_dir.mkdir(parents=True, exist_ok=True) + _ensure_test_image(run_dir) + + result = example_runner.run(snippet, output_subfolder=Path(EXAMPLE_OUTPUT_SUBFOLDER)) + for asset in result.assets: + assert_video_valid(asset)