From beb00fce31f93533ac9a790423d953c220151ff0 Mon Sep 17 00:00:00 2001 From: blonde <164845202+blondeCS@users.noreply.github.com> Date: Thu, 28 May 2026 14:45:45 +0800 Subject: [PATCH] [Doc] Refine vace offline inference example README (#3584) Signed-off-by: blondeCS <1580754914@qq.com> Co-authored-by: Claude Opus 4.6 --- .../vace/vace_video_generation.md | 124 +++++++++++++----- .../test_vace_video_generation.py | 35 +++++ 2 files changed, 127 insertions(+), 32 deletions(-) create mode 100644 tests/examples/offline_inference/test_vace_video_generation.py diff --git a/examples/offline_inference/vace/vace_video_generation.md b/examples/offline_inference/vace/vace_video_generation.md index bbaf99452..8d8b00e48 100644 --- a/examples/offline_inference/vace/vace_video_generation.md +++ b/examples/offline_inference/vace/vace_video_generation.md @@ -1,24 +1,105 @@ # VACE Video Generation -[VACE](https://github.com/ali-vilab/VACE) (Video All-in-one Creation Engine) supports multiple video tasks through a single model. +Generate videos from text prompts, images, or video conditions using vLLM-Omni's VACE diffusion pipeline. -| Model | Architecture | Model Weights (bf16) | HuggingFace | -|-------|-------------|----------------------|-------------| -| Wan2.1-VACE (1.3B) | Wan2.1 | ~10 GB | [Wan-AI/Wan2.1-VACE-1.3B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) | -| Wan2.1-VACE (14B) | Wan2.1 | ~38 GB | [Wan-AI/Wan2.1-VACE-14B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) | +- `vace_video_generation.py`: command-line script for multi-mode video generation with advanced options. -## Text-to-Video (T2V) +## Table of Contents + +- [Overview](#overview) +- [Quick Start](#quick-start) +- [Key Arguments](#key-arguments) +- [More CLI Examples](#more-cli-examples) + +## Overview + +[VACE](https://github.com/ali-vilab/VACE) (Video All-in-one Creation Engine) supports multiple video tasks through a single unified model, including text-to-video, image-to-video, first-last-frame interpolation, inpainting, and reference image-guided generation. + +### Supported Models + +| Model | Architecture | Peak VRAM (GiB) * | Model Weights (GiB) | HuggingFace | +|-------|-------------|-------------------|---------------------|-------------| +| Wan2.1-VACE (1.3B) | Wan2.1 | TBD | ~10 | [Wan-AI/Wan2.1-VACE-1.3B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) | +| Wan2.1-VACE (14B) | Wan2.1 | TBD | ~38 | [Wan-AI/Wan2.1-VACE-14B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) | + +!!! info +*Peak VRAM: based on basic single-card usage, 480x832 resolution, 81 frames, 30 inference steps, without any acceleration/optimization features. + +Default model: `Wan-AI/Wan2.1-VACE-14B-diffusers` + +## Quick Start + +### Python API + +Text-to-video generation: + +```python +import torch + +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +if __name__ == "__main__": + omni = Omni(model="Wan-AI/Wan2.1-VACE-1.3B-diffusers") + prompt = "A sleek robot stands in a vast warehouse filled with boxes" + outputs = omni.generate( + prompt, + OmniDiffusionSamplingParams( + height=480, + width=832, + num_frames=81, + num_inference_steps=30, + guidance_scale=5.0, + ), + ) + video = outputs[0].images + + from diffusers.utils import export_to_video + export_to_video(list(video[0]), "t2v_output.mp4", fps=16) + omni.close() +``` + +### Local CLI Usage ```bash python vace_video_generation.py \ --mode t2v \ --prompt "A sleek robot stands in a vast warehouse filled with boxes" \ --height 480 --width 832 --num-frames 81 \ - --num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \ --output t2v_output.mp4 ``` -## Image-to-Video (I2V) +## Key Arguments + +| Argument | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `--mode` | str | `"t2v"` | VACE task mode: `t2v`, `i2v`, `v2lf`, `flf2v`, `inpaint`, `r2v` | +| `--model` | str | `"Wan-AI/Wan2.1-VACE-14B-diffusers"` | Model ID or local path | +| `--prompt` | str | `"A cat walking in a garden"` | Text description of desired video | +| `--negative-prompt` | str | `""` | Negative prompt for classifier-free guidance | +| `--image` | str | `None` | Input image path (for I2V, R2V, FLF2V, inpaint modes) | +| `--last-image` | str | `None` | Last frame image path (for FLF2V mode) | +| `--height` | int | `480` | Output video height in pixels (should be a multiple of 16) | +| `--width` | int | `832` | Output video width in pixels (should be a multiple of 16) | +| `--num-frames` | int | `81` | Number of video frames to generate | +| `--num-inference-steps` | int | `30` | Number of denoising steps (more steps = higher quality, slower) | +| `--guidance-scale` | float | `5.0` | Classifier-free guidance scale | +| `--flow-shift` | float | `5.0` | Scheduler flow shift parameter | +| `--seed` | int | `42` | Random seed for deterministic sampling | +| `--fps` | int | `16` | Frames per second for the saved MP4 | +| `--output` | str | `"vace_output.mp4"` | Path to save the generated video | +| `--vae-use-tiling` | flag | on | Enable VAE tiling for memory optimization | +| `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel | +| `--ulysses-degree` | int | `1` | Ulysses sequence parallel degree for multi-GPU inference | +| `--ring-degree` | int | `1` | Ring sequence parallel degree for hybrid Ulysses + Ring inference | +| `--tensor-parallel-size` | int | — | Tensor parallel size | +| `--enforce-eager` | flag | off | Disable torch.compile | + +> If you encounter OOM errors, try `--vae-use-tiling` or multi-GPU parallelism options (`--ulysses-degree`, `--cfg-parallel-size`). + +## More CLI Examples + +### Image-to-Video (I2V) First frame is kept, remaining frames are generated: @@ -31,7 +112,7 @@ python vace_video_generation.py \ --output i2v_output.mp4 ``` -## First-Last-Frame Interpolation (FLF2V) +### First-Last-Frame Interpolation (FLF2V) ```bash python vace_video_generation.py \ @@ -42,7 +123,7 @@ python vace_video_generation.py \ --output flf2v_output.mp4 ``` -## Inpainting +### Inpainting Center vertical stripe is masked and regenerated: @@ -55,7 +136,7 @@ python vace_video_generation.py \ --output inpaint_output.mp4 ``` -## Reference Image-guided (R2V) +### Reference Image-guided (R2V) ```bash python vace_video_generation.py \ @@ -65,24 +146,3 @@ python vace_video_generation.py \ --height 480 --width 832 --num-frames 81 \ --output r2v_output.mp4 ``` - -## Key Arguments - -- `--mode`: VACE task mode (`t2v`, `i2v`, `flf2v`, `inpaint`, `r2v`). -- `--model`: Model ID (default: `Wan-AI/Wan2.1-VACE-1.3B-diffusers`). -- `--image`: Input image for I2V, inpainting, and R2V modes. -- `--last-image`: Last frame image for FLF2V mode. -- `--prompt`: Text description of desired video. -- `--height/--width`: Output resolution (default 480x832). Dimensions should be multiples of 16. -- `--num-frames`: Number of frames (default 81). -- `--guidance-scale`: CFG scale (default 5.0). -- `--flow-shift`: Scheduler flow shift (default 5.0). -- `--num-inference-steps`: Number of denoising steps (default 30). -- `--fps`: Frames per second for the saved MP4 (default 16). -- `--output`: Path to save the generated video. -- `--vae-use-tiling`: Enable VAE tiling for memory optimization. -- `--ulysses-degree`: Ulysses sequence parallelism degree for multi-GPU. -- `--cfg-parallel-size`: CFG parallel size for multi-GPU. -- `--tensor-parallel-size`: Tensor parallel size. - -> If you encounter OOM errors, try `--vae-use-tiling` or multi-GPU parallelism options. diff --git a/tests/examples/offline_inference/test_vace_video_generation.py b/tests/examples/offline_inference/test_vace_video_generation.py new file mode 100644 index 000000000..11746e58a --- /dev/null +++ b/tests/examples/offline_inference/test_vace_video_generation.py @@ -0,0 +1,35 @@ +""" +Offline inference tests: vace video generation. +See examples/offline_inference/vace/vace_video_generation.md +""" + +from pathlib import Path + +import pytest + +from tests.examples.helpers import EXAMPLES, ExampleRunner, ReadmeSnippet +from tests.helpers.assertions import assert_video_valid +from tests.helpers.mark import hardware_marks + +pytestmark = [ + pytest.mark.usefixtures("clean_gpu_memory_between_tests"), + pytest.mark.full_model, + pytest.mark.example, + *hardware_marks(res={"cuda": "H100"}), +] + +VACE_SCRIPT = EXAMPLES / "offline_inference" / "vace" / "vace_video_generation.py" +README_PATH = VACE_SCRIPT.with_name("vace_video_generation.md") +EXAMPLE_OUTPUT_SUBFOLDER = "example_offline_vace" + +README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH) + + +@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id) +def test_vace_video_generation(snippet: ReadmeSnippet, example_runner: ExampleRunner): + should_skip, reason = snippet.skip + if should_skip: + pytest.skip(reason) + result = example_runner.run(snippet, output_subfolder=Path(EXAMPLE_OUTPUT_SUBFOLDER)) + for asset in result.assets: + assert_video_valid(asset)