From d1fef41266a3625675e780f8955b812ea556d50a Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Sat, 11 Apr 2026 16:50:03 +0800 Subject: [PATCH] [Doc] Add LTX-2 online serving deployment recipes with optimization benchmarks (#1971) Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> --- .../examples/online_serving/text_to_video.md | 108 +++++++++++++++++- .../online_serving/text_to_video/README.md | 98 +++++++++++++++- .../text_to_video/run_curl_ltx2.sh | 66 +++++++++++ .../text_to_video/run_server_ltx2.sh | 84 ++++++++++++++ 4 files changed, 348 insertions(+), 8 deletions(-) create mode 100644 examples/online_serving/text_to_video/run_curl_ltx2.sh create mode 100644 examples/online_serving/text_to_video/run_server_ltx2.sh diff --git a/docs/user_guide/examples/online_serving/text_to_video.md b/docs/user_guide/examples/online_serving/text_to_video.md index d58296fcc..01e6d9d46 100644 --- a/docs/user_guide/examples/online_serving/text_to_video.md +++ b/docs/user_guide/examples/online_serving/text_to_video.md @@ -3,17 +3,28 @@ Source . -This example demonstrates how to deploy the Wan2.2 text-to-video model for online video generation using vLLM-Omni. +This example demonstrates how to deploy text-to-video models for online video generation using vLLM-Omni. -## Start Server +## Supported Models -### Basic Start +| Model | Model ID | +|-------|----------| +| Wan2.1 T2V (1.3B) | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | +| Wan2.1 T2V (14B) | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | +| Wan2.2 T2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | +| LTX-2 | `Lightricks/LTX-2` | + +## Wan2.2 T2V + +### Start Server + +#### Basic Start ```bash vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091 ``` -### Start with Parameters +#### Start with Parameters Or use the startup script: @@ -234,8 +245,94 @@ while true; do done ``` +## LTX-2 + +### Start Server + +#### Basic Start + +```bash +vllm serve Lightricks/LTX-2 --omni --port 8098 \ + --enforce-eager --flow-shift 1.0 --boundary-ratio 1.0 +``` + +#### Start with Optimization Presets + +Use the LTX-2 startup script with built-in optimization presets: + +```bash +# Baseline (1 GPU, eager) +bash run_server_ltx2.sh baseline + +# 4-GPU Ulysses sequence parallelism (lossless) +bash run_server_ltx2.sh ulysses4 + +# Cache-DiT lossy acceleration (1 GPU, ~1.4× speedup) +bash run_server_ltx2.sh cache-dit + +# Best combo: 4-GPU Ulysses SP + Cache-DiT (~2.2× speedup) +bash run_server_ltx2.sh best-combo +``` + +#### Optimization Benchmarks + +Benchmarked on H800, online serving (480×768, 41 frames, 20 steps, `seed=42`). +"Inference" is the server-reported inference time; excludes HTTP/poll overhead. + +| Preset | Server Command | Inference (s) | Speedup | Type | +|--------|---------------|---------------|---------|------| +| `baseline` | `--enforce-eager` | 10.3 | 1.00× | — | +| `compile` | *(default, no --enforce-eager)* | ~10.3 (warm) | ~1.00× | Lossless | +| `ulysses4` | `--enforce-eager --usp 4` | ~10.3 | ~1.00× | Lossless | +| `cache-dit` | `--enforce-eager --cache-backend cache_dit` | 7.4 avg | ~1.4× | Lossy | +| `best-combo` | `--enforce-eager --usp 4 --cache-backend cache_dit` | 4.7 avg | **~2.2×** | Lossless + Lossy | + +**Observations**: +- **torch.compile**: On H800, warm-request inference time matches the eager baseline (~10.3s). + The first request pays ~6s compilation overhead. Benefit depends on model architecture and GPU. +- **Ulysses SP (4 GPU)**: No measurable speedup alone for 41-frame generation at this resolution. + Communication overhead outweighs gains at this sequence length. +- **Cache-DiT**: Inference varies per request (6–10s) due to dynamic caching decisions. + Average is ~7.4s (~1.4× speedup) with slight quality tradeoff. +- **Best combo**: 4-GPU Ulysses SP + Cache-DiT synergize well — Cache-DiT reduces per-step + computation, making the communication overhead of Ulysses SP worthwhile. Average ~4.7s + (~2.2× speedup). +- **FP8 quantization**: Reduces VRAM but does not speed up LTX-2 on H800 (compute-bound). + +**Deployment Recommendations**: +- For **production with quality priority**: use `baseline` with `--enforce-eager` +- For **maximum throughput** (4 GPUs, quality tradeoff): use `best-combo` (~2.2× speedup) +- For **single-GPU throughput**: use `cache-dit` (~1.4× speedup) +- `--enforce-eager` is recommended to avoid torch.compile warmup latency on first request + +### Send Requests (curl) + +```bash +# Using the provided script +bash run_curl_ltx2.sh + +# Or directly +curl -sS -X POST http://localhost:8098/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A serene lakeside sunrise with mist over the water." \ + -F "width=768" \ + -F "height=480" \ + -F "num_frames=41" \ + -F "fps=24" \ + -F "num_inference_steps=20" \ + -F "guidance_scale=3.0" \ + -F "seed=42" +``` + ## Example materials +??? abstract "response.json" + ``````json + --8<-- "examples/online_serving/text_to_video/response.json" + `````` +??? abstract "run_curl_ltx2.sh" + ``````sh + --8<-- "examples/online_serving/text_to_video/run_curl_ltx2.sh" ??? abstract "run_curl_hunyuan_video_15.sh" ``````sh --8<-- "examples/online_serving/text_to_video/run_curl_hunyuan_video_15.sh" @@ -248,6 +345,9 @@ done ``````sh --8<-- "examples/online_serving/text_to_video/run_server.sh" `````` +??? abstract "run_server_ltx2.sh" + ``````sh + --8<-- "examples/online_serving/text_to_video/run_server_ltx2.sh" ??? abstract "run_server_hunyuan_video_15.sh" ``````sh --8<-- "examples/online_serving/text_to_video/run_server_hunyuan_video_15.sh" diff --git a/examples/online_serving/text_to_video/README.md b/examples/online_serving/text_to_video/README.md index 44e676671..c01e0602f 100644 --- a/examples/online_serving/text_to_video/README.md +++ b/examples/online_serving/text_to_video/README.md @@ -1,16 +1,27 @@ # Text-To-Video -This example demonstrates how to deploy the Wan2.2 text-to-video model for online video generation using vLLM-Omni. +This example demonstrates how to deploy text-to-video models for online video generation using vLLM-Omni. -## Start Server +## Supported Models -### Basic Start +| Model | Model ID | +|-------|----------| +| Wan2.1 T2V (1.3B) | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` | +| Wan2.1 T2V (14B) | `Wan-AI/Wan2.1-T2V-14B-Diffusers` | +| Wan2.2 T2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | +| LTX-2 | `Lightricks/LTX-2` | + +## Wan2.2 T2V + +### Start Server + +#### Basic Start ```bash vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091 ``` -### Start with Parameters +#### Start with Parameters Or use the startup script: @@ -230,3 +241,82 @@ while true; do sleep 2 done ``` + +## LTX-2 + +### Start Server + +#### Basic Start + +```bash +vllm serve Lightricks/LTX-2 --omni --port 8098 \ + --enforce-eager --flow-shift 1.0 --boundary-ratio 1.0 +``` + +#### Start with Optimization Presets + +Use the LTX-2 startup script with built-in optimization presets: + +```bash +# Baseline (1 GPU, eager) +bash run_server_ltx2.sh baseline + +# 4-GPU Ulysses sequence parallelism (lossless) +bash run_server_ltx2.sh ulysses4 + +# Cache-DiT lossy acceleration (1 GPU, ~1.4× speedup) +bash run_server_ltx2.sh cache-dit + +# Best combo: 4-GPU Ulysses SP + Cache-DiT (~2.2× speedup) +bash run_server_ltx2.sh best-combo +``` + +#### Optimization Benchmarks + +Benchmarked on H800, online serving (480×768, 41 frames, 20 steps, `seed=42`). +"Inference" is the server-reported inference time; excludes HTTP/poll overhead. + +| Preset | Server Command | Inference (s) | Speedup | Type | +|--------|---------------|---------------|---------|------| +| `baseline` | `--enforce-eager` | 10.3 | 1.00× | — | +| `compile` | *(default, no --enforce-eager)* | ~10.3 (warm) | ~1.00× | Lossless | +| `ulysses4` | `--enforce-eager --usp 4` | ~10.3 | ~1.00× | Lossless | +| `cache-dit` | `--enforce-eager --cache-backend cache_dit` | 7.4 avg | ~1.4× | Lossy | +| `best-combo` | `--enforce-eager --usp 4 --cache-backend cache_dit` | 4.7 avg | **~2.2×** | Lossless + Lossy | + +**Observations**: +- **torch.compile**: On H800, warm-request inference time matches the eager baseline (~10.3s). + The first request pays ~6s compilation overhead. Benefit depends on model architecture and GPU. +- **Ulysses SP (4 GPU)**: No measurable speedup alone for 41-frame generation at this resolution. + Communication overhead outweighs gains at this sequence length. +- **Cache-DiT**: Inference varies per request (6–10s) due to dynamic caching decisions. + Average is ~7.4s (~1.4× speedup) with slight quality tradeoff. +- **Best combo**: 4-GPU Ulysses SP + Cache-DiT synergize well — Cache-DiT reduces per-step + computation, making the communication overhead of Ulysses SP worthwhile. Average ~4.7s + (~2.2× speedup). +- **FP8 quantization**: Reduces VRAM but does not speed up LTX-2 on H800 (compute-bound). + +**Deployment Recommendations**: +- For **production with quality priority**: use `baseline` with `--enforce-eager` +- For **maximum throughput** (4 GPUs, quality tradeoff): use `best-combo` (~2.2× speedup) +- For **single-GPU throughput**: use `cache-dit` (~1.4× speedup) +- `--enforce-eager` is recommended to avoid torch.compile warmup latency on first request + +### Send Requests (curl) + +```bash +# Using the provided script +bash run_curl_ltx2.sh + +# Or directly +curl -sS -X POST http://localhost:8098/v1/videos \ + -H "Accept: application/json" \ + -F "prompt=A serene lakeside sunrise with mist over the water." \ + -F "width=768" \ + -F "height=480" \ + -F "num_frames=41" \ + -F "fps=24" \ + -F "num_inference_steps=20" \ + -F "guidance_scale=3.0" \ + -F "seed=42" +``` diff --git a/examples/online_serving/text_to_video/run_curl_ltx2.sh b/examples/online_serving/text_to_video/run_curl_ltx2.sh new file mode 100644 index 000000000..b82f672ea --- /dev/null +++ b/examples/online_serving/text_to_video/run_curl_ltx2.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# LTX-2 text-to-video curl example using the async video job API. +# Start the server first: bash run_server_ltx2.sh best-combo + +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8098}" +OUTPUT_PATH="${OUTPUT_PATH:-ltx2_output.mp4}" +POLL_INTERVAL="${POLL_INTERVAL:-2}" + +PROMPT="${PROMPT:-A serene lakeside sunrise with mist over the water.}" + +create_response=$( + curl -sS -X POST "${BASE_URL}/v1/videos" \ + -H "Accept: application/json" \ + -F "prompt=${PROMPT}" \ + -F "width=768" \ + -F "height=480" \ + -F "num_frames=41" \ + -F "fps=24" \ + -F "num_inference_steps=20" \ + -F "guidance_scale=3.0" \ + -F "seed=42" +) + +video_id="$(echo "${create_response}" | jq -r '.id')" +if [ -z "${video_id}" ] || [ "${video_id}" = "null" ]; then + echo "Failed to create video job:" + echo "${create_response}" | jq . + exit 1 +fi + +echo "Created video job ${video_id}" +echo "${create_response}" | jq . + +while true; do + status_response="$(curl -sS "${BASE_URL}/v1/videos/${video_id}")" + status="$(echo "${status_response}" | jq -r '.status')" + + case "${status}" in + queued|in_progress) + echo "Video job ${video_id} status: ${status}" + sleep "${POLL_INTERVAL}" + ;; + completed) + echo "${status_response}" | jq . + break + ;; + failed) + echo "Video generation failed:" + echo "${status_response}" | jq . + exit 1 + ;; + *) + echo "Unexpected status response:" + echo "${status_response}" | jq . + exit 1 + ;; + esac +done + +curl -sS -L "${BASE_URL}/v1/videos/${video_id}/content" -o "${OUTPUT_PATH}" +echo "Saved video to ${OUTPUT_PATH}" diff --git a/examples/online_serving/text_to_video/run_server_ltx2.sh b/examples/online_serving/text_to_video/run_server_ltx2.sh new file mode 100644 index 000000000..f4597d3cd --- /dev/null +++ b/examples/online_serving/text_to_video/run_server_ltx2.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# LTX-2 online serving startup script with optimization presets. +# +# Usage: +# bash run_server_ltx2.sh # baseline (1 GPU, eager) +# bash run_server_ltx2.sh ulysses4 # 4-GPU Ulysses SP +# bash run_server_ltx2.sh cache-dit # 1 GPU + Cache-DiT +# bash run_server_ltx2.sh best-combo # 4-GPU Ulysses SP + Cache-DiT +# +# Online serving benchmarks on H800 (480×768, 41 frames, 20 steps): +# baseline : 10.3s inference (1.00×) +# compile : ~10.3s warm (~1.00×) first request +6s warmup +# ulysses4 : ~10.3s (~1.00×) no gain at 41 frames +# cache-dit : 7.4s avg (~1.4×) lossy, variable per request +# best-combo : 4.7s avg (~2.2×) 4-GPU ulysses + cache-dit + +set -euo pipefail + +MODEL="${MODEL:-Lightricks/LTX-2}" +PORT="${PORT:-8098}" +FLOW_SHIFT="${FLOW_SHIFT:-1.0}" +BOUNDARY_RATIO="${BOUNDARY_RATIO:-1.0}" + +PRESET="${1:-baseline}" + +EXTRA_ARGS=() +case "$PRESET" in + baseline) + echo "=== LTX-2 Preset: baseline (1 GPU, enforce-eager) ===" + EXTRA_ARGS+=(--enforce-eager) + ;; + ulysses2) + echo "=== LTX-2 Preset: 2-GPU Ulysses SP (lossless) ===" + EXTRA_ARGS+=(--enforce-eager --usp 2) + ;; + ulysses4) + echo "=== LTX-2 Preset: 4-GPU Ulysses SP (lossless) ===" + EXTRA_ARGS+=(--enforce-eager --usp 4) + ;; + cache-dit) + echo "=== LTX-2 Preset: Cache-DiT (1 GPU, lossy) ===" + EXTRA_ARGS+=(--enforce-eager --cache-backend cache_dit) + ;; + best-combo) + echo "=== LTX-2 Preset: 4-GPU Ulysses SP + Cache-DiT (best combo) ===" + EXTRA_ARGS+=(--enforce-eager --usp 4 --cache-backend cache_dit) + ;; + compile) + echo "=== LTX-2 Preset: torch.compile (1 GPU, lossless) ===" + # torch.compile is the default (no --enforce-eager) + ;; + *) + echo "Usage: $0 {baseline|ulysses2|ulysses4|cache-dit|best-combo|compile}" + echo "" + echo "Presets:" + echo " baseline - 1 GPU, eager execution (reference)" + echo " ulysses2 - 2-GPU Ulysses SP (lossless)" + echo " ulysses4 - 4-GPU Ulysses SP (lossless)" + echo " cache-dit - 1 GPU + Cache-DiT (lossy, ~1.4× speedup)" + echo " best-combo - 4-GPU Ulysses SP + Cache-DiT (~2.2× speedup)" + echo " compile - 1 GPU + torch.compile (slower first request)" + echo "" + echo "Environment variables:" + echo " MODEL - Model path (default: Lightricks/LTX-2)" + echo " PORT - Server port (default: 8098)" + echo " FLOW_SHIFT - Scheduler flow shift (default: 1.0)" + echo " BOUNDARY_RATIO - Boundary ratio (default: 1.0)" + exit 1 + ;; +esac + +echo "Model: $MODEL" +echo "Port: $PORT" +echo "Flow shift: $FLOW_SHIFT" +echo "Boundary ratio: $BOUNDARY_RATIO" + +vllm serve "$MODEL" --omni \ + --port "$PORT" \ + --flow-shift "$FLOW_SHIFT" \ + --boundary-ratio "$BOUNDARY_RATIO" \ + "${EXTRA_ARGS[@]}"