Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
40284854bb | ||
|
|
30c72afdda | ||
|
|
4b541e4d56 |
No files matched your search
@@ -167,11 +167,9 @@ def main() -> None:
|
||||
]
|
||||
native_cmd = [
|
||||
sys.executable,
|
||||
"examples/offline_inference/text_to_video/text_to_video.py",
|
||||
"examples/offline_inference/text_to_video/text_to_video_lingbot.py",
|
||||
"--model",
|
||||
args.model,
|
||||
"--model-class-name",
|
||||
"LingBotVideoPipeline",
|
||||
"--prompt",
|
||||
args.prompt,
|
||||
"--output",
|
||||
|
||||
@@ -12,7 +12,6 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back
|
||||
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
|
||||
|---|---|---|---|---|---|
|
||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||
| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
|
||||
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||
@@ -36,25 +35,6 @@ python text_to_video.py \
|
||||
--output t2v_out.mp4
|
||||
```
|
||||
|
||||
### LingBot-Video
|
||||
|
||||
The shared runner recognizes both official LingBot checkpoint IDs and selects
|
||||
the native `LingBotVideoPipeline` automatically:
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model robbyant/lingbot-video-dense-1.3b \
|
||||
--prompt "a robotic arm picks up a red block" \
|
||||
--height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
|
||||
--guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
|
||||
--output lingbot_t2v.mp4
|
||||
```
|
||||
|
||||
Use `robbyant/lingbot-video-moe-30b-a3b` for the MoE checkpoint. For local
|
||||
paths without `lingbot` in the directory name, add
|
||||
`--model-class-name LingBotVideoPipeline`. Model-only options use
|
||||
`--extra-body`, for example `'{"batch_cfg": true, "output_type": "np"}'`.
|
||||
|
||||
### LTX-2
|
||||
|
||||
```bash
|
||||
|
||||
@@ -28,6 +28,7 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b
|
||||
- Qwen/Qwen-Image
|
||||
- Tongyi-MAI/Z-Image-Turbo
|
||||
- Wan2.2-I2V-A14B-Diffusers
|
||||
- Lightricks/LTX-2.5-Diffusers (source-preview dependencies)
|
||||
|
||||
If you find that a model not listed above also produces different outputs from running diffusers model directly.
|
||||
Please consider file an issue.
|
||||
@@ -116,6 +117,36 @@ The model loading and inference strictly follows the diffusers library, and they
|
||||
Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
|
||||
Some particular examples are below.
|
||||
|
||||
#### LTX-2.5 (source preview)
|
||||
|
||||
Use the gated Diffusers checkpoint
|
||||
[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers);
|
||||
[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) contains
|
||||
component weights and is not directly loadable by this backend. Accept the
|
||||
model terms, authenticate, and install the source-preview dependencies:
|
||||
|
||||
```bash
|
||||
hf auth login
|
||||
python -m pip install --upgrade 'transformers>=5.10.1' \
|
||||
'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20'
|
||||
```
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2.5-Diffusers \
|
||||
--omni \
|
||||
--diffusion-load-format diffusers \
|
||||
--dtype bfloat16 \
|
||||
--enable-cpu-offload \
|
||||
--vae-use-tiling
|
||||
```
|
||||
|
||||
The converted repository loads the distilled transformer. When no `sigmas`,
|
||||
`timesteps`, or `num_inference_steps` is supplied, the adapter applies the
|
||||
official eight-sigma unguided schedule and returns synchronized 48 kHz audio.
|
||||
Support is limited to one-stage distilled T2V; `transformer_full`, two-stage
|
||||
upsampling, image/video conditioning, and the diffusion video decoder are not
|
||||
supported. The general Diffusers backend [limitations](#limitations) apply.
|
||||
|
||||
#### Wan Series
|
||||
|
||||
The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
|
||||
|
||||
@@ -8,7 +8,6 @@ A unified script for text-to-video generation. Supports multiple models with mod
|
||||
|---|---|---|---|---|---|
|
||||
| `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) |
|
||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||
| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
|
||||
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||
@@ -58,31 +57,18 @@ python text_to_video.py \
|
||||
|
||||
### LingBot-Video
|
||||
|
||||
The shared runner recognizes the official dense and MoE checkpoint IDs, selects
|
||||
`LingBotVideoPipeline`, and builds the canonical video request envelope:
|
||||
The dedicated LingBot example explicitly selects the video output modality:
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
||||
--model robbyant/lingbot-video-dense-1.3b \
|
||||
--prompt "a robotic arm picks up a red block" \
|
||||
--height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
|
||||
--guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
|
||||
--output lingbot_t2v.mp4
|
||||
```
|
||||
|
||||
Use the same command with `robbyant/lingbot-video-moe-30b-a3b` for the MoE
|
||||
checkpoint. Requested frame counts are rounded upward to the causal
|
||||
VAE's `4n+1` grid. For a local checkpoint path whose name does not contain
|
||||
`lingbot`, pass `--model-class-name LingBotVideoPipeline`.
|
||||
|
||||
LingBot-only options such as `batch_cfg` and `output_type` use the shared
|
||||
model-extra channel, for example:
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model robbyant/lingbot-video-dense-1.3b \
|
||||
--extra-body '{"batch_cfg": true, "output_type": "np"}'
|
||||
```
|
||||
VAE's `4n+1` grid.
|
||||
|
||||
### LTX-2
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name, get_output_tensor_range
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -80,17 +80,6 @@ _MODEL_PRESETS = {
|
||||
"fps": 16,
|
||||
"output": "helios_output.mp4",
|
||||
},
|
||||
"lingbot": {
|
||||
"model_class_name": "LingBotVideoPipeline",
|
||||
"height": 192,
|
||||
"width": 320,
|
||||
"num_frames": 9,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 3.0,
|
||||
"fps": 24,
|
||||
"flow_shift": 3.0,
|
||||
"output": "lingbot_video_output.mp4",
|
||||
},
|
||||
"ltx2": {
|
||||
"height": 512,
|
||||
"width": 768,
|
||||
@@ -121,51 +110,27 @@ _MODEL_PRESETS = {
|
||||
def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
|
||||
model_lower = model.lower()
|
||||
class_lower = (model_class_name or "").lower()
|
||||
if "lingbot" in model_lower or "lingbotvideo" in class_lower:
|
||||
return _MODEL_PRESETS["lingbot"]
|
||||
if "ltx" in class_lower or "ltx" in model_lower:
|
||||
if "distilled" in class_lower or "distilled" in model_lower:
|
||||
return _MODEL_PRESETS["ltx2_distilled"]
|
||||
if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
|
||||
return _MODEL_PRESETS["ltx23"]
|
||||
if "distilled" in class_lower or "distilled" in model_lower:
|
||||
return _MODEL_PRESETS["ltx2_distilled"]
|
||||
if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
|
||||
return _MODEL_PRESETS["ltx23"]
|
||||
if "ltx2" in class_lower or "ltx-2" in model_lower or "ltx_2" in model_lower:
|
||||
return _MODEL_PRESETS["ltx2"]
|
||||
if "vace" in model_lower or "vace" in class_lower:
|
||||
if "vace" in model_lower:
|
||||
return _MODEL_PRESETS["vace"]
|
||||
# Edge must be matched before the generic cosmos branch (its "cosmos" substring would
|
||||
# otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset).
|
||||
if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower):
|
||||
return _MODEL_PRESETS["cosmos3_edge"]
|
||||
if "cosmos" in model_lower or "cosmos" in class_lower:
|
||||
if "cosmos" in model_lower:
|
||||
return _MODEL_PRESETS["cosmos"]
|
||||
if "hunyuan" in model_lower or "hunyuan" in class_lower:
|
||||
if "hunyuan" in model_lower:
|
||||
return _MODEL_PRESETS["hunyuan"]
|
||||
if "helios" in model_lower or "helios" in class_lower:
|
||||
if "helios" in model_lower:
|
||||
return _MODEL_PRESETS["helios"]
|
||||
return _MODEL_PRESETS["wan"]
|
||||
|
||||
|
||||
def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
|
||||
"""Build the canonical request envelope for the shared T2V example."""
|
||||
result: dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"modalities": ["video"],
|
||||
}
|
||||
if negative_prompt is not None:
|
||||
result["negative_prompt"] = negative_prompt
|
||||
return result
|
||||
|
||||
|
||||
def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor:
|
||||
"""Normalize decoded frames according to the pipeline's output contract."""
|
||||
if not tensor.is_floating_point():
|
||||
return tensor
|
||||
if source_range == "negative_one_to_one":
|
||||
return tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
if source_range == "zero_to_one":
|
||||
return tensor.clamp(0, 1)
|
||||
raise ValueError(f"Unsupported floating-point tensor range: {source_range!r}")
|
||||
|
||||
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
config = json.loads(value)
|
||||
@@ -195,7 +160,7 @@ def parse_extra_body(value: str) -> dict[str, Any]:
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate a video from a text prompt. "
|
||||
"Supports Wan2.2, HunyuanVideo-1.5, Helios, LingBot-Video, and other text-to-video models."
|
||||
"Supports Wan2.2, HunyuanVideo-1.5, Helios, and other text-to-video models."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
@@ -443,7 +408,6 @@ def main():
|
||||
for key, default_val in preset.items():
|
||||
if getattr(args, key.replace("-", "_"), None) is None:
|
||||
setattr(args, key.replace("-", "_"), default_val)
|
||||
model_class_name = args.model_class_name
|
||||
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
# Cache-dit config (Wan2.2 only)
|
||||
@@ -516,7 +480,6 @@ def main():
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni) or model_class_name
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
output_tensor_range = get_output_tensor_range(model_class_name)
|
||||
|
||||
if profiler_enabled:
|
||||
print("[Profiler] Starting profiling...")
|
||||
@@ -550,11 +513,12 @@ def main():
|
||||
lora_path=lora_path,
|
||||
)
|
||||
|
||||
negative_prompt = args.negative_prompt
|
||||
if negative_prompt is None and all(preset is not _MODEL_PRESETS[name] for name in ("lingbot", "ltx2", "ltx23")):
|
||||
prompt_dict = {"prompt": args.prompt}
|
||||
if args.negative_prompt is not None:
|
||||
prompt_dict["negative_prompt"] = args.negative_prompt
|
||||
elif preset not in (_MODEL_PRESETS["ltx2"], _MODEL_PRESETS["ltx23"]):
|
||||
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
||||
negative_prompt = ""
|
||||
prompt_dict = build_text_to_video_prompt(args.prompt, negative_prompt)
|
||||
prompt_dict["negative_prompt"] = ""
|
||||
|
||||
extra_args = {}
|
||||
if lora_request:
|
||||
@@ -666,7 +630,8 @@ def main():
|
||||
frame_tensor = frame_tensor[0]
|
||||
if frame_tensor.dim() == 3 and frame_tensor.shape[0] in (3, 4):
|
||||
frame_tensor = frame_tensor.permute(1, 2, 0)
|
||||
frame_tensor = _normalize_float_tensor(frame_tensor, output_tensor_range)
|
||||
if frame_tensor.is_floating_point():
|
||||
frame_tensor = frame_tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
return frame_tensor.float().numpy()
|
||||
if isinstance(frame, np.ndarray):
|
||||
frame_array = frame
|
||||
@@ -718,7 +683,8 @@ def main():
|
||||
video_tensor = video_tensor[0]
|
||||
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
|
||||
video_tensor = video_tensor.permute(1, 2, 3, 0)
|
||||
video_tensor = _normalize_float_tensor(video_tensor, output_tensor_range)
|
||||
if video_tensor.is_floating_point():
|
||||
video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
video_array = video_tensor.float().numpy()
|
||||
elif isinstance(frames, np.ndarray):
|
||||
video_array = frames
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate text-to-video with a LingBot-Video checkpoint.")
|
||||
parser.add_argument("--model", default="robbyant/lingbot-video-dense-1.3b")
|
||||
parser.add_argument("--prompt", default="a robotic arm picks up a red block")
|
||||
parser.add_argument("--negative-prompt", default=None)
|
||||
parser.add_argument("--output", default="lingbot_video_output.mp4")
|
||||
parser.add_argument("--height", type=int, default=192)
|
||||
parser.add_argument("--width", type=int, default=320)
|
||||
parser.add_argument("--num-frames", type=int, default=9)
|
||||
parser.add_argument("--num-inference-steps", type=int, default=2)
|
||||
parser.add_argument("--guidance-scale", type=float, default=3.0)
|
||||
parser.add_argument("--flow-shift", type=float, default=3.0)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--fps", type=int, default=24)
|
||||
parser.add_argument("--output-type", choices=["pt", "np"], default="pt")
|
||||
parser.add_argument("--batch-cfg", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _peak_memory_mb(output: Any) -> float:
|
||||
result = output[0] if isinstance(output, list) and output else output
|
||||
return float(getattr(result, "peak_memory_mb", 0.0) or 0.0)
|
||||
|
||||
|
||||
def _extract_video(output: Any) -> Any:
|
||||
if isinstance(output, list):
|
||||
output = output[0] if output else None
|
||||
if isinstance(output, OmniRequestOutput):
|
||||
if output.final_output_type != "image":
|
||||
raise ValueError(f"Unexpected output type {output.final_output_type!r}.")
|
||||
if not output.images:
|
||||
raise ValueError("No video frames found in OmniRequestOutput.")
|
||||
return output.images[0] if len(output.images) == 1 else output.images
|
||||
return output
|
||||
|
||||
|
||||
def _to_frame_array(video: Any) -> np.ndarray:
|
||||
if isinstance(video, torch.Tensor):
|
||||
tensor = video.detach().cpu()
|
||||
if tensor.dim() == 5:
|
||||
tensor = tensor[0]
|
||||
if tensor.dim() == 4 and tensor.shape[-1] not in (3, 4):
|
||||
if tensor.shape[0] in (3, 4):
|
||||
tensor = tensor.permute(1, 2, 3, 0)
|
||||
elif tensor.shape[1] in (3, 4):
|
||||
tensor = tensor.permute(0, 2, 3, 1)
|
||||
if tensor.is_floating_point():
|
||||
if float(tensor.min()) < 0.0:
|
||||
tensor = tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
else:
|
||||
tensor = tensor.clamp(0, 1)
|
||||
return tensor.float().numpy()
|
||||
if isinstance(video, np.ndarray):
|
||||
array = video[0] if video.ndim == 5 else video
|
||||
if np.issubdtype(array.dtype, np.integer):
|
||||
return array.astype(np.float32) / 255.0
|
||||
return array.astype(np.float32, copy=False)
|
||||
if isinstance(video, list):
|
||||
return np.stack([_to_frame_array(frame) for frame in video], axis=0)
|
||||
return np.asarray(video, dtype=np.float32)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
model_class_name="LingBotVideoPipeline",
|
||||
flow_shift=args.flow_shift,
|
||||
parallel_config=DiffusionParallelConfig(),
|
||||
)
|
||||
|
||||
prompt: dict[str, Any] = {
|
||||
"prompt": args.prompt,
|
||||
"modalities": ["video"],
|
||||
"num_frames": args.num_frames,
|
||||
}
|
||||
if args.negative_prompt is not None:
|
||||
prompt["negative_prompt"] = args.negative_prompt
|
||||
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_frames=args.num_frames,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
guidance_scale=args.guidance_scale,
|
||||
generator=generator,
|
||||
output_type=args.output_type,
|
||||
extra_args={"batch_cfg": args.batch_cfg, "shift": args.flow_shift},
|
||||
)
|
||||
|
||||
start = time.perf_counter()
|
||||
output = omni.generate(prompt, sampling_params)
|
||||
elapsed = time.perf_counter() - start
|
||||
print(f"Total generation time: {elapsed:.4f} seconds ({elapsed * 1000:.2f} ms)")
|
||||
peak_mb = _peak_memory_mb(output)
|
||||
if peak_mb:
|
||||
print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
|
||||
|
||||
video = _to_frame_array(_extract_video(output))
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
from diffusers.utils import export_to_video
|
||||
|
||||
export_to_video(list(video), str(output_path), fps=args.fps)
|
||||
print(f"Saved generated video to {output_path}")
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -25,6 +25,7 @@ However, as we strive to ensure output similarity between vLLM-Omni's diffuser b
|
||||
- Qwen/Qwen-Image
|
||||
- Tongyi-MAI/Z-Image-Turbo
|
||||
- Wan2.2-I2V-A14B-Diffusers
|
||||
- Lightricks/LTX-2.5-Diffusers (source-preview dependencies)
|
||||
|
||||
If you find that a model not listed above also produces different outputs from running diffusers model directly.
|
||||
Please consider file an issue or submit a PR to fix.
|
||||
@@ -118,6 +119,36 @@ The model loading and inference strictly follows the diffusers library, and they
|
||||
Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
|
||||
Some particular examples are below.
|
||||
|
||||
#### LTX-2.5 (source preview)
|
||||
|
||||
Use the gated Diffusers checkpoint
|
||||
[`Lightricks/LTX-2.5-Diffusers`](https://huggingface.co/Lightricks/LTX-2.5-Diffusers);
|
||||
[`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5) contains
|
||||
component weights and is not directly loadable by this backend. Accept the
|
||||
model terms, authenticate, and install the source-preview dependencies:
|
||||
|
||||
```bash
|
||||
hf auth login
|
||||
python -m pip install --upgrade 'transformers>=5.10.1' \
|
||||
'diffusers @ git+https://github.com/huggingface/diffusers.git@7564fb016dabda0c943416190fc92398c50b1b20'
|
||||
```
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2.5-Diffusers \
|
||||
--omni \
|
||||
--diffusion-load-format diffusers \
|
||||
--dtype bfloat16 \
|
||||
--enable-cpu-offload \
|
||||
--vae-use-tiling
|
||||
```
|
||||
|
||||
The converted repository loads the distilled transformer. When no `sigmas`,
|
||||
`timesteps`, or `num_inference_steps` is supplied, the adapter applies the
|
||||
official eight-sigma unguided schedule and returns synchronized 48 kHz audio.
|
||||
Support is limited to one-stage distilled T2V; `transformer_full`, two-stage
|
||||
upsampling, image/video conditioning, and the diffusion video decoder are not
|
||||
supported. The general Diffusers backend [limitations](#limitations) apply.
|
||||
|
||||
#### Wan Series
|
||||
|
||||
The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
|
||||
|
||||
@@ -27,7 +27,7 @@ parallel, cache, quantized, or expert-kernel backends.
|
||||
- Dense checkpoint: <https://huggingface.co/robbyant/lingbot-video-dense-1.3b>
|
||||
- MoE checkpoint: <https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b>
|
||||
- Upstream project: <https://github.com/Robbyant/lingbot-video>
|
||||
- Related offline example: [`examples/offline_inference/text_to_video/text_to_video.py`](../../examples/offline_inference/text_to_video/text_to_video.py)
|
||||
- Related offline example: [`examples/offline_inference/text_to_video/text_to_video_lingbot.py`](../../examples/offline_inference/text_to_video/text_to_video_lingbot.py)
|
||||
- Related online video API docs: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md)
|
||||
|
||||
## Hardware Support
|
||||
@@ -52,7 +52,7 @@ The MoE path is validated with BF16 expert weights.
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
python examples/offline_inference/text_to_video/text_to_video.py \
|
||||
python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
||||
--model robbyant/lingbot-video-dense-1.3b \
|
||||
--prompt "a robotic arm picks up a red block" \
|
||||
--output lingbot_t2v.mp4 \
|
||||
@@ -70,7 +70,7 @@ python examples/offline_inference/text_to_video/text_to_video.py \
|
||||
|
||||
```bash
|
||||
CUDA_VISIBLE_DEVICES=0 \
|
||||
python examples/offline_inference/text_to_video/text_to_video.py \
|
||||
python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
||||
--model robbyant/lingbot-video-moe-30b-a3b \
|
||||
--prompt "a robotic arm picks up a red block" \
|
||||
--output lingbot_moe_t2v.mp4 \
|
||||
|
||||
@@ -17,6 +17,10 @@ from vllm_omni.diffusion.data import (
|
||||
OmniDiffusionConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.models.diffusers_adapter import DiffusersAdapterPipeline
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
|
||||
LTX25_DISTILLED_SIGMAS,
|
||||
get_pipeline_utils,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
@@ -180,6 +184,32 @@ class TestPipelineArgumentsHandling:
|
||||
with pytest.raises(NotImplementedError):
|
||||
DiffusersAdapterPipeline(od_config=od_config)
|
||||
|
||||
def test_ltx25_original_repo_rejected_before_loading(self, mocker):
|
||||
mock_from_pretrained = mocker.patch(
|
||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained"
|
||||
)
|
||||
adapter = DiffusersAdapterPipeline(od_config=_make_od_config(model="Lightricks/LTX-2.5"))
|
||||
|
||||
with pytest.raises(ValueError, match="Lightricks/LTX-2.5-Diffusers"):
|
||||
adapter.load_weights()
|
||||
|
||||
mock_from_pretrained.assert_not_called()
|
||||
|
||||
def test_ltx25_requires_upstream_diffusers_feature(self, mocker):
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils._supports_ltx25_diffusers",
|
||||
return_value=False,
|
||||
)
|
||||
mock_from_pretrained = mocker.patch(
|
||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained"
|
||||
)
|
||||
adapter = DiffusersAdapterPipeline(od_config=_make_od_config(model="Lightricks/LTX-2.5-Diffusers"))
|
||||
|
||||
with pytest.raises(ImportError, match="7564fb016dabda0c943416190fc92398c50b1b20"):
|
||||
adapter.load_weights()
|
||||
|
||||
mock_from_pretrained.assert_not_called()
|
||||
|
||||
def test_adapter_load_weights_injects_supported_quantization_config(self, mocker):
|
||||
(
|
||||
FakePipelineQuantizationConfig,
|
||||
@@ -374,6 +404,28 @@ class TestPipelineArgumentsHandling:
|
||||
assert isinstance(output, DiffusionOutput)
|
||||
assert output.output == raw_output
|
||||
|
||||
def test_ltx25_preserves_video_and_singular_audio_output(self, tmp_path):
|
||||
frames = torch.zeros(1, 2, 3, 4, 3)
|
||||
audio = torch.zeros(1, 48000)
|
||||
(tmp_path / "model_index.json").write_text(
|
||||
'{"_class_name":"LTX2Pipeline","duration_head":["diffusers","LTX2DurationHead"]}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
od_config = _make_od_config(model=str(tmp_path))
|
||||
adapter = DiffusersAdapterPipeline(od_config=od_config)
|
||||
adapter._pipeline = SimpleNamespace(vocoder=SimpleNamespace(config=SimpleNamespace(output_sampling_rate=48000)))
|
||||
adapter._pipeline_utils = get_pipeline_utils("LTX2Pipeline")
|
||||
|
||||
output = adapter._wrap_output(SimpleNamespace(frames=frames, audio=audio))
|
||||
|
||||
assert isinstance(output.output, dict)
|
||||
assert output.output["payload"]["video"] is frames
|
||||
assert output.output["payload"]["audio"] is audio
|
||||
assert output.output["metadata"] == {"audio": {"sample_rate": 48000}}
|
||||
|
||||
frames_only = adapter._wrap_output(SimpleNamespace(frames=frames, audio=None))
|
||||
assert frames_only.output is frames
|
||||
|
||||
def test_adapter_build_call_kwargs(self, mocker):
|
||||
class MockPipeline:
|
||||
def __call__(
|
||||
@@ -629,6 +681,97 @@ class TestPipelineArgumentsHandling:
|
||||
with pytest.raises(ValueError):
|
||||
pipeline.forward(DiffusionRequestBatch(requests=[problematic_request]))
|
||||
|
||||
def test_ltx25_uses_distilled_sampling_defaults_from_renamed_local_directory(self, mocker, tmp_path):
|
||||
class LTX2Pipeline:
|
||||
def __call__(
|
||||
self,
|
||||
prompt=None,
|
||||
num_inference_steps=None,
|
||||
sigmas=None,
|
||||
timesteps=None,
|
||||
guidance_scale=None,
|
||||
audio_guidance_scale=None,
|
||||
stg_scale=None,
|
||||
audio_stg_scale=None,
|
||||
modality_scale=None,
|
||||
audio_modality_scale=None,
|
||||
guidance_rescale=None,
|
||||
audio_guidance_rescale=None,
|
||||
num_frames=None,
|
||||
frame_rate=None,
|
||||
output_type=None,
|
||||
generator=None,
|
||||
):
|
||||
return None
|
||||
|
||||
def to(self, device):
|
||||
return self
|
||||
|
||||
# Local model directories do not retain their Hub repository name.
|
||||
# Detect 2.5 from its pipeline component metadata instead.
|
||||
(tmp_path / "model_index.json").write_text(
|
||||
'{"_class_name":"LTX2Pipeline","duration_head":["diffusers","LTX2DurationHead"]}',
|
||||
encoding="utf-8",
|
||||
)
|
||||
enable_tiling = mocker.Mock()
|
||||
loaded_pipeline = LTX2Pipeline()
|
||||
loaded_pipeline.vae = SimpleNamespace(enable_tiling=enable_tiling)
|
||||
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils._supports_ltx25_diffusers",
|
||||
return_value=True,
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.models.diffusers_adapter.pipeline_diffusers_adapter.DiffusionPipeline.from_pretrained",
|
||||
return_value=loaded_pipeline,
|
||||
)
|
||||
adapter = DiffusersAdapterPipeline(
|
||||
od_config=_make_od_config(
|
||||
model=str(tmp_path),
|
||||
vae_use_tiling=True,
|
||||
)
|
||||
)
|
||||
adapter.load_weights()
|
||||
enable_tiling.assert_called_once_with()
|
||||
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
num_frames=121,
|
||||
frame_rate=24.0,
|
||||
seed=42,
|
||||
generator_device="cpu",
|
||||
)
|
||||
kwargs = adapter._build_call_kwargs(DiffusionRequestBatch(requests=[_make_request(sampling_params=sampling)]))
|
||||
|
||||
assert kwargs["sigmas"] == LTX25_DISTILLED_SIGMAS
|
||||
assert kwargs["num_inference_steps"] == 8
|
||||
assert kwargs["guidance_scale"] == 1.0
|
||||
assert kwargs["audio_guidance_scale"] == 1.0
|
||||
assert kwargs["stg_scale"] == 0.0
|
||||
assert kwargs["audio_stg_scale"] == 0.0
|
||||
assert kwargs["modality_scale"] == 1.0
|
||||
assert kwargs["audio_modality_scale"] == 1.0
|
||||
assert kwargs["guidance_rescale"] == 0.0
|
||||
assert kwargs["audio_guidance_rescale"] == 0.0
|
||||
|
||||
explicit_steps = OmniDiffusionSamplingParams(num_inference_steps=12, generator_device="cpu")
|
||||
step_kwargs = adapter._build_call_kwargs(
|
||||
DiffusionRequestBatch(requests=[_make_request(sampling_params=explicit_steps)])
|
||||
)
|
||||
assert step_kwargs["num_inference_steps"] == 12
|
||||
assert "sigmas" not in step_kwargs
|
||||
|
||||
custom_sigmas = [1.0, 0.5]
|
||||
explicit_schedule = OmniDiffusionSamplingParams(
|
||||
num_inference_steps=2,
|
||||
sigmas=custom_sigmas,
|
||||
generator_device="cpu",
|
||||
)
|
||||
schedule_kwargs = adapter._build_call_kwargs(
|
||||
DiffusionRequestBatch(requests=[_make_request(sampling_params=explicit_schedule)])
|
||||
)
|
||||
assert schedule_kwargs["num_inference_steps"] == 2
|
||||
assert schedule_kwargs["sigmas"] is custom_sigmas
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from examples.offline_inference.text_to_video.text_to_video import (
|
||||
_MODEL_PRESETS,
|
||||
_detect_preset,
|
||||
_normalize_float_tensor,
|
||||
build_text_to_video_prompt,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("negative_prompt", "expected_negative_prompt"),
|
||||
[(None, None), ("", ""), ("blurry", "blurry")],
|
||||
)
|
||||
def test_text_to_video_builds_canonical_prompt(
|
||||
negative_prompt: str | None,
|
||||
expected_negative_prompt: str | None,
|
||||
) -> None:
|
||||
result = build_text_to_video_prompt("a robot waves", negative_prompt)
|
||||
|
||||
assert result["prompt"] == "a robot waves"
|
||||
assert result["modalities"] == ["video"]
|
||||
if expected_negative_prompt is None:
|
||||
assert "negative_prompt" not in result
|
||||
else:
|
||||
assert result["negative_prompt"] == expected_negative_prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "model_class_name", "preset_name"),
|
||||
[
|
||||
("robbyant/lingbot-video-dense-1.3b", None, "lingbot"),
|
||||
("/models/custom-checkpoint", "LingBotVideoPipeline", "lingbot"),
|
||||
("Lightricks/LTX-2", "LTX2DistilledPipeline", "ltx2_distilled"),
|
||||
("BestWishYsh/Helios-Distilled", None, "helios"),
|
||||
("/models/cosmos-edge", "Cosmos3EdgeVFMTransformer", "cosmos3_edge"),
|
||||
("/models/vace", "WanVACEPipeline", "vace"),
|
||||
],
|
||||
)
|
||||
def test_detect_preset_is_scoped_to_model_family(
|
||||
model: str,
|
||||
model_class_name: str | None,
|
||||
preset_name: str,
|
||||
) -> None:
|
||||
assert _detect_preset(model, model_class_name) is _MODEL_PRESETS[preset_name]
|
||||
|
||||
|
||||
def test_lingbot_preset_matches_lingbot_defaults() -> None:
|
||||
assert _MODEL_PRESETS["lingbot"] == {
|
||||
"model_class_name": "LingBotVideoPipeline",
|
||||
"height": 192,
|
||||
"width": 320,
|
||||
"num_frames": 9,
|
||||
"num_inference_steps": 2,
|
||||
"guidance_scale": 3.0,
|
||||
"fps": 24,
|
||||
"flow_shift": 3.0,
|
||||
"output": "lingbot_video_output.mp4",
|
||||
}
|
||||
|
||||
|
||||
def test_normalize_float_tensor_preserves_zero_to_one_frames() -> None:
|
||||
frames = torch.tensor([0.0, 0.25, 1.0, 1.5])
|
||||
|
||||
assert torch.equal(_normalize_float_tensor(frames, "zero_to_one"), torch.tensor([0.0, 0.25, 1.0, 1.0]))
|
||||
|
||||
|
||||
def test_normalize_float_tensor_maps_negative_one_to_one_frames() -> None:
|
||||
frames = torch.tensor([-1.5, -1.0, 0.0, 1.0, 1.5])
|
||||
|
||||
assert torch.equal(
|
||||
_normalize_float_tensor(frames, "negative_one_to_one"),
|
||||
torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0]),
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_float_tensor_uses_one_contract_for_ambiguous_frames() -> None:
|
||||
frames = [torch.tensor([0.0, 0.5, 1.0]), torch.tensor([-1.0, 0.0, 1.0])]
|
||||
|
||||
normalized = [_normalize_float_tensor(frame, "negative_one_to_one") for frame in frames]
|
||||
|
||||
assert torch.equal(normalized[0], torch.tensor([0.5, 0.75, 1.0]))
|
||||
assert torch.equal(normalized[1], torch.tensor([0.0, 0.5, 1.0]))
|
||||
@@ -17,7 +17,6 @@ from vllm_omni.model_extras import (
|
||||
build_x_to_text_prompt,
|
||||
get_extra_body_params,
|
||||
get_extra_output_params,
|
||||
get_output_tensor_range,
|
||||
get_x_to_text_model_family,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
)
|
||||
@@ -135,8 +134,6 @@ def test_lingbot_extra_registry_declares_request_params() -> None:
|
||||
}
|
||||
)
|
||||
assert get_extra_output_params("LingBotVideoPipeline") == frozenset()
|
||||
assert get_output_tensor_range("LingBotVideoPipeline") == "zero_to_one"
|
||||
assert get_output_tensor_range("WanPipeline") == "negative_one_to_one"
|
||||
assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False
|
||||
|
||||
|
||||
|
||||
@@ -23,7 +23,11 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import BasePipelineUtils, get_pipeline_utils
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.pipeline_utils import (
|
||||
BasePipelineUtils,
|
||||
get_pipeline_utils,
|
||||
validate_model_compatibility,
|
||||
)
|
||||
from vllm_omni.diffusion.models.diffusers_adapter.quantization_utils import (
|
||||
apply_diffusers_quantization_config,
|
||||
convert_diffusers_quantization_config,
|
||||
@@ -110,6 +114,7 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
|
||||
model_id = self.od_config.model
|
||||
dtype = self.od_config.dtype
|
||||
validate_model_compatibility(self.od_config)
|
||||
|
||||
load_kwargs = {
|
||||
"torch_dtype": dtype,
|
||||
@@ -130,6 +135,8 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
logger.debug(f"Loading diffusers pipeline with kwargs: {load_kwargs}")
|
||||
|
||||
self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs)
|
||||
if pipeline_class_name is None:
|
||||
self._pipeline_utils = get_pipeline_utils(self._pipeline.__class__.__name__)
|
||||
self._pipeline_utils.apply_post_load_updates(self._pipeline, self.od_config)
|
||||
|
||||
self._pipeline.to(self.device)
|
||||
@@ -146,14 +153,14 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
# VAE slicing and tiling: try-catch because not all models have VAE
|
||||
if self.od_config.vae_use_slicing:
|
||||
try:
|
||||
self._pipeline.enable_vae_slicing()
|
||||
self._pipeline_utils.enable_vae_optimization(self._pipeline, "slicing")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to enable VAE slicing for diffusers pipeline {self._pipeline.__class__.__name__}: {e}"
|
||||
)
|
||||
if self.od_config.vae_use_tiling:
|
||||
try:
|
||||
self._pipeline.enable_vae_tiling()
|
||||
self._pipeline_utils.enable_vae_optimization(self._pipeline, "tiling")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to enable VAE tiling for diffusers pipeline {self._pipeline.__class__.__name__}: {e}"
|
||||
@@ -393,6 +400,8 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
else:
|
||||
kwargs["generator"] = torch.Generator(device=sampling.generator_device)
|
||||
|
||||
self._pipeline_utils.update_call_kwargs(self.od_config, kwargs)
|
||||
|
||||
logger.info(
|
||||
"Calling diffusers pipeline with kwargs: %s", DiffusersAdapterPipeline._summarize_call_kwargs_value(kwargs)
|
||||
)
|
||||
@@ -497,6 +506,8 @@ class DiffusersAdapterPipeline(nn.Module, DiffusionPipelineProfilerMixin):
|
||||
"""
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
|
||||
output = self._pipeline_utils.normalize_output(self._pipeline, self.od_config, output)
|
||||
|
||||
if hasattr(output, "images"):
|
||||
# Preserve diffusers image format (`output_type`)
|
||||
return DiffusionOutput(output=output.images)
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
@@ -8,6 +12,74 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LTX25_ORIGINAL_MODEL_ID = "Lightricks/LTX-2.5"
|
||||
LTX25_DIFFUSERS_MODEL_ID = "Lightricks/LTX-2.5-Diffusers"
|
||||
LTX25_DIFFUSERS_COMMIT = "7564fb016dabda0c943416190fc92398c50b1b20"
|
||||
|
||||
# Diffusers' public LTX-2.5 schedule intentionally excludes the scheduler's
|
||||
# terminal zero. FlowMatchEulerDiscreteScheduler appends that internally.
|
||||
LTX25_DISTILLED_SIGMAS = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875]
|
||||
|
||||
_LTX25_DISTILLED_GUIDANCE_DEFAULTS: dict[str, float] = {
|
||||
"guidance_scale": 1.0,
|
||||
"audio_guidance_scale": 1.0,
|
||||
"stg_scale": 0.0,
|
||||
"audio_stg_scale": 0.0,
|
||||
"modality_scale": 1.0,
|
||||
"audio_modality_scale": 1.0,
|
||||
"guidance_rescale": 0.0,
|
||||
"audio_guidance_rescale": 0.0,
|
||||
}
|
||||
|
||||
|
||||
def _supports_ltx25_diffusers() -> bool:
|
||||
"""Check the standard-pipeline capabilities added with LTX-2.5."""
|
||||
try:
|
||||
import diffusers
|
||||
import transformers
|
||||
|
||||
pipeline_class = diffusers.LTX2Pipeline
|
||||
init_parameters = inspect.signature(pipeline_class.__init__).parameters
|
||||
return (
|
||||
"duration_head" in init_parameters
|
||||
and "prompt_enhancer" in init_parameters
|
||||
and hasattr(transformers, "Gemma4UnifiedForConditionalGeneration")
|
||||
)
|
||||
except (AttributeError, ImportError, RuntimeError):
|
||||
return False
|
||||
|
||||
|
||||
def is_ltx25_diffusers_model(od_config: OmniDiffusionConfig) -> bool:
|
||||
"""Return whether config targets the official converted LTX-2.5 repo."""
|
||||
normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower()
|
||||
if normalized_model == LTX25_DIFFUSERS_MODEL_ID.lower():
|
||||
return True
|
||||
|
||||
# A downloaded checkpoint may be served from any directory name. The
|
||||
# duration-head component is the pipeline-level marker introduced for
|
||||
# LTX-2.5; older LTX-2.0/2.3 indexes do not contain it.
|
||||
model_path = Path(str(od_config.model)).expanduser()
|
||||
subfolder = od_config.diffusers_load_kwargs.get("subfolder")
|
||||
if subfolder is not None:
|
||||
model_path /= str(subfolder)
|
||||
try:
|
||||
with (model_path / "model_index.json").open(encoding="utf-8") as model_index_file:
|
||||
model_index = json.load(model_index_file)
|
||||
except (OSError, TypeError, ValueError):
|
||||
return False
|
||||
if not isinstance(model_index, dict):
|
||||
return False
|
||||
|
||||
duration_head = model_index.get("duration_head")
|
||||
return (
|
||||
model_index.get("_class_name") == "LTX2Pipeline"
|
||||
and isinstance(duration_head, list)
|
||||
and bool(duration_head)
|
||||
and duration_head[0] is not None
|
||||
)
|
||||
|
||||
|
||||
class BasePipelineUtils:
|
||||
"""No-op hooks for pipeline-specific diffusers adapter behavior."""
|
||||
@@ -18,9 +90,112 @@ class BasePipelineUtils:
|
||||
def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None:
|
||||
pass
|
||||
|
||||
def enable_vae_optimization(self, pipeline: DiffusionPipeline, optimization: str) -> None:
|
||||
getattr(pipeline, f"enable_vae_{optimization}")()
|
||||
|
||||
def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
|
||||
pass
|
||||
|
||||
def update_call_kwargs(
|
||||
self,
|
||||
od_config: OmniDiffusionConfig,
|
||||
call_kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
def normalize_output(
|
||||
self,
|
||||
pipeline: DiffusionPipeline,
|
||||
od_config: OmniDiffusionConfig,
|
||||
output: Any,
|
||||
) -> Any:
|
||||
return output
|
||||
|
||||
|
||||
def validate_model_compatibility(od_config: OmniDiffusionConfig) -> None:
|
||||
"""Reject known model/dependency combinations before loading weights."""
|
||||
normalized_model = str(od_config.model).rstrip("/\\").replace("\\", "/").lower()
|
||||
if normalized_model == LTX25_ORIGINAL_MODEL_ID.lower():
|
||||
raise ValueError(
|
||||
f"{LTX25_ORIGINAL_MODEL_ID} is a component-weight repository without a Diffusers "
|
||||
f"model_index.json. Use the official converted repository {LTX25_DIFFUSERS_MODEL_ID} "
|
||||
"with --diffusion-load-format diffusers."
|
||||
)
|
||||
|
||||
if is_ltx25_diffusers_model(od_config) and not _supports_ltx25_diffusers():
|
||||
raise ImportError(
|
||||
f"{LTX25_DIFFUSERS_MODEL_ID} requires unreleased LTX-2.5 support from Diffusers and "
|
||||
"Gemma 4 Unified support from Transformers. Install the source-preview dependencies with "
|
||||
"`python -m pip install --upgrade 'transformers>=5.10.1' "
|
||||
f"'diffusers @ git+https://github.com/huggingface/diffusers.git@{LTX25_DIFFUSERS_COMMIT}'`."
|
||||
)
|
||||
|
||||
|
||||
class LTX2PipelineUtils(BasePipelineUtils):
|
||||
"""Diffusers call policy for the official default LTX-2.5 checkpoint."""
|
||||
|
||||
def enable_vae_optimization(self, pipeline: DiffusionPipeline, optimization: str) -> None:
|
||||
pipeline_method = getattr(pipeline, f"enable_vae_{optimization}", None)
|
||||
if callable(pipeline_method):
|
||||
pipeline_method()
|
||||
else:
|
||||
getattr(pipeline.vae, f"enable_{optimization}")()
|
||||
|
||||
def update_call_kwargs(
|
||||
self,
|
||||
od_config: OmniDiffusionConfig,
|
||||
call_kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
if not is_ltx25_diffusers_model(od_config):
|
||||
return
|
||||
|
||||
has_sigmas = call_kwargs.get("sigmas") is not None
|
||||
has_timesteps = call_kwargs.get("timesteps") is not None
|
||||
has_step_count = call_kwargs.get("num_inference_steps") is not None
|
||||
if not has_sigmas and not has_timesteps:
|
||||
if has_step_count:
|
||||
logger.warning(
|
||||
"LTX-2.5 distilled inference received num_inference_steps without sigmas. "
|
||||
"Diffusers will use a generic schedule; omit num_inference_steps to use the "
|
||||
"official distilled schedule."
|
||||
)
|
||||
else:
|
||||
call_kwargs["sigmas"] = list(LTX25_DISTILLED_SIGMAS)
|
||||
call_kwargs["num_inference_steps"] = len(LTX25_DISTILLED_SIGMAS)
|
||||
|
||||
for key, value in _LTX25_DISTILLED_GUIDANCE_DEFAULTS.items():
|
||||
if call_kwargs.get(key) is None:
|
||||
call_kwargs[key] = value
|
||||
|
||||
def normalize_output(
|
||||
self,
|
||||
pipeline: DiffusionPipeline,
|
||||
od_config: OmniDiffusionConfig,
|
||||
output: Any,
|
||||
) -> Any:
|
||||
if not is_ltx25_diffusers_model(od_config):
|
||||
return output
|
||||
|
||||
frames = getattr(output, "frames", None)
|
||||
audio = getattr(output, "audio", None)
|
||||
if frames is None or audio is None:
|
||||
return output
|
||||
|
||||
envelope: dict[str, Any] = {"payload": {"video": frames, "audio": audio}}
|
||||
config = getattr(getattr(pipeline, "vocoder", None), "config", None)
|
||||
value = (
|
||||
config.get("output_sampling_rate")
|
||||
if hasattr(config, "get")
|
||||
else getattr(config, "output_sampling_rate", None)
|
||||
)
|
||||
try:
|
||||
sample_rate = None if isinstance(value, bool) else int(value)
|
||||
except (TypeError, ValueError):
|
||||
sample_rate = None
|
||||
if sample_rate is not None and sample_rate > 0:
|
||||
envelope["metadata"] = {"audio": {"sample_rate": sample_rate}}
|
||||
return envelope
|
||||
|
||||
|
||||
class WanPipelineUtils(BasePipelineUtils):
|
||||
def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
|
||||
@@ -49,6 +224,7 @@ class WanPipelineUtils(BasePipelineUtils):
|
||||
|
||||
|
||||
PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = {
|
||||
"LTX2Pipeline": LTX2PipelineUtils,
|
||||
"WanPipeline": WanPipelineUtils,
|
||||
"WanImageToVideoPipeline": WanPipelineUtils,
|
||||
"WanVACEPipeline": WanPipelineUtils,
|
||||
|
||||
@@ -9,7 +9,6 @@ from vllm_omni.model_extras.registry import (
|
||||
get_extra_body_params,
|
||||
get_extra_output_params,
|
||||
get_model_class_name,
|
||||
get_output_tensor_range,
|
||||
get_x_to_text_model_family,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
)
|
||||
@@ -22,7 +21,6 @@ __all__ = [
|
||||
"get_extra_body_params",
|
||||
"get_extra_output_params",
|
||||
"get_model_class_name",
|
||||
"get_output_tensor_range",
|
||||
"get_x_to_text_model_family",
|
||||
"should_init_extra_args_for_non_diffusion_stages",
|
||||
]
|
||||
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any, Literal
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -93,7 +93,6 @@ ImageToVideoPromptBuilder = Callable[
|
||||
dict[str, Any],
|
||||
]
|
||||
XToTextPromptBuilder = Callable[[str, str, bool], tuple[dict[str, Any], list[int] | None]]
|
||||
OutputTensorRange = Literal["negative_one_to_one", "zero_to_one"]
|
||||
|
||||
|
||||
def default_x_to_text_prompt(
|
||||
@@ -194,7 +193,6 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = {
|
||||
},
|
||||
"LingBotVideoPipeline": {
|
||||
"extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS,
|
||||
"output_tensor_range": "zero_to_one",
|
||||
# Shared T2I/I2V envelopes select the output modality. LingBot's
|
||||
# pipeline owns model-specific validation and normalization.
|
||||
},
|
||||
@@ -264,18 +262,6 @@ def get_extra_output_params(model_class_name: str | None) -> frozenset[str]:
|
||||
return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset()
|
||||
|
||||
|
||||
def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange:
|
||||
"""Return the declared range for floating-point tensor outputs.
|
||||
|
||||
The default preserves the shared examples' historical handling. Pipelines
|
||||
that already return normalized tensors declare ``zero_to_one`` explicitly.
|
||||
"""
|
||||
spec = _get_spec(model_class_name)
|
||||
if spec is None:
|
||||
return "negative_one_to_one"
|
||||
return spec.get("output_tensor_range", "negative_one_to_one")
|
||||
|
||||
|
||||
def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None) -> bool:
|
||||
spec = _get_spec(model_class_name)
|
||||
return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False))
|
||||
|
||||
Reference in new issue
Block a user