[Misc] Consolidate LingBot text-to-video runner (#6076)
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
c57178c0e1
commit
8b8d3f56db
10 files changed
+208
-159
No files matched your search
@@ -167,9 +167,11 @@ def main() -> None:
|
|||||||
]
|
]
|
||||||
native_cmd = [
|
native_cmd = [
|
||||||
sys.executable,
|
sys.executable,
|
||||||
"examples/offline_inference/text_to_video/text_to_video_lingbot.py",
|
"examples/offline_inference/text_to_video/text_to_video.py",
|
||||||
"--model",
|
"--model",
|
||||||
args.model,
|
args.model,
|
||||||
|
"--model-class-name",
|
||||||
|
"LingBotVideoPipeline",
|
||||||
"--prompt",
|
"--prompt",
|
||||||
args.prompt,
|
args.prompt,
|
||||||
"--output",
|
"--output",
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back
|
|||||||
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
|
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
|
||||||
|---|---|---|---|---|---|
|
|---|---|---|---|---|---|
|
||||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||||
|
| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
|
||||||
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
||||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||||
@@ -35,6 +36,25 @@ python text_to_video.py \
|
|||||||
--output t2v_out.mp4
|
--output t2v_out.mp4
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### LingBot-Video
|
||||||
|
|
||||||
|
The shared runner recognizes both official LingBot checkpoint IDs and selects
|
||||||
|
the native `LingBotVideoPipeline` automatically:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python text_to_video.py \
|
||||||
|
--model robbyant/lingbot-video-dense-1.3b \
|
||||||
|
--prompt "a robotic arm picks up a red block" \
|
||||||
|
--height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
|
||||||
|
--guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
|
||||||
|
--output lingbot_t2v.mp4
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `robbyant/lingbot-video-moe-30b-a3b` for the MoE checkpoint. For local
|
||||||
|
paths without `lingbot` in the directory name, add
|
||||||
|
`--model-class-name LingBotVideoPipeline`. Model-only options use
|
||||||
|
`--extra-body`, for example `'{"batch_cfg": true, "output_type": "np"}'`.
|
||||||
|
|
||||||
### LTX-2
|
### LTX-2
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ A unified script for text-to-video generation. Supports multiple models with mod
|
|||||||
|---|---|---|---|---|---|
|
|---|---|---|---|---|---|
|
||||||
| `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) |
|
| `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) |
|
||||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||||
|
| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
|
||||||
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
||||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||||
@@ -57,18 +58,31 @@ python text_to_video.py \
|
|||||||
|
|
||||||
### LingBot-Video
|
### LingBot-Video
|
||||||
|
|
||||||
The dedicated LingBot example explicitly selects the video output modality:
|
The shared runner recognizes the official dense and MoE checkpoint IDs, selects
|
||||||
|
`LingBotVideoPipeline`, and builds the canonical video request envelope:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
python text_to_video.py \
|
||||||
--model robbyant/lingbot-video-dense-1.3b \
|
--model robbyant/lingbot-video-dense-1.3b \
|
||||||
|
--prompt "a robotic arm picks up a red block" \
|
||||||
--height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
|
--height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
|
||||||
|
--guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
|
||||||
--output lingbot_t2v.mp4
|
--output lingbot_t2v.mp4
|
||||||
```
|
```
|
||||||
|
|
||||||
Use the same command with `robbyant/lingbot-video-moe-30b-a3b` for the MoE
|
Use the same command with `robbyant/lingbot-video-moe-30b-a3b` for the MoE
|
||||||
checkpoint. Requested frame counts are rounded upward to the causal
|
checkpoint. Requested frame counts are rounded upward to the causal
|
||||||
VAE's `4n+1` grid.
|
VAE's `4n+1` grid. For a local checkpoint path whose name does not contain
|
||||||
|
`lingbot`, pass `--model-class-name LingBotVideoPipeline`.
|
||||||
|
|
||||||
|
LingBot-only options such as `batch_cfg` and `output_type` use the shared
|
||||||
|
model-extra channel, for example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python text_to_video.py \
|
||||||
|
--model robbyant/lingbot-video-dense-1.3b \
|
||||||
|
--extra-body '{"batch_cfg": true, "output_type": "np"}'
|
||||||
|
```
|
||||||
|
|
||||||
### LTX-2
|
### LTX-2
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from vllm_omni.entrypoints.omni import Omni
|
|||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
from vllm_omni.lora.request import LoRARequest
|
from vllm_omni.lora.request import LoRARequest
|
||||||
from vllm_omni.lora.utils import stable_lora_int_id
|
from vllm_omni.lora.utils import stable_lora_int_id
|
||||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name, get_output_tensor_range
|
||||||
from vllm_omni.outputs import OmniRequestOutput
|
from vllm_omni.outputs import OmniRequestOutput
|
||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
@@ -80,6 +80,17 @@ _MODEL_PRESETS = {
|
|||||||
"fps": 16,
|
"fps": 16,
|
||||||
"output": "helios_output.mp4",
|
"output": "helios_output.mp4",
|
||||||
},
|
},
|
||||||
|
"lingbot": {
|
||||||
|
"model_class_name": "LingBotVideoPipeline",
|
||||||
|
"height": 192,
|
||||||
|
"width": 320,
|
||||||
|
"num_frames": 9,
|
||||||
|
"num_inference_steps": 2,
|
||||||
|
"guidance_scale": 3.0,
|
||||||
|
"fps": 24,
|
||||||
|
"flow_shift": 3.0,
|
||||||
|
"output": "lingbot_video_output.mp4",
|
||||||
|
},
|
||||||
"ltx2": {
|
"ltx2": {
|
||||||
"height": 512,
|
"height": 512,
|
||||||
"width": 768,
|
"width": 768,
|
||||||
@@ -110,27 +121,51 @@ _MODEL_PRESETS = {
|
|||||||
def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
|
def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
|
||||||
model_lower = model.lower()
|
model_lower = model.lower()
|
||||||
class_lower = (model_class_name or "").lower()
|
class_lower = (model_class_name or "").lower()
|
||||||
if "distilled" in class_lower or "distilled" in model_lower:
|
if "lingbot" in model_lower or "lingbotvideo" in class_lower:
|
||||||
return _MODEL_PRESETS["ltx2_distilled"]
|
return _MODEL_PRESETS["lingbot"]
|
||||||
if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
|
if "ltx" in class_lower or "ltx" in model_lower:
|
||||||
return _MODEL_PRESETS["ltx23"]
|
if "distilled" in class_lower or "distilled" in model_lower:
|
||||||
if "ltx2" in class_lower or "ltx-2" in model_lower or "ltx_2" in model_lower:
|
return _MODEL_PRESETS["ltx2_distilled"]
|
||||||
|
if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
|
||||||
|
return _MODEL_PRESETS["ltx23"]
|
||||||
return _MODEL_PRESETS["ltx2"]
|
return _MODEL_PRESETS["ltx2"]
|
||||||
if "vace" in model_lower:
|
if "vace" in model_lower or "vace" in class_lower:
|
||||||
return _MODEL_PRESETS["vace"]
|
return _MODEL_PRESETS["vace"]
|
||||||
# Edge must be matched before the generic cosmos branch (its "cosmos" substring would
|
# Edge must be matched before the generic cosmos branch (its "cosmos" substring would
|
||||||
# otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset).
|
# otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset).
|
||||||
if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower):
|
if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower):
|
||||||
return _MODEL_PRESETS["cosmos3_edge"]
|
return _MODEL_PRESETS["cosmos3_edge"]
|
||||||
if "cosmos" in model_lower:
|
if "cosmos" in model_lower or "cosmos" in class_lower:
|
||||||
return _MODEL_PRESETS["cosmos"]
|
return _MODEL_PRESETS["cosmos"]
|
||||||
if "hunyuan" in model_lower:
|
if "hunyuan" in model_lower or "hunyuan" in class_lower:
|
||||||
return _MODEL_PRESETS["hunyuan"]
|
return _MODEL_PRESETS["hunyuan"]
|
||||||
if "helios" in model_lower:
|
if "helios" in model_lower or "helios" in class_lower:
|
||||||
return _MODEL_PRESETS["helios"]
|
return _MODEL_PRESETS["helios"]
|
||||||
return _MODEL_PRESETS["wan"]
|
return _MODEL_PRESETS["wan"]
|
||||||
|
|
||||||
|
|
||||||
|
def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
|
||||||
|
"""Build the canonical request envelope for the shared T2V example."""
|
||||||
|
result: dict[str, Any] = {
|
||||||
|
"prompt": prompt,
|
||||||
|
"modalities": ["video"],
|
||||||
|
}
|
||||||
|
if negative_prompt is not None:
|
||||||
|
result["negative_prompt"] = negative_prompt
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor:
|
||||||
|
"""Normalize decoded frames according to the pipeline's output contract."""
|
||||||
|
if not tensor.is_floating_point():
|
||||||
|
return tensor
|
||||||
|
if source_range == "negative_one_to_one":
|
||||||
|
return tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||||
|
if source_range == "zero_to_one":
|
||||||
|
return tensor.clamp(0, 1)
|
||||||
|
raise ValueError(f"Unsupported floating-point tensor range: {source_range!r}")
|
||||||
|
|
||||||
|
|
||||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
config = json.loads(value)
|
config = json.loads(value)
|
||||||
@@ -160,7 +195,7 @@ def parse_extra_body(value: str) -> dict[str, Any]:
|
|||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Generate a video from a text prompt. "
|
description="Generate a video from a text prompt. "
|
||||||
"Supports Wan2.2, HunyuanVideo-1.5, Helios, and other text-to-video models."
|
"Supports Wan2.2, HunyuanVideo-1.5, Helios, LingBot-Video, and other text-to-video models."
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--model",
|
"--model",
|
||||||
@@ -408,6 +443,7 @@ def main():
|
|||||||
for key, default_val in preset.items():
|
for key, default_val in preset.items():
|
||||||
if getattr(args, key.replace("-", "_"), None) is None:
|
if getattr(args, key.replace("-", "_"), None) is None:
|
||||||
setattr(args, key.replace("-", "_"), default_val)
|
setattr(args, key.replace("-", "_"), default_val)
|
||||||
|
model_class_name = args.model_class_name
|
||||||
|
|
||||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||||
# Cache-dit config (Wan2.2 only)
|
# Cache-dit config (Wan2.2 only)
|
||||||
@@ -480,6 +516,7 @@ def main():
|
|||||||
omni = Omni(**omni_kwargs)
|
omni = Omni(**omni_kwargs)
|
||||||
model_class_name = get_model_class_name(omni) or model_class_name
|
model_class_name = get_model_class_name(omni) or model_class_name
|
||||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||||
|
output_tensor_range = get_output_tensor_range(model_class_name)
|
||||||
|
|
||||||
if profiler_enabled:
|
if profiler_enabled:
|
||||||
print("[Profiler] Starting profiling...")
|
print("[Profiler] Starting profiling...")
|
||||||
@@ -513,12 +550,11 @@ def main():
|
|||||||
lora_path=lora_path,
|
lora_path=lora_path,
|
||||||
)
|
)
|
||||||
|
|
||||||
prompt_dict = {"prompt": args.prompt}
|
negative_prompt = args.negative_prompt
|
||||||
if args.negative_prompt is not None:
|
if negative_prompt is None and all(preset is not _MODEL_PRESETS[name] for name in ("lingbot", "ltx2", "ltx23")):
|
||||||
prompt_dict["negative_prompt"] = args.negative_prompt
|
|
||||||
elif preset not in (_MODEL_PRESETS["ltx2"], _MODEL_PRESETS["ltx23"]):
|
|
||||||
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
||||||
prompt_dict["negative_prompt"] = ""
|
negative_prompt = ""
|
||||||
|
prompt_dict = build_text_to_video_prompt(args.prompt, negative_prompt)
|
||||||
|
|
||||||
extra_args = {}
|
extra_args = {}
|
||||||
if lora_request:
|
if lora_request:
|
||||||
@@ -630,8 +666,7 @@ def main():
|
|||||||
frame_tensor = frame_tensor[0]
|
frame_tensor = frame_tensor[0]
|
||||||
if frame_tensor.dim() == 3 and frame_tensor.shape[0] in (3, 4):
|
if frame_tensor.dim() == 3 and frame_tensor.shape[0] in (3, 4):
|
||||||
frame_tensor = frame_tensor.permute(1, 2, 0)
|
frame_tensor = frame_tensor.permute(1, 2, 0)
|
||||||
if frame_tensor.is_floating_point():
|
frame_tensor = _normalize_float_tensor(frame_tensor, output_tensor_range)
|
||||||
frame_tensor = frame_tensor.clamp(-1, 1) * 0.5 + 0.5
|
|
||||||
return frame_tensor.float().numpy()
|
return frame_tensor.float().numpy()
|
||||||
if isinstance(frame, np.ndarray):
|
if isinstance(frame, np.ndarray):
|
||||||
frame_array = frame
|
frame_array = frame
|
||||||
@@ -683,8 +718,7 @@ def main():
|
|||||||
video_tensor = video_tensor[0]
|
video_tensor = video_tensor[0]
|
||||||
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
|
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
|
||||||
video_tensor = video_tensor.permute(1, 2, 3, 0)
|
video_tensor = video_tensor.permute(1, 2, 3, 0)
|
||||||
if video_tensor.is_floating_point():
|
video_tensor = _normalize_float_tensor(video_tensor, output_tensor_range)
|
||||||
video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
|
|
||||||
video_array = video_tensor.float().numpy()
|
video_array = video_tensor.float().numpy()
|
||||||
elif isinstance(frames, np.ndarray):
|
elif isinstance(frames, np.ndarray):
|
||||||
video_array = frames
|
video_array = frames
|
||||||
|
|||||||
@@ -1,131 +0,0 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import torch
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
|
||||||
from vllm_omni.outputs import OmniRequestOutput
|
|
||||||
from vllm_omni.platforms import current_omni_platform
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
|
||||||
parser = argparse.ArgumentParser(description="Generate text-to-video with a LingBot-Video checkpoint.")
|
|
||||||
parser.add_argument("--model", default="robbyant/lingbot-video-dense-1.3b")
|
|
||||||
parser.add_argument("--prompt", default="a robotic arm picks up a red block")
|
|
||||||
parser.add_argument("--negative-prompt", default=None)
|
|
||||||
parser.add_argument("--output", default="lingbot_video_output.mp4")
|
|
||||||
parser.add_argument("--height", type=int, default=192)
|
|
||||||
parser.add_argument("--width", type=int, default=320)
|
|
||||||
parser.add_argument("--num-frames", type=int, default=9)
|
|
||||||
parser.add_argument("--num-inference-steps", type=int, default=2)
|
|
||||||
parser.add_argument("--guidance-scale", type=float, default=3.0)
|
|
||||||
parser.add_argument("--flow-shift", type=float, default=3.0)
|
|
||||||
parser.add_argument("--seed", type=int, default=42)
|
|
||||||
parser.add_argument("--fps", type=int, default=24)
|
|
||||||
parser.add_argument("--output-type", choices=["pt", "np"], default="pt")
|
|
||||||
parser.add_argument("--batch-cfg", action="store_true")
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def _peak_memory_mb(output: Any) -> float:
|
|
||||||
result = output[0] if isinstance(output, list) and output else output
|
|
||||||
return float(getattr(result, "peak_memory_mb", 0.0) or 0.0)
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_video(output: Any) -> Any:
|
|
||||||
if isinstance(output, list):
|
|
||||||
output = output[0] if output else None
|
|
||||||
if isinstance(output, OmniRequestOutput):
|
|
||||||
if output.final_output_type != "image":
|
|
||||||
raise ValueError(f"Unexpected output type {output.final_output_type!r}.")
|
|
||||||
if not output.images:
|
|
||||||
raise ValueError("No video frames found in OmniRequestOutput.")
|
|
||||||
return output.images[0] if len(output.images) == 1 else output.images
|
|
||||||
return output
|
|
||||||
|
|
||||||
|
|
||||||
def _to_frame_array(video: Any) -> np.ndarray:
|
|
||||||
if isinstance(video, torch.Tensor):
|
|
||||||
tensor = video.detach().cpu()
|
|
||||||
if tensor.dim() == 5:
|
|
||||||
tensor = tensor[0]
|
|
||||||
if tensor.dim() == 4 and tensor.shape[-1] not in (3, 4):
|
|
||||||
if tensor.shape[0] in (3, 4):
|
|
||||||
tensor = tensor.permute(1, 2, 3, 0)
|
|
||||||
elif tensor.shape[1] in (3, 4):
|
|
||||||
tensor = tensor.permute(0, 2, 3, 1)
|
|
||||||
if tensor.is_floating_point():
|
|
||||||
if float(tensor.min()) < 0.0:
|
|
||||||
tensor = tensor.clamp(-1, 1) * 0.5 + 0.5
|
|
||||||
else:
|
|
||||||
tensor = tensor.clamp(0, 1)
|
|
||||||
return tensor.float().numpy()
|
|
||||||
if isinstance(video, np.ndarray):
|
|
||||||
array = video[0] if video.ndim == 5 else video
|
|
||||||
if np.issubdtype(array.dtype, np.integer):
|
|
||||||
return array.astype(np.float32) / 255.0
|
|
||||||
return array.astype(np.float32, copy=False)
|
|
||||||
if isinstance(video, list):
|
|
||||||
return np.stack([_to_frame_array(frame) for frame in video], axis=0)
|
|
||||||
return np.asarray(video, dtype=np.float32)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
args = parse_args()
|
|
||||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
|
||||||
omni = Omni(
|
|
||||||
model=args.model,
|
|
||||||
model_class_name="LingBotVideoPipeline",
|
|
||||||
flow_shift=args.flow_shift,
|
|
||||||
parallel_config=DiffusionParallelConfig(),
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt: dict[str, Any] = {
|
|
||||||
"prompt": args.prompt,
|
|
||||||
"modalities": ["video"],
|
|
||||||
"num_frames": args.num_frames,
|
|
||||||
}
|
|
||||||
if args.negative_prompt is not None:
|
|
||||||
prompt["negative_prompt"] = args.negative_prompt
|
|
||||||
|
|
||||||
sampling_params = OmniDiffusionSamplingParams(
|
|
||||||
height=args.height,
|
|
||||||
width=args.width,
|
|
||||||
num_frames=args.num_frames,
|
|
||||||
num_inference_steps=args.num_inference_steps,
|
|
||||||
guidance_scale=args.guidance_scale,
|
|
||||||
generator=generator,
|
|
||||||
output_type=args.output_type,
|
|
||||||
extra_args={"batch_cfg": args.batch_cfg, "shift": args.flow_shift},
|
|
||||||
)
|
|
||||||
|
|
||||||
start = time.perf_counter()
|
|
||||||
output = omni.generate(prompt, sampling_params)
|
|
||||||
elapsed = time.perf_counter() - start
|
|
||||||
print(f"Total generation time: {elapsed:.4f} seconds ({elapsed * 1000:.2f} ms)")
|
|
||||||
peak_mb = _peak_memory_mb(output)
|
|
||||||
if peak_mb:
|
|
||||||
print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
|
|
||||||
|
|
||||||
video = _to_frame_array(_extract_video(output))
|
|
||||||
output_path = Path(args.output)
|
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
from diffusers.utils import export_to_video
|
|
||||||
|
|
||||||
export_to_video(list(video), str(output_path), fps=args.fps)
|
|
||||||
print(f"Saved generated video to {output_path}")
|
|
||||||
omni.close()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -27,7 +27,7 @@ parallel, cache, quantized, or expert-kernel backends.
|
|||||||
- Dense checkpoint: <https://huggingface.co/robbyant/lingbot-video-dense-1.3b>
|
- Dense checkpoint: <https://huggingface.co/robbyant/lingbot-video-dense-1.3b>
|
||||||
- MoE checkpoint: <https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b>
|
- MoE checkpoint: <https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b>
|
||||||
- Upstream project: <https://github.com/Robbyant/lingbot-video>
|
- Upstream project: <https://github.com/Robbyant/lingbot-video>
|
||||||
- Related offline example: [`examples/offline_inference/text_to_video/text_to_video_lingbot.py`](../../examples/offline_inference/text_to_video/text_to_video_lingbot.py)
|
- Related offline example: [`examples/offline_inference/text_to_video/text_to_video.py`](../../examples/offline_inference/text_to_video/text_to_video.py)
|
||||||
- Related online video API docs: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md)
|
- Related online video API docs: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md)
|
||||||
|
|
||||||
## Hardware Support
|
## Hardware Support
|
||||||
@@ -52,7 +52,7 @@ The MoE path is validated with BF16 expert weights.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
CUDA_VISIBLE_DEVICES=0 \
|
CUDA_VISIBLE_DEVICES=0 \
|
||||||
python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
python examples/offline_inference/text_to_video/text_to_video.py \
|
||||||
--model robbyant/lingbot-video-dense-1.3b \
|
--model robbyant/lingbot-video-dense-1.3b \
|
||||||
--prompt "a robotic arm picks up a red block" \
|
--prompt "a robotic arm picks up a red block" \
|
||||||
--output lingbot_t2v.mp4 \
|
--output lingbot_t2v.mp4 \
|
||||||
@@ -70,7 +70,7 @@ python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
CUDA_VISIBLE_DEVICES=0 \
|
CUDA_VISIBLE_DEVICES=0 \
|
||||||
python examples/offline_inference/text_to_video/text_to_video_lingbot.py \
|
python examples/offline_inference/text_to_video/text_to_video.py \
|
||||||
--model robbyant/lingbot-video-moe-30b-a3b \
|
--model robbyant/lingbot-video-moe-30b-a3b \
|
||||||
--prompt "a robotic arm picks up a red block" \
|
--prompt "a robotic arm picks up a red block" \
|
||||||
--output lingbot_moe_t2v.mp4 \
|
--output lingbot_moe_t2v.mp4 \
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from examples.offline_inference.text_to_video.text_to_video import (
|
||||||
|
_MODEL_PRESETS,
|
||||||
|
_detect_preset,
|
||||||
|
_normalize_float_tensor,
|
||||||
|
build_text_to_video_prompt,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("negative_prompt", "expected_negative_prompt"),
|
||||||
|
[(None, None), ("", ""), ("blurry", "blurry")],
|
||||||
|
)
|
||||||
|
def test_text_to_video_builds_canonical_prompt(
|
||||||
|
negative_prompt: str | None,
|
||||||
|
expected_negative_prompt: str | None,
|
||||||
|
) -> None:
|
||||||
|
result = build_text_to_video_prompt("a robot waves", negative_prompt)
|
||||||
|
|
||||||
|
assert result["prompt"] == "a robot waves"
|
||||||
|
assert result["modalities"] == ["video"]
|
||||||
|
if expected_negative_prompt is None:
|
||||||
|
assert "negative_prompt" not in result
|
||||||
|
else:
|
||||||
|
assert result["negative_prompt"] == expected_negative_prompt
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("model", "model_class_name", "preset_name"),
|
||||||
|
[
|
||||||
|
("robbyant/lingbot-video-dense-1.3b", None, "lingbot"),
|
||||||
|
("/models/custom-checkpoint", "LingBotVideoPipeline", "lingbot"),
|
||||||
|
("Lightricks/LTX-2", "LTX2DistilledPipeline", "ltx2_distilled"),
|
||||||
|
("BestWishYsh/Helios-Distilled", None, "helios"),
|
||||||
|
("/models/cosmos-edge", "Cosmos3EdgeVFMTransformer", "cosmos3_edge"),
|
||||||
|
("/models/vace", "WanVACEPipeline", "vace"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_detect_preset_is_scoped_to_model_family(
|
||||||
|
model: str,
|
||||||
|
model_class_name: str | None,
|
||||||
|
preset_name: str,
|
||||||
|
) -> None:
|
||||||
|
assert _detect_preset(model, model_class_name) is _MODEL_PRESETS[preset_name]
|
||||||
|
|
||||||
|
|
||||||
|
def test_lingbot_preset_matches_lingbot_defaults() -> None:
|
||||||
|
assert _MODEL_PRESETS["lingbot"] == {
|
||||||
|
"model_class_name": "LingBotVideoPipeline",
|
||||||
|
"height": 192,
|
||||||
|
"width": 320,
|
||||||
|
"num_frames": 9,
|
||||||
|
"num_inference_steps": 2,
|
||||||
|
"guidance_scale": 3.0,
|
||||||
|
"fps": 24,
|
||||||
|
"flow_shift": 3.0,
|
||||||
|
"output": "lingbot_video_output.mp4",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_float_tensor_preserves_zero_to_one_frames() -> None:
|
||||||
|
frames = torch.tensor([0.0, 0.25, 1.0, 1.5])
|
||||||
|
|
||||||
|
assert torch.equal(_normalize_float_tensor(frames, "zero_to_one"), torch.tensor([0.0, 0.25, 1.0, 1.0]))
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_float_tensor_maps_negative_one_to_one_frames() -> None:
|
||||||
|
frames = torch.tensor([-1.5, -1.0, 0.0, 1.0, 1.5])
|
||||||
|
|
||||||
|
assert torch.equal(
|
||||||
|
_normalize_float_tensor(frames, "negative_one_to_one"),
|
||||||
|
torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_float_tensor_uses_one_contract_for_ambiguous_frames() -> None:
|
||||||
|
frames = [torch.tensor([0.0, 0.5, 1.0]), torch.tensor([-1.0, 0.0, 1.0])]
|
||||||
|
|
||||||
|
normalized = [_normalize_float_tensor(frame, "negative_one_to_one") for frame in frames]
|
||||||
|
|
||||||
|
assert torch.equal(normalized[0], torch.tensor([0.5, 0.75, 1.0]))
|
||||||
|
assert torch.equal(normalized[1], torch.tensor([0.0, 0.5, 1.0]))
|
||||||
@@ -17,6 +17,7 @@ from vllm_omni.model_extras import (
|
|||||||
build_x_to_text_prompt,
|
build_x_to_text_prompt,
|
||||||
get_extra_body_params,
|
get_extra_body_params,
|
||||||
get_extra_output_params,
|
get_extra_output_params,
|
||||||
|
get_output_tensor_range,
|
||||||
get_x_to_text_model_family,
|
get_x_to_text_model_family,
|
||||||
should_init_extra_args_for_non_diffusion_stages,
|
should_init_extra_args_for_non_diffusion_stages,
|
||||||
)
|
)
|
||||||
@@ -134,6 +135,8 @@ def test_lingbot_extra_registry_declares_request_params() -> None:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
assert get_extra_output_params("LingBotVideoPipeline") == frozenset()
|
assert get_extra_output_params("LingBotVideoPipeline") == frozenset()
|
||||||
|
assert get_output_tensor_range("LingBotVideoPipeline") == "zero_to_one"
|
||||||
|
assert get_output_tensor_range("WanPipeline") == "negative_one_to_one"
|
||||||
assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False
|
assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from vllm_omni.model_extras.registry import (
|
|||||||
get_extra_body_params,
|
get_extra_body_params,
|
||||||
get_extra_output_params,
|
get_extra_output_params,
|
||||||
get_model_class_name,
|
get_model_class_name,
|
||||||
|
get_output_tensor_range,
|
||||||
get_x_to_text_model_family,
|
get_x_to_text_model_family,
|
||||||
should_init_extra_args_for_non_diffusion_stages,
|
should_init_extra_args_for_non_diffusion_stages,
|
||||||
)
|
)
|
||||||
@@ -21,6 +22,7 @@ __all__ = [
|
|||||||
"get_extra_body_params",
|
"get_extra_body_params",
|
||||||
"get_extra_output_params",
|
"get_extra_output_params",
|
||||||
"get_model_class_name",
|
"get_model_class_name",
|
||||||
|
"get_output_tensor_range",
|
||||||
"get_x_to_text_model_family",
|
"get_x_to_text_model_family",
|
||||||
"should_init_extra_args_for_non_diffusion_stages",
|
"should_init_extra_args_for_non_diffusion_stages",
|
||||||
]
|
]
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from typing import Any
|
from typing import Any, Literal
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
@@ -93,6 +93,7 @@ ImageToVideoPromptBuilder = Callable[
|
|||||||
dict[str, Any],
|
dict[str, Any],
|
||||||
]
|
]
|
||||||
XToTextPromptBuilder = Callable[[str, str, bool], tuple[dict[str, Any], list[int] | None]]
|
XToTextPromptBuilder = Callable[[str, str, bool], tuple[dict[str, Any], list[int] | None]]
|
||||||
|
OutputTensorRange = Literal["negative_one_to_one", "zero_to_one"]
|
||||||
|
|
||||||
|
|
||||||
def default_x_to_text_prompt(
|
def default_x_to_text_prompt(
|
||||||
@@ -193,6 +194,7 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = {
|
|||||||
},
|
},
|
||||||
"LingBotVideoPipeline": {
|
"LingBotVideoPipeline": {
|
||||||
"extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS,
|
"extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS,
|
||||||
|
"output_tensor_range": "zero_to_one",
|
||||||
# Shared T2I/I2V envelopes select the output modality. LingBot's
|
# Shared T2I/I2V envelopes select the output modality. LingBot's
|
||||||
# pipeline owns model-specific validation and normalization.
|
# pipeline owns model-specific validation and normalization.
|
||||||
},
|
},
|
||||||
@@ -262,6 +264,18 @@ def get_extra_output_params(model_class_name: str | None) -> frozenset[str]:
|
|||||||
return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset()
|
return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset()
|
||||||
|
|
||||||
|
|
||||||
|
def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange:
|
||||||
|
"""Return the declared range for floating-point tensor outputs.
|
||||||
|
|
||||||
|
The default preserves the shared examples' historical handling. Pipelines
|
||||||
|
that already return normalized tensors declare ``zero_to_one`` explicitly.
|
||||||
|
"""
|
||||||
|
spec = _get_spec(model_class_name)
|
||||||
|
if spec is None:
|
||||||
|
return "negative_one_to_one"
|
||||||
|
return spec.get("output_tensor_range", "negative_one_to_one")
|
||||||
|
|
||||||
|
|
||||||
def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None) -> bool:
|
def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None) -> bool:
|
||||||
spec = _get_spec(model_class_name)
|
spec = _get_spec(model_class_name)
|
||||||
return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False))
|
return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False))
|
||||||
|
|||||||
Reference in new issue
Block a user