From 8b8d3f56db4a1e39c635db73fec96112918d8083 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Thu, 13 Aug 2026 08:42:37 +0800 Subject: [PATCH] [Misc] Consolidate LingBot text-to-video runner (#6076) Signed-off-by: Hongsheng Liu --- .../lingbot_video/dense_pipeline_parity.py | 4 +- .../offline_inference/text_to_video.md | 20 +++ .../text_to_video/text_to_video.md | 20 ++- .../text_to_video/text_to_video.py | 74 +++++++--- .../text_to_video/text_to_video_lingbot.py | 131 ------------------ recipes/Robbyant/LingBot-Video.md | 6 +- .../test_text_to_video_prompts.py | 91 ++++++++++++ tests/model_extras/test_model_extras.py | 3 + vllm_omni/model_extras/__init__.py | 2 + vllm_omni/model_extras/registry.py | 16 ++- 10 files changed, 208 insertions(+), 159 deletions(-) delete mode 100644 examples/offline_inference/text_to_video/text_to_video_lingbot.py create mode 100644 tests/examples/offline_inference/test_text_to_video_prompts.py diff --git a/benchmarks/lingbot_video/dense_pipeline_parity.py b/benchmarks/lingbot_video/dense_pipeline_parity.py index 0c0c1d7a0..cc781a95b 100644 --- a/benchmarks/lingbot_video/dense_pipeline_parity.py +++ b/benchmarks/lingbot_video/dense_pipeline_parity.py @@ -167,9 +167,11 @@ def main() -> None: ] native_cmd = [ sys.executable, - "examples/offline_inference/text_to_video/text_to_video_lingbot.py", + "examples/offline_inference/text_to_video/text_to_video.py", "--model", args.model, + "--model-class-name", + "LingBotVideoPipeline", "--prompt", args.prompt, "--output", diff --git a/docs/user_guide/examples/offline_inference/text_to_video.md b/docs/user_guide/examples/offline_inference/text_to_video.md index d32a1f562..1a2082f82 100644 --- a/docs/user_guide/examples/offline_inference/text_to_video.md +++ b/docs/user_guide/examples/offline_inference/text_to_video.md @@ -12,6 +12,7 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back | Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) | |---|---|---|---|---|---| | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB | +| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) | | `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required | @@ -35,6 +36,25 @@ python text_to_video.py \ --output t2v_out.mp4 ``` +### LingBot-Video + +The shared runner recognizes both official LingBot checkpoint IDs and selects +the native `LingBotVideoPipeline` automatically: + +```bash +python text_to_video.py \ + --model robbyant/lingbot-video-dense-1.3b \ + --prompt "a robotic arm picks up a red block" \ + --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \ + --guidance-scale 3.0 --flow-shift 3.0 --fps 24 \ + --output lingbot_t2v.mp4 +``` + +Use `robbyant/lingbot-video-moe-30b-a3b` for the MoE checkpoint. For local +paths without `lingbot` in the directory name, add +`--model-class-name LingBotVideoPipeline`. Model-only options use +`--extra-body`, for example `'{"batch_cfg": true, "output_type": "np"}'`. + ### LTX-2 ```bash diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index 50b207526..339c7dac9 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -8,6 +8,7 @@ A unified script for text-to-video generation. Supports multiple models with mod |---|---|---|---|---|---| | `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) | | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB | +| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) | | `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB | | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required | @@ -57,18 +58,31 @@ python text_to_video.py \ ### LingBot-Video -The dedicated LingBot example explicitly selects the video output modality: +The shared runner recognizes the official dense and MoE checkpoint IDs, selects +`LingBotVideoPipeline`, and builds the canonical video request envelope: ```bash -python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ +python text_to_video.py \ --model robbyant/lingbot-video-dense-1.3b \ + --prompt "a robotic arm picks up a red block" \ --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \ + --guidance-scale 3.0 --flow-shift 3.0 --fps 24 \ --output lingbot_t2v.mp4 ``` Use the same command with `robbyant/lingbot-video-moe-30b-a3b` for the MoE checkpoint. Requested frame counts are rounded upward to the causal -VAE's `4n+1` grid. +VAE's `4n+1` grid. For a local checkpoint path whose name does not contain +`lingbot`, pass `--model-class-name LingBotVideoPipeline`. + +LingBot-only options such as `batch_cfg` and `output_type` use the shared +model-extra channel, for example: + +```bash +python text_to_video.py \ + --model robbyant/lingbot-video-dense-1.3b \ + --extra-body '{"batch_cfg": true, "output_type": "np"}' +``` ### LTX-2 diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 2e272b7a7..129ca860a 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -16,7 +16,7 @@ from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.utils import stable_lora_int_id -from vllm_omni.model_extras import get_extra_body_params, get_model_class_name +from vllm_omni.model_extras import get_extra_body_params, get_model_class_name, get_output_tensor_range from vllm_omni.outputs import OmniRequestOutput from vllm_omni.platforms import current_omni_platform @@ -80,6 +80,17 @@ _MODEL_PRESETS = { "fps": 16, "output": "helios_output.mp4", }, + "lingbot": { + "model_class_name": "LingBotVideoPipeline", + "height": 192, + "width": 320, + "num_frames": 9, + "num_inference_steps": 2, + "guidance_scale": 3.0, + "fps": 24, + "flow_shift": 3.0, + "output": "lingbot_video_output.mp4", + }, "ltx2": { "height": 512, "width": 768, @@ -110,27 +121,51 @@ _MODEL_PRESETS = { def _detect_preset(model: str, model_class_name: str | None = None) -> dict: model_lower = model.lower() class_lower = (model_class_name or "").lower() - if "distilled" in class_lower or "distilled" in model_lower: - return _MODEL_PRESETS["ltx2_distilled"] - if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower: - return _MODEL_PRESETS["ltx23"] - if "ltx2" in class_lower or "ltx-2" in model_lower or "ltx_2" in model_lower: + if "lingbot" in model_lower or "lingbotvideo" in class_lower: + return _MODEL_PRESETS["lingbot"] + if "ltx" in class_lower or "ltx" in model_lower: + if "distilled" in class_lower or "distilled" in model_lower: + return _MODEL_PRESETS["ltx2_distilled"] + if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower: + return _MODEL_PRESETS["ltx23"] return _MODEL_PRESETS["ltx2"] - if "vace" in model_lower: + if "vace" in model_lower or "vace" in class_lower: return _MODEL_PRESETS["vace"] # Edge must be matched before the generic cosmos branch (its "cosmos" substring would # otherwise pick up the Nano/Super 720p / gs 6.0 / flow_shift 10.0 preset). if ("cosmos" in model_lower or "cosmos" in class_lower) and ("edge" in model_lower or "edge" in class_lower): return _MODEL_PRESETS["cosmos3_edge"] - if "cosmos" in model_lower: + if "cosmos" in model_lower or "cosmos" in class_lower: return _MODEL_PRESETS["cosmos"] - if "hunyuan" in model_lower: + if "hunyuan" in model_lower or "hunyuan" in class_lower: return _MODEL_PRESETS["hunyuan"] - if "helios" in model_lower: + if "helios" in model_lower or "helios" in class_lower: return _MODEL_PRESETS["helios"] return _MODEL_PRESETS["wan"] +def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]: + """Build the canonical request envelope for the shared T2V example.""" + result: dict[str, Any] = { + "prompt": prompt, + "modalities": ["video"], + } + if negative_prompt is not None: + result["negative_prompt"] = negative_prompt + return result + + +def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor: + """Normalize decoded frames according to the pipeline's output contract.""" + if not tensor.is_floating_point(): + return tensor + if source_range == "negative_one_to_one": + return tensor.clamp(-1, 1) * 0.5 + 0.5 + if source_range == "zero_to_one": + return tensor.clamp(0, 1) + raise ValueError(f"Unsupported floating-point tensor range: {source_range!r}") + + def parse_profiler_config(value: str) -> dict[str, Any]: try: config = json.loads(value) @@ -160,7 +195,7 @@ def parse_extra_body(value: str) -> dict[str, Any]: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Generate a video from a text prompt. " - "Supports Wan2.2, HunyuanVideo-1.5, Helios, and other text-to-video models." + "Supports Wan2.2, HunyuanVideo-1.5, Helios, LingBot-Video, and other text-to-video models." ) parser.add_argument( "--model", @@ -408,6 +443,7 @@ def main(): for key, default_val in preset.items(): if getattr(args, key.replace("-", "_"), None) is None: setattr(args, key.replace("-", "_"), default_val) + model_class_name = args.model_class_name generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) # Cache-dit config (Wan2.2 only) @@ -480,6 +516,7 @@ def main(): omni = Omni(**omni_kwargs) model_class_name = get_model_class_name(omni) or model_class_name declared_extra_body_params = get_extra_body_params(model_class_name) + output_tensor_range = get_output_tensor_range(model_class_name) if profiler_enabled: print("[Profiler] Starting profiling...") @@ -513,12 +550,11 @@ def main(): lora_path=lora_path, ) - prompt_dict = {"prompt": args.prompt} - if args.negative_prompt is not None: - prompt_dict["negative_prompt"] = args.negative_prompt - elif preset not in (_MODEL_PRESETS["ltx2"], _MODEL_PRESETS["ltx23"]): + negative_prompt = args.negative_prompt + if negative_prompt is None and all(preset is not _MODEL_PRESETS[name] for name in ("lingbot", "ltx2", "ltx23")): # Preserve the historical empty-prompt behavior for non-LTX examples. - prompt_dict["negative_prompt"] = "" + negative_prompt = "" + prompt_dict = build_text_to_video_prompt(args.prompt, negative_prompt) extra_args = {} if lora_request: @@ -630,8 +666,7 @@ def main(): frame_tensor = frame_tensor[0] if frame_tensor.dim() == 3 and frame_tensor.shape[0] in (3, 4): frame_tensor = frame_tensor.permute(1, 2, 0) - if frame_tensor.is_floating_point(): - frame_tensor = frame_tensor.clamp(-1, 1) * 0.5 + 0.5 + frame_tensor = _normalize_float_tensor(frame_tensor, output_tensor_range) return frame_tensor.float().numpy() if isinstance(frame, np.ndarray): frame_array = frame @@ -683,8 +718,7 @@ def main(): video_tensor = video_tensor[0] elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4): video_tensor = video_tensor.permute(1, 2, 3, 0) - if video_tensor.is_floating_point(): - video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5 + video_tensor = _normalize_float_tensor(video_tensor, output_tensor_range) video_array = video_tensor.float().numpy() elif isinstance(frames, np.ndarray): video_array = frames diff --git a/examples/offline_inference/text_to_video/text_to_video_lingbot.py b/examples/offline_inference/text_to_video/text_to_video_lingbot.py deleted file mode 100644 index 108e2dc1f..000000000 --- a/examples/offline_inference/text_to_video/text_to_video_lingbot.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from __future__ import annotations - -import argparse -import time -from pathlib import Path -from typing import Any - -import numpy as np -import torch - -from vllm_omni.diffusion.data import DiffusionParallelConfig -from vllm_omni.entrypoints.omni import Omni -from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.platforms import current_omni_platform - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Generate text-to-video with a LingBot-Video checkpoint.") - parser.add_argument("--model", default="robbyant/lingbot-video-dense-1.3b") - parser.add_argument("--prompt", default="a robotic arm picks up a red block") - parser.add_argument("--negative-prompt", default=None) - parser.add_argument("--output", default="lingbot_video_output.mp4") - parser.add_argument("--height", type=int, default=192) - parser.add_argument("--width", type=int, default=320) - parser.add_argument("--num-frames", type=int, default=9) - parser.add_argument("--num-inference-steps", type=int, default=2) - parser.add_argument("--guidance-scale", type=float, default=3.0) - parser.add_argument("--flow-shift", type=float, default=3.0) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--fps", type=int, default=24) - parser.add_argument("--output-type", choices=["pt", "np"], default="pt") - parser.add_argument("--batch-cfg", action="store_true") - return parser.parse_args() - - -def _peak_memory_mb(output: Any) -> float: - result = output[0] if isinstance(output, list) and output else output - return float(getattr(result, "peak_memory_mb", 0.0) or 0.0) - - -def _extract_video(output: Any) -> Any: - if isinstance(output, list): - output = output[0] if output else None - if isinstance(output, OmniRequestOutput): - if output.final_output_type != "image": - raise ValueError(f"Unexpected output type {output.final_output_type!r}.") - if not output.images: - raise ValueError("No video frames found in OmniRequestOutput.") - return output.images[0] if len(output.images) == 1 else output.images - return output - - -def _to_frame_array(video: Any) -> np.ndarray: - if isinstance(video, torch.Tensor): - tensor = video.detach().cpu() - if tensor.dim() == 5: - tensor = tensor[0] - if tensor.dim() == 4 and tensor.shape[-1] not in (3, 4): - if tensor.shape[0] in (3, 4): - tensor = tensor.permute(1, 2, 3, 0) - elif tensor.shape[1] in (3, 4): - tensor = tensor.permute(0, 2, 3, 1) - if tensor.is_floating_point(): - if float(tensor.min()) < 0.0: - tensor = tensor.clamp(-1, 1) * 0.5 + 0.5 - else: - tensor = tensor.clamp(0, 1) - return tensor.float().numpy() - if isinstance(video, np.ndarray): - array = video[0] if video.ndim == 5 else video - if np.issubdtype(array.dtype, np.integer): - return array.astype(np.float32) / 255.0 - return array.astype(np.float32, copy=False) - if isinstance(video, list): - return np.stack([_to_frame_array(frame) for frame in video], axis=0) - return np.asarray(video, dtype=np.float32) - - -def main() -> None: - args = parse_args() - generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) - omni = Omni( - model=args.model, - model_class_name="LingBotVideoPipeline", - flow_shift=args.flow_shift, - parallel_config=DiffusionParallelConfig(), - ) - - prompt: dict[str, Any] = { - "prompt": args.prompt, - "modalities": ["video"], - "num_frames": args.num_frames, - } - if args.negative_prompt is not None: - prompt["negative_prompt"] = args.negative_prompt - - sampling_params = OmniDiffusionSamplingParams( - height=args.height, - width=args.width, - num_frames=args.num_frames, - num_inference_steps=args.num_inference_steps, - guidance_scale=args.guidance_scale, - generator=generator, - output_type=args.output_type, - extra_args={"batch_cfg": args.batch_cfg, "shift": args.flow_shift}, - ) - - start = time.perf_counter() - output = omni.generate(prompt, sampling_params) - elapsed = time.perf_counter() - start - print(f"Total generation time: {elapsed:.4f} seconds ({elapsed * 1000:.2f} ms)") - peak_mb = _peak_memory_mb(output) - if peak_mb: - print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)") - - video = _to_frame_array(_extract_video(output)) - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - from diffusers.utils import export_to_video - - export_to_video(list(video), str(output_path), fps=args.fps) - print(f"Saved generated video to {output_path}") - omni.close() - - -if __name__ == "__main__": - main() diff --git a/recipes/Robbyant/LingBot-Video.md b/recipes/Robbyant/LingBot-Video.md index 289d36990..29d112217 100644 --- a/recipes/Robbyant/LingBot-Video.md +++ b/recipes/Robbyant/LingBot-Video.md @@ -27,7 +27,7 @@ parallel, cache, quantized, or expert-kernel backends. - Dense checkpoint: - MoE checkpoint: - Upstream project: -- Related offline example: [`examples/offline_inference/text_to_video/text_to_video_lingbot.py`](../../examples/offline_inference/text_to_video/text_to_video_lingbot.py) +- Related offline example: [`examples/offline_inference/text_to_video/text_to_video.py`](../../examples/offline_inference/text_to_video/text_to_video.py) - Related online video API docs: [`docs/serving/videos_api.md`](../../docs/serving/videos_api.md) ## Hardware Support @@ -52,7 +52,7 @@ The MoE path is validated with BF16 expert weights. ```bash CUDA_VISIBLE_DEVICES=0 \ -python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ +python examples/offline_inference/text_to_video/text_to_video.py \ --model robbyant/lingbot-video-dense-1.3b \ --prompt "a robotic arm picks up a red block" \ --output lingbot_t2v.mp4 \ @@ -70,7 +70,7 @@ python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ ```bash CUDA_VISIBLE_DEVICES=0 \ -python examples/offline_inference/text_to_video/text_to_video_lingbot.py \ +python examples/offline_inference/text_to_video/text_to_video.py \ --model robbyant/lingbot-video-moe-30b-a3b \ --prompt "a robotic arm picks up a red block" \ --output lingbot_moe_t2v.mp4 \ diff --git a/tests/examples/offline_inference/test_text_to_video_prompts.py b/tests/examples/offline_inference/test_text_to_video_prompts.py new file mode 100644 index 000000000..ff7244d65 --- /dev/null +++ b/tests/examples/offline_inference/test_text_to_video_prompts.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import pytest +import torch + +from examples.offline_inference.text_to_video.text_to_video import ( + _MODEL_PRESETS, + _detect_preset, + _normalize_float_tensor, + build_text_to_video_prompt, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +@pytest.mark.parametrize( + ("negative_prompt", "expected_negative_prompt"), + [(None, None), ("", ""), ("blurry", "blurry")], +) +def test_text_to_video_builds_canonical_prompt( + negative_prompt: str | None, + expected_negative_prompt: str | None, +) -> None: + result = build_text_to_video_prompt("a robot waves", negative_prompt) + + assert result["prompt"] == "a robot waves" + assert result["modalities"] == ["video"] + if expected_negative_prompt is None: + assert "negative_prompt" not in result + else: + assert result["negative_prompt"] == expected_negative_prompt + + +@pytest.mark.parametrize( + ("model", "model_class_name", "preset_name"), + [ + ("robbyant/lingbot-video-dense-1.3b", None, "lingbot"), + ("/models/custom-checkpoint", "LingBotVideoPipeline", "lingbot"), + ("Lightricks/LTX-2", "LTX2DistilledPipeline", "ltx2_distilled"), + ("BestWishYsh/Helios-Distilled", None, "helios"), + ("/models/cosmos-edge", "Cosmos3EdgeVFMTransformer", "cosmos3_edge"), + ("/models/vace", "WanVACEPipeline", "vace"), + ], +) +def test_detect_preset_is_scoped_to_model_family( + model: str, + model_class_name: str | None, + preset_name: str, +) -> None: + assert _detect_preset(model, model_class_name) is _MODEL_PRESETS[preset_name] + + +def test_lingbot_preset_matches_lingbot_defaults() -> None: + assert _MODEL_PRESETS["lingbot"] == { + "model_class_name": "LingBotVideoPipeline", + "height": 192, + "width": 320, + "num_frames": 9, + "num_inference_steps": 2, + "guidance_scale": 3.0, + "fps": 24, + "flow_shift": 3.0, + "output": "lingbot_video_output.mp4", + } + + +def test_normalize_float_tensor_preserves_zero_to_one_frames() -> None: + frames = torch.tensor([0.0, 0.25, 1.0, 1.5]) + + assert torch.equal(_normalize_float_tensor(frames, "zero_to_one"), torch.tensor([0.0, 0.25, 1.0, 1.0])) + + +def test_normalize_float_tensor_maps_negative_one_to_one_frames() -> None: + frames = torch.tensor([-1.5, -1.0, 0.0, 1.0, 1.5]) + + assert torch.equal( + _normalize_float_tensor(frames, "negative_one_to_one"), + torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0]), + ) + + +def test_normalize_float_tensor_uses_one_contract_for_ambiguous_frames() -> None: + frames = [torch.tensor([0.0, 0.5, 1.0]), torch.tensor([-1.0, 0.0, 1.0])] + + normalized = [_normalize_float_tensor(frame, "negative_one_to_one") for frame in frames] + + assert torch.equal(normalized[0], torch.tensor([0.5, 0.75, 1.0])) + assert torch.equal(normalized[1], torch.tensor([0.0, 0.5, 1.0])) diff --git a/tests/model_extras/test_model_extras.py b/tests/model_extras/test_model_extras.py index fde3ea878..d57671adc 100644 --- a/tests/model_extras/test_model_extras.py +++ b/tests/model_extras/test_model_extras.py @@ -17,6 +17,7 @@ from vllm_omni.model_extras import ( build_x_to_text_prompt, get_extra_body_params, get_extra_output_params, + get_output_tensor_range, get_x_to_text_model_family, should_init_extra_args_for_non_diffusion_stages, ) @@ -134,6 +135,8 @@ def test_lingbot_extra_registry_declares_request_params() -> None: } ) assert get_extra_output_params("LingBotVideoPipeline") == frozenset() + assert get_output_tensor_range("LingBotVideoPipeline") == "zero_to_one" + assert get_output_tensor_range("WanPipeline") == "negative_one_to_one" assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False diff --git a/vllm_omni/model_extras/__init__.py b/vllm_omni/model_extras/__init__.py index 1a3d71870..b11ceadec 100644 --- a/vllm_omni/model_extras/__init__.py +++ b/vllm_omni/model_extras/__init__.py @@ -9,6 +9,7 @@ from vllm_omni.model_extras.registry import ( get_extra_body_params, get_extra_output_params, get_model_class_name, + get_output_tensor_range, get_x_to_text_model_family, should_init_extra_args_for_non_diffusion_stages, ) @@ -21,6 +22,7 @@ __all__ = [ "get_extra_body_params", "get_extra_output_params", "get_model_class_name", + "get_output_tensor_range", "get_x_to_text_model_family", "should_init_extra_args_for_non_diffusion_stages", ] diff --git a/vllm_omni/model_extras/registry.py b/vllm_omni/model_extras/registry.py index 4c41c5920..8d7ee1e29 100644 --- a/vllm_omni/model_extras/registry.py +++ b/vllm_omni/model_extras/registry.py @@ -4,7 +4,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping -from typing import Any +from typing import Any, Literal from PIL import Image @@ -93,6 +93,7 @@ ImageToVideoPromptBuilder = Callable[ dict[str, Any], ] XToTextPromptBuilder = Callable[[str, str, bool], tuple[dict[str, Any], list[int] | None]] +OutputTensorRange = Literal["negative_one_to_one", "zero_to_one"] def default_x_to_text_prompt( @@ -193,6 +194,7 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = { }, "LingBotVideoPipeline": { "extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS, + "output_tensor_range": "zero_to_one", # Shared T2I/I2V envelopes select the output modality. LingBot's # pipeline owns model-specific validation and normalization. }, @@ -262,6 +264,18 @@ def get_extra_output_params(model_class_name: str | None) -> frozenset[str]: return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset() +def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange: + """Return the declared range for floating-point tensor outputs. + + The default preserves the shared examples' historical handling. Pipelines + that already return normalized tensors declare ``zero_to_one`` explicitly. + """ + spec = _get_spec(model_class_name) + if spec is None: + return "negative_one_to_one" + return spec.get("output_tensor_range", "negative_one_to_one") + + def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None) -> bool: spec = _get_spec(model_class_name) return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False))