[Misc] Address LingBot runner review feedback
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
0a593bdded
commit
f82ec476bf
6 files changed
+46
-10
No files matched your search
@@ -170,6 +170,8 @@ def main() -> None:
|
||||
"examples/offline_inference/text_to_video/text_to_video.py",
|
||||
"--model",
|
||||
args.model,
|
||||
"--model-class-name",
|
||||
"LingBotVideoPipeline",
|
||||
"--prompt",
|
||||
args.prompt,
|
||||
"--output",
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.lora.utils import stable_lora_int_id
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name, get_output_tensor_range
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -155,13 +155,15 @@ def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict
|
||||
return result
|
||||
|
||||
|
||||
def _normalize_float_tensor(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Normalize decoded frames without remapping tensors already in [0, 1]."""
|
||||
def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor:
|
||||
"""Normalize decoded frames according to the pipeline's output contract."""
|
||||
if not tensor.is_floating_point():
|
||||
return tensor
|
||||
if float(tensor.min()) < 0.0:
|
||||
if source_range == "negative_one_to_one":
|
||||
return tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
return tensor.clamp(0, 1)
|
||||
if source_range == "zero_to_one":
|
||||
return tensor.clamp(0, 1)
|
||||
raise ValueError(f"Unsupported floating-point tensor range: {source_range!r}")
|
||||
|
||||
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
@@ -514,6 +516,7 @@ def main():
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni) or model_class_name
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
output_tensor_range = get_output_tensor_range(model_class_name)
|
||||
|
||||
if profiler_enabled:
|
||||
print("[Profiler] Starting profiling...")
|
||||
@@ -663,7 +666,7 @@ def main():
|
||||
frame_tensor = frame_tensor[0]
|
||||
if frame_tensor.dim() == 3 and frame_tensor.shape[0] in (3, 4):
|
||||
frame_tensor = frame_tensor.permute(1, 2, 0)
|
||||
frame_tensor = _normalize_float_tensor(frame_tensor)
|
||||
frame_tensor = _normalize_float_tensor(frame_tensor, output_tensor_range)
|
||||
return frame_tensor.float().numpy()
|
||||
if isinstance(frame, np.ndarray):
|
||||
frame_array = frame
|
||||
@@ -715,7 +718,7 @@ def main():
|
||||
video_tensor = video_tensor[0]
|
||||
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
|
||||
video_tensor = video_tensor.permute(1, 2, 3, 0)
|
||||
video_tensor = _normalize_float_tensor(video_tensor)
|
||||
video_tensor = _normalize_float_tensor(video_tensor, output_tensor_range)
|
||||
video_array = video_tensor.float().numpy()
|
||||
elif isinstance(frames, np.ndarray):
|
||||
video_array = frames
|
||||
|
||||
@@ -70,10 +70,22 @@ def test_lingbot_preset_matches_lingbot_defaults() -> None:
|
||||
def test_normalize_float_tensor_preserves_zero_to_one_frames() -> None:
|
||||
frames = torch.tensor([0.0, 0.25, 1.0, 1.5])
|
||||
|
||||
assert torch.equal(_normalize_float_tensor(frames), torch.tensor([0.0, 0.25, 1.0, 1.0]))
|
||||
assert torch.equal(_normalize_float_tensor(frames, "zero_to_one"), torch.tensor([0.0, 0.25, 1.0, 1.0]))
|
||||
|
||||
|
||||
def test_normalize_float_tensor_maps_negative_one_to_one_frames() -> None:
|
||||
frames = torch.tensor([-1.5, -1.0, 0.0, 1.0, 1.5])
|
||||
|
||||
assert torch.equal(_normalize_float_tensor(frames), torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0]))
|
||||
assert torch.equal(
|
||||
_normalize_float_tensor(frames, "negative_one_to_one"),
|
||||
torch.tensor([0.0, 0.0, 0.5, 1.0, 1.0]),
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_float_tensor_uses_one_contract_for_ambiguous_frames() -> None:
|
||||
frames = [torch.tensor([0.0, 0.5, 1.0]), torch.tensor([-1.0, 0.0, 1.0])]
|
||||
|
||||
normalized = [_normalize_float_tensor(frame, "negative_one_to_one") for frame in frames]
|
||||
|
||||
assert torch.equal(normalized[0], torch.tensor([0.5, 0.75, 1.0]))
|
||||
assert torch.equal(normalized[1], torch.tensor([0.0, 0.5, 1.0]))
|
||||
@@ -17,6 +17,7 @@ from vllm_omni.model_extras import (
|
||||
build_x_to_text_prompt,
|
||||
get_extra_body_params,
|
||||
get_extra_output_params,
|
||||
get_output_tensor_range,
|
||||
get_x_to_text_model_family,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
)
|
||||
@@ -134,6 +135,8 @@ def test_lingbot_extra_registry_declares_request_params() -> None:
|
||||
}
|
||||
)
|
||||
assert get_extra_output_params("LingBotVideoPipeline") == frozenset()
|
||||
assert get_output_tensor_range("LingBotVideoPipeline") == "zero_to_one"
|
||||
assert get_output_tensor_range("WanPipeline") == "negative_one_to_one"
|
||||
assert should_init_extra_args_for_non_diffusion_stages("LingBotVideoPipeline") is False
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm_omni.model_extras.registry import (
|
||||
get_extra_body_params,
|
||||
get_extra_output_params,
|
||||
get_model_class_name,
|
||||
get_output_tensor_range,
|
||||
get_x_to_text_model_family,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
)
|
||||
@@ -21,6 +22,7 @@ __all__ = [
|
||||
"get_extra_body_params",
|
||||
"get_extra_output_params",
|
||||
"get_model_class_name",
|
||||
"get_output_tensor_range",
|
||||
"get_x_to_text_model_family",
|
||||
"should_init_extra_args_for_non_diffusion_stages",
|
||||
]
|
||||
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -93,6 +93,7 @@ ImageToVideoPromptBuilder = Callable[
|
||||
dict[str, Any],
|
||||
]
|
||||
XToTextPromptBuilder = Callable[[str, str, bool], tuple[dict[str, Any], list[int] | None]]
|
||||
OutputTensorRange = Literal["negative_one_to_one", "zero_to_one"]
|
||||
|
||||
|
||||
def default_x_to_text_prompt(
|
||||
@@ -193,6 +194,7 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = {
|
||||
},
|
||||
"LingBotVideoPipeline": {
|
||||
"extra_body_params": LINGBOT_VIDEO_EXTRA_BODY_PARAMS,
|
||||
"output_tensor_range": "zero_to_one",
|
||||
# Shared T2I/I2V envelopes select the output modality. LingBot's
|
||||
# pipeline owns model-specific validation and normalization.
|
||||
},
|
||||
@@ -262,6 +264,18 @@ def get_extra_output_params(model_class_name: str | None) -> frozenset[str]:
|
||||
return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset()
|
||||
|
||||
|
||||
def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange:
|
||||
"""Return the declared range for floating-point tensor outputs.
|
||||
|
||||
The default preserves the shared examples' historical handling. Pipelines
|
||||
that already return normalized tensors declare ``zero_to_one`` explicitly.
|
||||
"""
|
||||
spec = _get_spec(model_class_name)
|
||||
if spec is None:
|
||||
return "negative_one_to_one"
|
||||
return spec.get("output_tensor_range", "negative_one_to_one")
|
||||
|
||||
|
||||
def should_init_extra_args_for_non_diffusion_stages(model_class_name: str | None) -> bool:
|
||||
spec = _get_spec(model_class_name)
|
||||
return bool(spec and spec.get("init_extra_args_for_non_diffusion_stages", False))
|
||||
|
||||
Reference in new issue
Block a user