From bc0c9f4b45c45c59aa2f92471842e8c18ae403ca Mon Sep 17 00:00:00 2001 From: junpengw67-max Date: Sat, 12 Sep 2026 12:23:24 +0800 Subject: [PATCH] [Bugfix][Model][Lance] Support decoded video frames in video editing (#5128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 吴俊鹏 <248679769+junpengw67-max@users.noreply.github.com> Co-authored-by: 吴俊鹏 <248679769+junpengw67-max@users.noreply.github.com> --- .../models/lance/test_pipeline_lance.py | 171 ++++++++++++++++++ .../diffusion/models/lance/pipeline_lance.py | 3 + vllm_omni/model_executor/models/utils.py | 51 +++++- 3 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 tests/diffusion/models/lance/test_pipeline_lance.py diff --git a/tests/diffusion/models/lance/test_pipeline_lance.py b/tests/diffusion/models/lance/test_pipeline_lance.py new file mode 100644 index 000000000..32ff9bd10 --- /dev/null +++ b/tests/diffusion/models/lance/test_pipeline_lance.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import numpy as np +import pytest +from PIL import Image + +from vllm_omni.diffusion.models.lance.lance_transformer import LanceBagel +from vllm_omni.diffusion.models.lance.pipeline_lance import LancePipeline +from vllm_omni.model_executor.models.utils import normalize_decoded_video_frames + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class FramesWithMetadata(list[Image.Image]): + def __init__(self, frames: list[Image.Image], **metadata: object) -> None: + super().__init__(frames) + for name, value in metadata.items(): + setattr(self, name, value) + + +class StopAfterVideoPreprocessError(Exception): + pass + + +def test_normalize_decoded_video_frames_preserves_rgb_pixels_and_fps(): + red = Image.new("RGB", (3, 2), color=(255, 0, 0)) + rgba = Image.new("RGBA", (3, 2), color=(0, 255, 0, 64)) + rgba_bytes = rgba.tobytes() + + video, fps = normalize_decoded_video_frames(FramesWithMetadata([red, rgba], fps=8), default_fps=24.0) + + assert video.shape == (2, 2, 3, 3) + assert video.dtype == np.uint8 + assert video.flags.c_contiguous + np.testing.assert_array_equal(video[0, 0, 0], [255, 0, 0]) + np.testing.assert_array_equal(video[1, 0, 0], [0, 255, 0]) + assert fps == 8.0 + assert rgba.mode == "RGBA" + assert rgba.tobytes() == rgba_bytes + + +def test_normalize_decoded_video_frames_uses_frame_rate_when_fps_is_missing(): + frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, frame_rate=12) + + _, fps = normalize_decoded_video_frames(frames, default_fps=24.0) + + assert fps == 12.0 + + +def test_normalize_decoded_video_frames_prefers_valid_fps_over_frame_rate(): + frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps=8, frame_rate=12) + + _, fps = normalize_decoded_video_frames(frames, default_fps=24.0) + + assert fps == 8.0 + + +def test_normalize_decoded_video_frames_uses_default_fps_for_plain_list_and_tuple(): + frames = [Image.new("RGB", (2, 2)), Image.new("RGB", (2, 2))] + + assert normalize_decoded_video_frames(frames, default_fps=24.0)[1] == 24.0 + assert normalize_decoded_video_frames(tuple(frames), default_fps=24.0)[1] == 24.0 + + +def test_normalize_decoded_video_frames_accepts_numeric_string_fps(): + frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps="12.5") + + _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0) + + assert frame_rate == 12.5 + + +@pytest.mark.parametrize( + ("fps", "frame_rate", "expected"), + [ + (float("nan"), 12, 12.0), + (0, 15, 15.0), + (-1, 8, 8.0), + (float("inf"), 10, 10.0), + ("invalid", "6.5", 6.5), + ], +) +def test_normalize_decoded_video_frames_uses_valid_frame_rate_after_invalid_fps(fps, frame_rate, expected): + frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps=fps, frame_rate=frame_rate) + + _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0) + + assert frame_rate == expected + + +def test_normalize_decoded_video_frames_uses_default_when_both_metadata_values_are_invalid(): + frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps="invalid", frame_rate=0) + + _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0) + + assert frame_rate == 24.0 + + +@pytest.mark.parametrize( + ("frames", "match"), + [ + ([], "empty decoded video frame sequence"), + ([Image.new("RGB", (2, 2)), object()], "must be a PIL.Image.Image"), + ( + [Image.new("RGB", (2, 2)), Image.new("RGB", (3, 2))], + "must have identical dimensions", + ), + ], +) +def test_normalize_decoded_video_frames_rejects_invalid_sequences(frames, match): + with pytest.raises(ValueError, match=match): + normalize_decoded_video_frames(frames, default_fps=24.0) + + +@pytest.mark.parametrize( + ("frames", "match"), + [ + ([Image.new("RGB", (0, 16))], r"index 0.*0x16"), + ([Image.new("RGB", (16, 0))], r"index 0.*16x0"), + ([Image.new("RGB", (0, 0))], r"index 0.*0x0"), + ( + [Image.new("RGB", (16, 16)), Image.new("RGB", (0, 16))], + r"index 1.*0x16", + ), + ], +) +def test_normalize_decoded_video_frames_rejects_zero_dimensions(frames, match): + with pytest.raises(ValueError, match=match): + normalize_decoded_video_frames(frames, default_fps=24.0) + + +@pytest.mark.parametrize( + ("video_input", "expected_fps"), + [ + (FramesWithMetadata([Image.new("RGB", (3, 2))] * 2, fps=8), 8.0), + (tuple([Image.new("RGBA", (3, 2))] * 2), 12.0), + ], +) +def test_lance_video_edit_normalizes_decoded_frame_sequences(monkeypatch, video_input, expected_fps): + captured = {} + + def capture_preprocessed_input(video, fps): + captured["video"] = video + captured["fps"] = fps + raise StopAfterVideoPreprocessError + + monkeypatch.setattr(LanceBagel, "_lance_video_preprocess", staticmethod(capture_preprocessed_input)) + request = SimpleNamespace( + prompts=[ + { + "modalities": ["video"], + "user_text": "edit the video", + "multi_modal_data": {"video": video_input}, + "extra_args": {"origin_fps": 12.0}, + } + ], + sampling_params=SimpleNamespace(extra_args={}, num_inference_steps=1), + ) + + with pytest.raises(StopAfterVideoPreprocessError): + object.__new__(LancePipeline)._forward_video_edit(request) + + video = captured["video"] + assert isinstance(video, np.ndarray) + assert video.shape == (2, 2, 3, 3) + assert video.dtype == np.uint8 + assert video.flags.c_contiguous + assert captured["fps"] == expected_fps diff --git a/vllm_omni/diffusion/models/lance/pipeline_lance.py b/vllm_omni/diffusion/models/lance/pipeline_lance.py index dc885ac44..f33a35b96 100644 --- a/vllm_omni/diffusion/models/lance/pipeline_lance.py +++ b/vllm_omni/diffusion/models/lance/pipeline_lance.py @@ -62,6 +62,7 @@ from vllm_omni.diffusion.models.bagel.pipeline_bagel import ( from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) +from vllm_omni.model_executor.models.utils import normalize_decoded_video_frames from .lance_transformer import ( LanceBagel, @@ -1454,6 +1455,8 @@ class LancePipeline(BagelPipeline): frames_bgr.append(_cv2.cvtColor(f, _cv2.COLOR_BGR2RGB)) cap.release() video_raw = _np.stack(frames_bgr, axis=0) + elif isinstance(video_input, (list, tuple)): + video_raw, origin_fps = normalize_decoded_video_frames(video_input, default_fps=origin_fps_default) elif isinstance(video_input, _np.ndarray): video_raw = video_input origin_fps = origin_fps_default diff --git a/vllm_omni/model_executor/models/utils.py b/vllm_omni/model_executor/models/utils.py index fd9ddd41b..73e9d5572 100644 --- a/vllm_omni/model_executor/models/utils.py +++ b/vllm_omni/model_executor/models/utils.py @@ -1,10 +1,59 @@ -from collections.abc import Callable +import math +from collections.abc import Callable, Sequence from contextlib import contextmanager +import numpy as np import torch +from PIL import Image from vllm.model_executor.models.utils import maybe_prefix +def normalize_decoded_video_frames( + video_input: Sequence[Image.Image], + *, + default_fps: float, +) -> tuple[np.ndarray, float]: + """Convert decoded image frames to a contiguous THWC uint8 video array.""" + if not video_input: + raise ValueError("video_edit received an empty decoded video frame sequence.") + + frame_rate = float(default_fps) + for fps in ( + getattr(video_input, "fps", None), + getattr(video_input, "frame_rate", None), + ): + try: + candidate = float(fps) + except (TypeError, ValueError): + continue + if math.isfinite(candidate) and candidate > 0: + frame_rate = candidate + break + + frames = [] + frame_size = None + for index, frame in enumerate(video_input): + if not isinstance(frame, Image.Image): + raise ValueError( + f"video_edit decoded video frame at index {index} must be a PIL.Image.Image, got {type(frame)}." + ) + width, height = frame.size + if width <= 0 or height <= 0: + raise ValueError( + f"video_edit decoded video frame at index {index} must have positive dimensions, got {width}x{height}." + ) + if frame_size is None: + frame_size = frame.size + elif frame.size != frame_size: + raise ValueError( + "video_edit decoded video frames must have identical dimensions; " + f"frame 0 is {frame_size}, frame {index} is {frame.size}." + ) + frames.append(np.asarray(frame.convert("RGB"), dtype=np.uint8)) + + return np.ascontiguousarray(np.stack(frames)), frame_rate + + @contextmanager def transformers_keys_to_ignore_compat(): """Make ``trust_remote_code`` weight loading robust to the transformers 5.9