[Bugfix][Model][Lance] Support decoded video frames in video editing (#5128)

Signed-off-by: 吴俊鹏 <248679769+junpengw67-max@users.noreply.github.com>
Co-authored-by: 吴俊鹏 <248679769+junpengw67-max@users.noreply.github.com>

tests/diffusion/models/lance/test_pipeline_lance.py

@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from vllm_omni.diffusion.models.lance.lance_transformer import LanceBagel
+from vllm_omni.diffusion.models.lance.pipeline_lance import LancePipeline
+from vllm_omni.model_executor.models.utils import normalize_decoded_video_frames
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class FramesWithMetadata(list[Image.Image]):
+    def __init__(self, frames: list[Image.Image], **metadata: object) -> None:
+        super().__init__(frames)
+        for name, value in metadata.items():
+            setattr(self, name, value)
+
+
+class StopAfterVideoPreprocessError(Exception):
+    pass
+
+
+def test_normalize_decoded_video_frames_preserves_rgb_pixels_and_fps():
+    red = Image.new("RGB", (3, 2), color=(255, 0, 0))
+    rgba = Image.new("RGBA", (3, 2), color=(0, 255, 0, 64))
+    rgba_bytes = rgba.tobytes()
+
+    video, fps = normalize_decoded_video_frames(FramesWithMetadata([red, rgba], fps=8), default_fps=24.0)
+
+    assert video.shape == (2, 2, 3, 3)
+    assert video.dtype == np.uint8
+    assert video.flags.c_contiguous
+    np.testing.assert_array_equal(video[0, 0, 0], [255, 0, 0])
+    np.testing.assert_array_equal(video[1, 0, 0], [0, 255, 0])
+    assert fps == 8.0
+    assert rgba.mode == "RGBA"
+    assert rgba.tobytes() == rgba_bytes
+
+
+def test_normalize_decoded_video_frames_uses_frame_rate_when_fps_is_missing():
+    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, frame_rate=12)
+
+    _, fps = normalize_decoded_video_frames(frames, default_fps=24.0)
+
+    assert fps == 12.0
+
+
+def test_normalize_decoded_video_frames_prefers_valid_fps_over_frame_rate():
+    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps=8, frame_rate=12)
+
+    _, fps = normalize_decoded_video_frames(frames, default_fps=24.0)
+
+    assert fps == 8.0
+
+
+def test_normalize_decoded_video_frames_uses_default_fps_for_plain_list_and_tuple():
+    frames = [Image.new("RGB", (2, 2)), Image.new("RGB", (2, 2))]
+
+    assert normalize_decoded_video_frames(frames, default_fps=24.0)[1] == 24.0
+    assert normalize_decoded_video_frames(tuple(frames), default_fps=24.0)[1] == 24.0
+
+
+def test_normalize_decoded_video_frames_accepts_numeric_string_fps():
+    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps="12.5")
+
+    _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0)
+
+    assert frame_rate == 12.5
+
+
+@pytest.mark.parametrize(
+    ("fps", "frame_rate", "expected"),
+    [
+        (float("nan"), 12, 12.0),
+        (0, 15, 15.0),
+        (-1, 8, 8.0),
+        (float("inf"), 10, 10.0),
+        ("invalid", "6.5", 6.5),
+    ],
+)
+def test_normalize_decoded_video_frames_uses_valid_frame_rate_after_invalid_fps(fps, frame_rate, expected):
+    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps=fps, frame_rate=frame_rate)
+
+    _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0)
+
+    assert frame_rate == expected
+
+
+def test_normalize_decoded_video_frames_uses_default_when_both_metadata_values_are_invalid():
+    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps="invalid", frame_rate=0)
+
+    _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0)
+
+    assert frame_rate == 24.0
+
+
+@pytest.mark.parametrize(
+    ("frames", "match"),
+    [
+        ([], "empty decoded video frame sequence"),
+        ([Image.new("RGB", (2, 2)), object()], "must be a PIL.Image.Image"),
+        (
+            [Image.new("RGB", (2, 2)), Image.new("RGB", (3, 2))],
+            "must have identical dimensions",
+        ),
+    ],
+)
+def test_normalize_decoded_video_frames_rejects_invalid_sequences(frames, match):
+    with pytest.raises(ValueError, match=match):
+        normalize_decoded_video_frames(frames, default_fps=24.0)
+
+
+@pytest.mark.parametrize(
+    ("frames", "match"),
+    [
+        ([Image.new("RGB", (0, 16))], r"index 0.*0x16"),
+        ([Image.new("RGB", (16, 0))], r"index 0.*16x0"),
+        ([Image.new("RGB", (0, 0))], r"index 0.*0x0"),
+        (
+            [Image.new("RGB", (16, 16)), Image.new("RGB", (0, 16))],
+            r"index 1.*0x16",
+        ),
+    ],
+)
+def test_normalize_decoded_video_frames_rejects_zero_dimensions(frames, match):
+    with pytest.raises(ValueError, match=match):
+        normalize_decoded_video_frames(frames, default_fps=24.0)
+
+
+@pytest.mark.parametrize(
+    ("video_input", "expected_fps"),
+    [
+        (FramesWithMetadata([Image.new("RGB", (3, 2))] * 2, fps=8), 8.0),
+        (tuple([Image.new("RGBA", (3, 2))] * 2), 12.0),
+    ],
+)
+def test_lance_video_edit_normalizes_decoded_frame_sequences(monkeypatch, video_input, expected_fps):
+    captured = {}
+
+    def capture_preprocessed_input(video, fps):
+        captured["video"] = video
+        captured["fps"] = fps
+        raise StopAfterVideoPreprocessError
+
+    monkeypatch.setattr(LanceBagel, "_lance_video_preprocess", staticmethod(capture_preprocessed_input))
+    request = SimpleNamespace(
+        prompts=[
+            {
+                "modalities": ["video"],
+                "user_text": "edit the video",
+                "multi_modal_data": {"video": video_input},
+                "extra_args": {"origin_fps": 12.0},
+            }
+        ],
+        sampling_params=SimpleNamespace(extra_args={}, num_inference_steps=1),
+    )
+
+    with pytest.raises(StopAfterVideoPreprocessError):
+        object.__new__(LancePipeline)._forward_video_edit(request)
+
+    video = captured["video"]
+    assert isinstance(video, np.ndarray)
+    assert video.shape == (2, 2, 3, 3)
+    assert video.dtype == np.uint8
+    assert video.flags.c_contiguous
+    assert captured["fps"] == expected_fps

vllm_omni/diffusion/models/lance/pipeline_lance.py

@@ -62,6 +62,7 @@ from vllm_omni.diffusion.models.bagel.pipeline_bagel import (
 from vllm_omni.model_executor.model_loader.weight_utils import (
     download_weights_from_hf_specific,
 )
+from vllm_omni.model_executor.models.utils import normalize_decoded_video_frames
 
 from .lance_transformer import (
     LanceBagel,
@@ -1454,6 +1455,8 @@ class LancePipeline(BagelPipeline):
                     frames_bgr.append(_cv2.cvtColor(f, _cv2.COLOR_BGR2RGB))
                 cap.release()
                 video_raw = _np.stack(frames_bgr, axis=0)
+        elif isinstance(video_input, (list, tuple)):
+            video_raw, origin_fps = normalize_decoded_video_frames(video_input, default_fps=origin_fps_default)
         elif isinstance(video_input, _np.ndarray):
             video_raw = video_input
             origin_fps = origin_fps_default

vllm_omni/model_executor/models/utils.py

@@ -1,10 +1,59 @@
-from collections.abc import Callable
+import math
+from collections.abc import Callable, Sequence
 from contextlib import contextmanager
 
+import numpy as np
 import torch
+from PIL import Image
 from vllm.model_executor.models.utils import maybe_prefix
 
 
+def normalize_decoded_video_frames(
+    video_input: Sequence[Image.Image],
+    *,
+    default_fps: float,
+) -> tuple[np.ndarray, float]:
+    """Convert decoded image frames to a contiguous THWC uint8 video array."""
+    if not video_input:
+        raise ValueError("video_edit received an empty decoded video frame sequence.")
+
+    frame_rate = float(default_fps)
+    for fps in (
+        getattr(video_input, "fps", None),
+        getattr(video_input, "frame_rate", None),
+    ):
+        try:
+            candidate = float(fps)
+        except (TypeError, ValueError):
+            continue
+        if math.isfinite(candidate) and candidate > 0:
+            frame_rate = candidate
+            break
+
+    frames = []
+    frame_size = None
+    for index, frame in enumerate(video_input):
+        if not isinstance(frame, Image.Image):
+            raise ValueError(
+                f"video_edit decoded video frame at index {index} must be a PIL.Image.Image, got {type(frame)}."
+            )
+        width, height = frame.size
+        if width <= 0 or height <= 0:
+            raise ValueError(
+                f"video_edit decoded video frame at index {index} must have positive dimensions, got {width}x{height}."
+            )
+        if frame_size is None:
+            frame_size = frame.size
+        elif frame.size != frame_size:
+            raise ValueError(
+                "video_edit decoded video frames must have identical dimensions; "
+                f"frame 0 is {frame_size}, frame {index} is {frame.size}."
+            )
+        frames.append(np.asarray(frame.convert("RGB"), dtype=np.uint8))
+
+    return np.ascontiguousarray(np.stack(frames)), frame_rate
+
+
 @contextmanager
 def transformers_keys_to_ignore_compat():
     """Make ``trust_remote_code`` weight loading robust to the transformers 5.9