Cosmos3 transfer (#4379)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
This commit is contained in:
MaciejBalaNV authored and GitHub committed 2026-06-21 09:59:18 +08:00
1 parent cead615288
commit a1e1ac0155
12 files changed
+2003 -58

No files matched your search

@@ -28,6 +28,29 @@ from vllm_omni.entrypoints.openai.protocol.videos import (
)
class VideoFrames(list[Image.Image]):
"""Decoded video frames plus source metadata."""
def __init__(self, frames: list[Image.Image] | None = None, *, fps: float | None = None) -> None:
super().__init__(frames or [])
self.fps = fps
self.frame_rate = fps
def positive_float(value: Any) -> float | None:
if value is None:
return None
if hasattr(value, "item") and not isinstance(value, (bytes, str)):
value = value.item()
try:
result = float(value)
except (TypeError, ValueError):
return None
if result <= 0:
return None
return result
def _decode_image_bytes(image_bytes: bytes, *, source: str) -> Image.Image:
try:
return Image.open(BytesIO(image_bytes)).convert("RGB")
@@ -41,7 +64,7 @@ def _decode_video_bytes(
source: str,
max_frames: int | None = None,
keep: Literal["first", "last"] = "first",
) -> list[Image.Image]:
) -> VideoFrames:
try:
import av
except ImportError as exc: # pragma: no cover - av is a serving dependency via media_utils
@@ -56,8 +79,16 @@ def _decode_video_bytes(
tail_frames: deque[Image.Image] | None = (
deque(maxlen=max_frames) if keep == "last" and max_frames is not None else None
)
fps: float | None = None
try:
with av.open(BytesIO(video_bytes)) as container:
video_stream = container.streams.video[0] if container.streams.video else None
if video_stream is not None:
fps = (
positive_float(getattr(video_stream, "average_rate", None))
or positive_float(getattr(video_stream, "base_rate", None))
or positive_float(getattr(video_stream, "guessed_rate", None))
)
for frame in container.decode(video=0):
image = frame.to_image().convert("RGB")
if tail_frames is not None:
@@ -73,7 +104,7 @@ def _decode_video_bytes(
frames = list(tail_frames)
if not frames:
raise InvalidInputReferenceError(f"Invalid {source}: provided content is not a valid video.")
return frames
return VideoFrames(frames, fps=fps)
def _decode_media_bytes(
@@ -82,7 +113,7 @@ def _decode_media_bytes(
source: str,
max_video_frames: int | None = None,
video_keep: Literal["first", "last"] = "first",
) -> Image.Image | list[Image.Image]:
) -> Image.Image | VideoFrames:
try:
return _decode_image_bytes(media_bytes, source=source)
except InvalidInputReferenceError:
@@ -138,7 +169,7 @@ def _decode_base64_video(
source: str,
max_frames: int | None = None,
keep: Literal["first", "last"] = "first",
) -> list[Image.Image]:
) -> VideoFrames:
if video_reference:
if video_reference.startswith("data:video"):
_, b64_data = video_reference.split(",", 1)
@@ -158,7 +189,7 @@ async def decode_video_url(
*,
max_frames: int | None = None,
keep: Literal["first", "last"] = "first",
) -> list[Image.Image]:
) -> VideoFrames:
if video_url.startswith("data:video"):
return _decode_base64_video(
video_url,
@@ -240,7 +271,7 @@ async def decode_input_reference(
*,
max_video_frames: int | None = None,
video_keep: Literal["first", "last"] = "first",
) -> Image.Image | list[Image.Image] | None:
) -> Image.Image | VideoFrames | None:
"""Decode media input from multipart bytes, data URLs, or typed references."""
provided = sum(item is not None for item in (input_reference_bytes, image_reference, video_reference))