Cosmos3 transfer (#4379)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
This commit is contained in:
1 parent
cead615288
commit
a1e1ac0155
12 files changed
+2003
-58
No files matched your search
@@ -28,6 +28,29 @@ from vllm_omni.entrypoints.openai.protocol.videos import (
|
||||
)
|
||||
|
||||
|
||||
class VideoFrames(list[Image.Image]):
|
||||
"""Decoded video frames plus source metadata."""
|
||||
|
||||
def __init__(self, frames: list[Image.Image] | None = None, *, fps: float | None = None) -> None:
|
||||
super().__init__(frames or [])
|
||||
self.fps = fps
|
||||
self.frame_rate = fps
|
||||
|
||||
|
||||
def positive_float(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
if hasattr(value, "item") and not isinstance(value, (bytes, str)):
|
||||
value = value.item()
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if result <= 0:
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
def _decode_image_bytes(image_bytes: bytes, *, source: str) -> Image.Image:
|
||||
try:
|
||||
return Image.open(BytesIO(image_bytes)).convert("RGB")
|
||||
@@ -41,7 +64,7 @@ def _decode_video_bytes(
|
||||
source: str,
|
||||
max_frames: int | None = None,
|
||||
keep: Literal["first", "last"] = "first",
|
||||
) -> list[Image.Image]:
|
||||
) -> VideoFrames:
|
||||
try:
|
||||
import av
|
||||
except ImportError as exc: # pragma: no cover - av is a serving dependency via media_utils
|
||||
@@ -56,8 +79,16 @@ def _decode_video_bytes(
|
||||
tail_frames: deque[Image.Image] | None = (
|
||||
deque(maxlen=max_frames) if keep == "last" and max_frames is not None else None
|
||||
)
|
||||
fps: float | None = None
|
||||
try:
|
||||
with av.open(BytesIO(video_bytes)) as container:
|
||||
video_stream = container.streams.video[0] if container.streams.video else None
|
||||
if video_stream is not None:
|
||||
fps = (
|
||||
positive_float(getattr(video_stream, "average_rate", None))
|
||||
or positive_float(getattr(video_stream, "base_rate", None))
|
||||
or positive_float(getattr(video_stream, "guessed_rate", None))
|
||||
)
|
||||
for frame in container.decode(video=0):
|
||||
image = frame.to_image().convert("RGB")
|
||||
if tail_frames is not None:
|
||||
@@ -73,7 +104,7 @@ def _decode_video_bytes(
|
||||
frames = list(tail_frames)
|
||||
if not frames:
|
||||
raise InvalidInputReferenceError(f"Invalid {source}: provided content is not a valid video.")
|
||||
return frames
|
||||
return VideoFrames(frames, fps=fps)
|
||||
|
||||
|
||||
def _decode_media_bytes(
|
||||
@@ -82,7 +113,7 @@ def _decode_media_bytes(
|
||||
source: str,
|
||||
max_video_frames: int | None = None,
|
||||
video_keep: Literal["first", "last"] = "first",
|
||||
) -> Image.Image | list[Image.Image]:
|
||||
) -> Image.Image | VideoFrames:
|
||||
try:
|
||||
return _decode_image_bytes(media_bytes, source=source)
|
||||
except InvalidInputReferenceError:
|
||||
@@ -138,7 +169,7 @@ def _decode_base64_video(
|
||||
source: str,
|
||||
max_frames: int | None = None,
|
||||
keep: Literal["first", "last"] = "first",
|
||||
) -> list[Image.Image]:
|
||||
) -> VideoFrames:
|
||||
if video_reference:
|
||||
if video_reference.startswith("data:video"):
|
||||
_, b64_data = video_reference.split(",", 1)
|
||||
@@ -158,7 +189,7 @@ async def decode_video_url(
|
||||
*,
|
||||
max_frames: int | None = None,
|
||||
keep: Literal["first", "last"] = "first",
|
||||
) -> list[Image.Image]:
|
||||
) -> VideoFrames:
|
||||
if video_url.startswith("data:video"):
|
||||
return _decode_base64_video(
|
||||
video_url,
|
||||
@@ -240,7 +271,7 @@ async def decode_input_reference(
|
||||
*,
|
||||
max_video_frames: int | None = None,
|
||||
video_keep: Literal["first", "last"] = "first",
|
||||
) -> Image.Image | list[Image.Image] | None:
|
||||
) -> Image.Image | VideoFrames | None:
|
||||
"""Decode media input from multipart bytes, data URLs, or typed references."""
|
||||
|
||||
provided = sum(item is not None for item in (input_reference_bytes, image_reference, video_reference))
|
||||
|
||||
Reference in new issue
Block a user