Cosmos3 video to video generation (#4266)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
This commit is contained in:
MaciejBalaNV authored and GitHub committed 2026-06-09 03:04:05 +08:00
1 parent 9c9d4ed4a8
commit 2f25195750
14 files changed
+1037 -94

No files matched your search

@@ -13,12 +13,14 @@ import threading
import time
from types import SimpleNamespace
import numpy as np
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from PIL import Image
from pytest_mock import MockerFixture
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
from vllm_omni.entrypoints.openai import api_server
from vllm_omni.entrypoints.openai.api_server import router
from vllm_omni.entrypoints.openai.protocol.videos import (
@@ -84,7 +86,8 @@ class BlockingVideoHandler:
if self.stage_configs is None:
self.stage_configs = stage_configs
async def generate_video_bytes(self, request, reference_id, *, reference_image=None):
async def generate_video_bytes(self, request, reference_id, *, reference_image=None, reference_video=None):
del request, reference_id, reference_image, reference_video
self.started.set()
try:
await asyncio.Future()
@@ -130,6 +133,30 @@ def _make_test_image_data_url(size=(64, 64)) -> str:
return f"data:image/png;base64,{encoded}"
def _make_test_video_bytes(size=(32, 24), num_frames=3) -> bytes:
width, height = size
frames = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
for idx in range(num_frames):
frames[idx, :, :, 0] = idx * 40
frames[idx, :, :, 1] = 128
frames[idx, :, :, 2] = 255 - idx * 40
return mux_video_audio_bytes(frames, fps=8, video_codec_options={"preset": "ultrafast", "threads": "0"})
def _make_test_video_data_url(size=(32, 24), num_frames=3) -> str:
encoded = base64.b64encode(_make_test_video_bytes(size, num_frames)).decode("utf-8")
return f"data:video/mp4;base64,{encoded}"
def _cosmos3_stage_configs():
return [
SimpleNamespace(
stage_type="diffusion",
engine_args=SimpleNamespace(model_class_name="Cosmos3OmniDiffusersPipeline"),
)
]
def _wait_for_status(client: TestClient, video_id: str, status: str, timeout_s: float = 2.0):
deadline = time.time() + timeout_s
last_payload = None
@@ -332,6 +359,104 @@ def test_i2v_video_generation_with_image_reference_form(test_client, mocker: Moc
assert input_image.size == (40, 24)
def test_v2v_video_generation_form(test_client, mocker: MockerFixture):
video_bytes = _make_test_video_bytes((32, 24), num_frames=3)
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
data={"prompt": "Continue this motion."},
files={"input_reference": ("input.mp4", video_bytes, "video/mp4")},
)
assert response.status_code == 200
video_id = response.json()["id"]
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
engine = test_client.app.state.openai_serving_video._engine_client
prompt = engine.captured_prompt
assert "multi_modal_data" in prompt
assert "video" in prompt["multi_modal_data"]
input_video = prompt["multi_modal_data"]["video"]
assert len(input_video) == 3
assert all(isinstance(frame, Image.Image) for frame in input_video)
assert input_video[0].size == (32, 24)
def test_v2v_video_generation_with_video_reference_form(test_client, mocker: MockerFixture):
mocker.patch(
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
return_value=b"fake-video",
)
response = test_client.post(
"/v1/videos",
data={
"prompt": "Continue this motion.",
"video_reference": json.dumps({"video_url": _make_test_video_data_url((32, 24), 2)}),
},
)
assert response.status_code == 200
video_id = response.json()["id"]
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
engine = test_client.app.state.openai_serving_video._engine_client
input_video = engine.captured_prompt["multi_modal_data"]["video"]
assert len(input_video) == 2
assert input_video[0].size == (32, 24)
def test_decode_video_bytes_can_keep_last_frames():
from vllm_omni.entrypoints.openai.video_api_utils import _decode_video_bytes
frames = _decode_video_bytes(
_make_test_video_bytes((32, 24), num_frames=6),
source="input_reference",
max_frames=2,
keep="last",
)
assert len(frames) == 2
red_means = [np.asarray(frame)[:, :, 0].mean() for frame in frames]
assert red_means[0] > 100
assert red_means[1] > red_means[0]
def test_cosmos3_reference_video_limit_uses_v2v_condition_frames():
request = VideoGenerationRequest(
prompt="Continue this motion.",
num_frames=189,
extra_params={"condition_frame_indexes_vision": [0, 2]},
)
spec = api_server._reference_video_decode_spec(request, _cosmos3_stage_configs())
assert spec.max_frames == 9
assert spec.keep == "first"
def test_cosmos3_reference_video_limit_preserves_action_frames():
request = VideoGenerationRequest(
prompt="Predict the action.",
num_frames=17,
extra_params={"action_mode": "inverse_dynamics", "action_chunk_size": 16},
)
assert api_server._reference_video_decode_spec(request, _cosmos3_stage_configs()).max_frames == 17
def test_cosmos3_reference_video_limit_caps_condition_frames_to_output_frames():
request = VideoGenerationRequest(
prompt="Continue this motion.",
num_frames=5,
extra_params={"condition_frame_indexes_vision": [0, 20]},
)
assert api_server._reference_video_decode_spec(request, _cosmos3_stage_configs()).max_frames == 5
def test_seconds_defaults_fps_and_frames(test_client, mocker: MockerFixture):
fps_values = []
@@ -794,7 +919,20 @@ def test_rejects_input_reference_and_image_reference_together(test_client):
files={"input_reference": ("input.png", _make_test_image_bytes(), "image/png")},
)
assert response.status_code == 400
assert "either input_reference or image_reference" in response.json()["detail"].lower()
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
def test_rejects_image_reference_and_video_reference_together(test_client):
response = test_client.post(
"/v1/videos",
data={
"prompt": "bad refs",
"image_reference": '{"image_url": "https://example.com/cat.png"}',
"video_reference": '{"video_url": "https://example.com/cat.mp4"}',
},
)
assert response.status_code == 400
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
def test_invalid_seconds_returns_422(test_client):
@@ -856,6 +994,18 @@ def test_unsupported_image_reference_file_id_returns_400(test_client):
assert response.json()["detail"] == "Invalid image_reference: file_id is not supported yet."
def test_unsupported_video_reference_file_id_returns_400(test_client):
response = test_client.post(
"/v1/videos",
data={
"prompt": "unsupported ref",
"video_reference": '{"file_id": "file-123"}',
},
)
assert response.status_code == 400
assert response.json()["detail"] == "Invalid video_reference: file_id is not supported yet."
def test_invalid_uploaded_input_reference_returns_400(test_client):
response = test_client.post(
"/v1/videos",
@@ -863,7 +1013,7 @@ def test_invalid_uploaded_input_reference_returns_400(test_client):
files={"input_reference": ("input.png", b"not-an-image", "image/png")},
)
assert response.status_code == 400
assert response.json()["detail"] == "Invalid input_reference: provided content is not a valid image."
assert response.json()["detail"] == "Invalid input_reference: provided content is not a valid image or video."
def test_video_request_validation():
@@ -880,6 +1030,8 @@ def test_video_request_validation():
with pytest.raises(ValueError):
VideoGenerationRequest(prompt="test", image_reference={"file_id": "file-1", "image_url": "https://example.com"})
with pytest.raises(ValueError):
VideoGenerationRequest(prompt="test", video_reference={"file_id": "file-1", "video_url": "https://example.com"})
with pytest.raises(ValueError):
VideoGenerationRequest(prompt="test", frame_interpolation_exp=0)
with pytest.raises(ValueError):
@@ -1265,6 +1417,23 @@ def test_sync_i2v_with_image_reference(test_client, mocker: MockerFixture):
assert response.content == b"ref-video"
def test_sync_v2v_returns_video_bytes(test_client, mocker: MockerFixture):
video_bytes = _make_test_video_bytes((32, 24), num_frames=3)
_mock_encode_video_bytes(mocker, b"v2v-video-data")
response = test_client.post(
"/v1/videos/sync",
data={"prompt": "Continue this motion."},
files={"input_reference": ("input.mp4", video_bytes, "video/mp4")},
)
assert response.status_code == 200
assert response.content == b"v2v-video-data"
engine = test_client.app.state.openai_serving_video._engine_client
input_video = engine.captured_prompt["multi_modal_data"]["video"]
assert len(input_video) == 3
assert input_video[0].size == (32, 24)
def test_sync_missing_handler_returns_503():
app = FastAPI()
app.include_router(router)
@@ -1297,7 +1466,7 @@ def test_sync_rejects_both_references(test_client):
files={"input_reference": ("input.png", _make_test_image_bytes(), "image/png")},
)
assert response.status_code == 400
assert "either input_reference or image_reference" in response.json()["detail"].lower()
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
def test_sync_generation_error_returns_500(test_client, mocker: MockerFixture):