From 85b0bad2d3fbc63b5c674e148e4030c442f04f0b Mon Sep 17 00:00:00 2001 From: FredHuangNV <164937677+FredHuangNV@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:38:56 -0700 Subject: [PATCH] [Frontend][Model] Add Cosmos3 control uploads to video API (#7027) Signed-off-by: Fred Huang --- docs/serving/videos_api.md | 19 ++ recipes/cosmos3/Cosmos3-Nano.md | 12 ++ .../openai_api/test_video_server.py | 183 +++++++++++++++++ vllm_omni/diffusion/model_metadata.py | 14 +- vllm_omni/entrypoints/openai/api_server.py | 186 +++++++++++++++++- vllm_omni/entrypoints/openai/serving_video.py | 29 +++ 6 files changed, 434 insertions(+), 9 deletions(-) diff --git a/docs/serving/videos_api.md b/docs/serving/videos_api.md index 5a1d8ecf2..09f023d27 100644 --- a/docs/serving/videos_api.md +++ b/docs/serving/videos_api.md @@ -68,6 +68,8 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4 | Parameter | Type | Default | Description | | ----------- | ------ | --------- | ------------- | | `input_reference` | file | null | Uploaded reference image or video for image-to-video/video-to-video requests | +| `control_reference` | file | null | Optional uploaded image/video control, up to 512 MiB, for models that declare control-upload support | +| `control_type` | string | null | Model control name associated with `control_reference`; currently Cosmos3 supports `edge`, `blur`, `depth`, `seg`, and `wsm` | | `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` or `video_reference` | | `video_reference` | string | null | JSON-encoded reference video payload; do not combine with `input_reference` or `image_reference` | | `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs | @@ -177,6 +179,23 @@ corresponding addition. `negative_metadata_mode` accepts `same`, `inverse`, or `none` and defaults to `same` for transfer. See the Cosmos3 recipe for complete examples. +A client that cannot place the control on the server filesystem can upload one +control with `control_reference` and identify it with `control_type`. The API +streams the upload to request-scoped storage, supplies its path to the model, +and removes it after synchronous or asynchronous generation completes. Other +options for the selected control can remain in `extra_params`; do not also set +`control` or `control_path` there. Uploads larger than 512 MiB are rejected. + +```bash +curl -s http://localhost:8091/v1/videos/sync \ + -F "prompt=Preserve the scene while following the world-state control" \ + -F "input_reference=@input.mp4;type=video/mp4" \ + -F "control_reference=@wsm.mp4;type=video/mp4" \ + -F "control_type=wsm" \ + -F 'extra_params={"wsm":{"control_weight":1.0}}' \ + -o output.mp4 +``` + HTTP redirects for `image_reference.image_url` follow vLLM's `VLLM_MEDIA_URL_ALLOW_REDIRECTS` setting. Before starting the server, set it to `1` (the default) to allow redirects or `0` to reject them. A redirect target diff --git a/recipes/cosmos3/Cosmos3-Nano.md b/recipes/cosmos3/Cosmos3-Nano.md index 4aaebf0be..490f5b1f4 100644 --- a/recipes/cosmos3/Cosmos3-Nano.md +++ b/recipes/cosmos3/Cosmos3-Nano.md @@ -207,6 +207,18 @@ curl -sS -X POST http://localhost:8000/v1/videos/sync \ -F 'extra_params={"depth":{"control_path":"/path/to/depth_control.mp4"},"max_frames":121,"resolution":"720","num_video_frames_per_chunk":121}' \ -o cosmos3_transfer_depth.mp4 +# The same transfer control can be uploaded by a remote client instead of +# being placed on the server filesystem. Uploads are limited to 512 MiB. +curl -sS -X POST http://localhost:8000/v1/videos/sync \ + -H "Accept: video/mp4" \ + -F "model=nvidia/Cosmos3-Nano" \ + -F "prompt=Generate a realistic scene following the provided world-state control." \ + -F "input_reference=@/path/to/input.mp4;type=video/mp4" \ + -F "control_reference=@/path/to/wsm.mp4;type=video/mp4" \ + -F "control_type=wsm" \ + -F 'extra_params={"wsm":{"control_weight":1.0},"max_frames":121,"resolution":"720","num_video_frames_per_chunk":121}' \ + -o cosmos3_transfer_wsm.mp4 + # Text-to-video-with-sound curl -sS -X POST http://localhost:8000/v1/videos/sync \ -H "Accept: video/mp4" \ diff --git a/tests/entrypoints/openai_api/test_video_server.py b/tests/entrypoints/openai_api/test_video_server.py index 969691ebc..4a5931d6c 100644 --- a/tests/entrypoints/openai_api/test_video_server.py +++ b/tests/entrypoints/openai_api/test_video_server.py @@ -75,6 +75,7 @@ class FakeAsyncOmni: self.model_class_name = "WanPipeline" self.captured_prompt = None self.captured_reference_video_bytes = None + self.captured_control_reference_bytes = {} self.captured_sampling_params_list = None def get_diffusion_od_config(self): @@ -83,6 +84,10 @@ class FakeAsyncOmni: async def generate(self, prompt, request_id, sampling_params_list): self.captured_prompt = prompt self.captured_sampling_params_list = sampling_params_list + for control_type in ("edge", "blur", "depth", "seg", "wsm"): + control_params = sampling_params_list[0].extra_args.get(control_type) + if isinstance(control_params, dict) and isinstance(control_params.get("control_path"), str): + self.captured_control_reference_bytes[control_type] = Path(control_params["control_path"]).read_bytes() reference_videos = prompt.get("multi_modal_data", {}).get("video") if ( isinstance(reference_videos, list) @@ -338,6 +343,8 @@ def _cosmos3_stage_configs(): return [ SimpleNamespace( stage_type="diffusion", + final_output=True, + final_output_type="video", engine_args=SimpleNamespace(model_class_name="Cosmos3OmniDiffusersPipeline"), ) ] @@ -919,6 +926,17 @@ def test_mixed_reference_capability_uses_model_metadata_when_config_defaults_fal assert handler.supports_mixed_reference_inputs +@pytest.mark.parametrize("model_class_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"]) +def test_control_upload_capability_is_declared_only_by_cosmos3(test_client, model_class_name): + handler = test_client.app.state.openai_serving_video + handler._engine_client.model_class_name = model_class_name + + assert handler.supported_control_upload_types == frozenset({"edge", "blur", "depth", "seg", "wsm"}) + + handler._engine_client.model_class_name = "WanPipeline" + assert handler.supported_control_upload_types == frozenset() + + def test_decode_video_bytes_can_keep_first_frames(): from vllm_omni.entrypoints.openai.video_api_utils import _decode_video_bytes @@ -2218,6 +2236,171 @@ def test_sync_v2v_returns_video_bytes(test_client, mocker: MockerFixture): assert input_video[0].size == (32, 24) +@pytest.mark.parametrize("endpoint", ["/v1/videos", "/v1/videos/sync"]) +@pytest.mark.parametrize("control_type", ["wsm", "depth"]) +def test_cosmos3_accepts_optional_uploaded_control( + endpoint, + control_type, + test_client, + mocker: MockerFixture, +): + control_bytes = f"{control_type}-control".encode() + _mock_encode_video_bytes(mocker, b"controlled-video") + engine = test_client.app.state.openai_serving_video._engine_client + engine.model_class_name = "Cosmos3OmniDiffusersPipeline" + + response = test_client.post( + endpoint, + data={ + "prompt": "Follow the uploaded control.", + "control_type": control_type, + "extra_params": json.dumps({control_type: {"control_weight": 0.75}}), + }, + files=[ + ("input_reference", ("input.mp4", _make_test_video_bytes(), "video/mp4")), + ("control_reference", (f"{control_type}.mp4", control_bytes, "video/mp4")), + ], + ) + + assert response.status_code == 200 + if endpoint.endswith("/sync"): + assert response.content == b"controlled-video" + else: + video_id = response.json()["id"] + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + + captured = engine.captured_sampling_params_list[0].extra_args[control_type] + assert captured["control_weight"] == 0.75 + assert engine.captured_control_reference_bytes[control_type] == control_bytes + assert len(engine.captured_prompt["multi_modal_data"]["video"]) == 3 + assert not Path(captured["control_path"]).exists() + + +@pytest.mark.parametrize("endpoint", ["/v1/videos", "/v1/videos/sync"]) +def test_cosmos3_uploaded_control_selects_transfer_reference_video_decode_policy( + endpoint, + test_client, + mocker: MockerFixture, +): + _mock_encode_video_bytes(mocker, b"controlled-video") + test_client.app.state.stage_configs = _cosmos3_stage_configs() + + response = test_client.post( + endpoint, + data={ + "prompt": "Preserve all conditioning motion.", + "control_type": "wsm", + "num_frames": "9", + "extra_params": json.dumps({"num_first_chunk_conditional_frames": 9}), + }, + files=[ + ( + "input_reference", + ("input.mp4", _make_test_video_bytes(num_frames=6), "video/mp4"), + ), + ("control_reference", ("wsm.mp4", b"control", "video/mp4")), + ], + ) + + assert response.status_code == 200 + if not endpoint.endswith("/sync"): + video_id = response.json()["id"] + _wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value) + + engine = test_client.app.state.openai_serving_video._engine_client + assert len(engine.captured_prompt["multi_modal_data"]["video"]) == 6 + + +def test_cosmos3_control_upload_is_optional(test_client, mocker: MockerFixture): + _mock_encode_video_bytes(mocker) + engine = test_client.app.state.openai_serving_video._engine_client + engine.model_class_name = "Cosmos3OmniDiffusersPipeline" + + response = test_client.post("/v1/videos/sync", data={"prompt": "No control for this request."}) + + assert response.status_code == 200 + assert "wsm" not in engine.captured_sampling_params_list[0].extra_args + + +def test_control_upload_does_not_affect_models_without_capability(test_client): + response = test_client.post( + "/v1/videos/sync", + data={"prompt": "Unsupported control.", "control_type": "wsm"}, + files={"control_reference": ("wsm.mp4", b"control", "video/mp4")}, + ) + + assert response.status_code == 400 + assert "not supported by this model" in response.json()["detail"] + assert test_client.app.state.openai_serving_video._engine_client.captured_prompt is None + + +@pytest.mark.parametrize( + ("data", "files", "message"), + [ + ( + {"prompt": "Missing type."}, + {"control_reference": ("wsm.mp4", b"control", "video/mp4")}, + "requires control_type", + ), + ( + {"prompt": "Missing file.", "control_type": "wsm"}, + None, + "requires a control_reference", + ), + ( + {"prompt": "Unknown type.", "control_type": "unknown"}, + {"control_reference": ("control.mp4", b"control", "video/mp4")}, + "not supported by this model", + ), + ], +) +def test_cosmos3_control_upload_validates_contract(data, files, message, test_client): + test_client.app.state.openai_serving_video._engine_client.model_class_name = "Cosmos3OmniDiffusersPipeline" + + response = test_client.post("/v1/videos/sync", data=data, files=files) + + assert response.status_code == 400 + assert message in response.json()["detail"] + + +def test_cosmos3_control_upload_rejects_existing_control_source(test_client): + test_client.app.state.openai_serving_video._engine_client.model_class_name = "Cosmos3OmniDiffusersPipeline" + response = test_client.post( + "/v1/videos/sync", + data={ + "prompt": "Ambiguous control.", + "control_type": "wsm", + "extra_params": json.dumps({"wsm": {"control_path": "/already/present.mp4"}}), + }, + files={"control_reference": ("wsm.mp4", b"control", "video/mp4")}, + ) + + assert response.status_code == 400 + assert "not both" in response.json()["detail"] + + +@pytest.mark.parametrize( + ("control_bytes", "message"), + [ + (b"", "must not be empty"), + (b"control", "size limit"), + ], +) +def test_cosmos3_control_upload_rejects_invalid_size(control_bytes, message, test_client, monkeypatch): + test_client.app.state.openai_serving_video._engine_client.model_class_name = "Cosmos3OmniDiffusersPipeline" + monkeypatch.setattr(api_server, "CONTROL_REFERENCE_MAX_BYTES", 3) + + response = test_client.post( + "/v1/videos/sync", + data={"prompt": "Invalid control.", "control_type": "wsm"}, + files={"control_reference": ("wsm.mp4", control_bytes, "video/mp4")}, + ) + + assert response.status_code == 400 + assert message in response.json()["detail"] + assert test_client.app.state.openai_serving_video._engine_client.captured_prompt is None + + def test_sync_missing_handler_returns_503(): app = FastAPI() app.include_router(router) diff --git a/vllm_omni/diffusion/model_metadata.py b/vllm_omni/diffusion/model_metadata.py index 38df83846..10d495cfd 100644 --- a/vllm_omni/diffusion/model_metadata.py +++ b/vllm_omni/diffusion/model_metadata.py @@ -11,6 +11,10 @@ class DiffusionModelMetadata: supports_multimodal_inputs: bool = False max_multimodal_image_inputs: int | None = None supports_mixed_reference_inputs: bool = False + # Multipart controls are exposed as ``control_reference`` plus + # ``control_type`` by the video API. A pipeline that opts in receives the + # persisted upload through ``extra_args[control_type]["control_path"]``. + supported_control_upload_types: tuple[str, ...] = () attention_mask_free: bool = False final_output_type: str | None = None @@ -94,8 +98,14 @@ _DIFFUSION_MODEL_METADATA: dict[str, DiffusionModelMetadata] = { "LongCatVideoAvatarPipeline": DiffusionModelMetadata(final_output_type="video"), "MagiHumanPipeline": DiffusionModelMetadata(final_output_type="video"), "DreamIDOmniPipeline": DiffusionModelMetadata(final_output_type="video"), - "Cosmos3OmniDiffusersPipeline": DiffusionModelMetadata(final_output_type="video"), - "Cosmos3OmniPipeline": DiffusionModelMetadata(final_output_type="video"), + "Cosmos3OmniDiffusersPipeline": DiffusionModelMetadata( + supported_control_upload_types=("edge", "blur", "depth", "seg", "wsm"), + final_output_type="video", + ), + "Cosmos3OmniPipeline": DiffusionModelMetadata( + supported_control_upload_types=("edge", "blur", "depth", "seg", "wsm"), + final_output_type="video", + ), "SanaVideoPipeline": DiffusionModelMetadata(final_output_type="video"), "SanaImageToVideoPipeline": DiffusionModelMetadata(final_output_type="video"), "SanaWmPipeline": DiffusionModelMetadata( diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 3d9403bf8..b305cb0cd 100755 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -15,8 +15,8 @@ import tempfile import time import uuid from argparse import Namespace -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager, suppress from http import HTTPStatus from numbers import Integral from pathlib import Path @@ -181,6 +181,9 @@ MINIMAX_H3_MAX_REFERENCE_COUNT = 12 MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"}) MINIMAX_H3_REFERENCE_VIDEO_SUFFIXES = frozenset({".mp4", ".mov"}) MINIMAX_H3_REFERENCE_AUDIO_SUFFIXES = frozenset({".wav", ".mp3"}) +CONTROL_REFERENCE_IMAGE_SUFFIXES = frozenset({".bmp", ".gif", ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp"}) +CONTROL_REFERENCE_VIDEO_SUFFIXES = frozenset({".mkv", ".mov", ".mp4", ".webm"}) +CONTROL_REFERENCE_MAX_BYTES = 512 * 1024 * 1024 profiler_router = APIRouter() @@ -3073,6 +3076,7 @@ async def _cleanup_video(video_id: str): def _cleanup_video_references( reference_video: ReferenceVideo | None, reference_audio: ReferenceAudio | None, + control_path: str | None = None, ) -> None: if reference_video is not None: for path in reference_video.cleanup_paths: @@ -3083,6 +3087,8 @@ def _cleanup_video_references( for path in cleanup_paths: if os.path.exists(path): os.unlink(path) + if control_path is not None and os.path.exists(control_path): + os.unlink(control_path) async def _run_video_generation_job( @@ -3092,11 +3098,13 @@ async def _run_video_generation_job( reference_image: ReferenceImage | None = None, reference_video: ReferenceVideo | None = None, reference_audio: ReferenceAudio | None = None, + control_path: str | None = None, app_state: Any | None = None, ) -> None: job = await VIDEO_STORE.get(video_id) if job is None: logger.warning("Video job %s missing before generation task started; skipping", video_id) + _cleanup_video_references(reference_video, reference_audio, control_path) return await VIDEO_STORE.update_fields(video_id, {"status": VideoGenerationStatus.IN_PROGRESS}) @@ -3165,7 +3173,7 @@ async def _run_video_generation_job( await VIDEO_STORE.pop(video_id) raise finally: - _cleanup_video_references(reference_video, reference_audio) + _cleanup_video_references(reference_video, reference_audio, control_path) VIDEO_SYNC_TIMEOUT_S = float(os.environ.get("VLLM_OMNI_VIDEO_SYNC_TIMEOUT", 600.0)) @@ -3191,6 +3199,121 @@ async def _persist_uploaded_video_references(uploads: list[UploadFile]) -> list[ return paths +async def _persist_uploaded_control_reference( + upload: UploadFile, + *, + max_bytes: int = CONTROL_REFERENCE_MAX_BYTES, +) -> str: + """Stream one model control upload to request-scoped local storage.""" + kind = _uploaded_media_kind(upload) + suffix = Path(upload.filename or "").suffix.lower() + supported_suffixes = CONTROL_REFERENCE_IMAGE_SUFFIXES | CONTROL_REFERENCE_VIDEO_SUFFIXES + if kind == "audio" or (suffix and suffix not in supported_suffixes): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="control_reference must be an image or video file.", + ) + if not suffix: + suffix = ".png" if kind == "image" else ".mp4" + + declared_size = getattr(upload, "size", None) + if isinstance(declared_size, Integral) and int(declared_size) > max_bytes: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail=f"control_reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.", + ) + + fd, path = tempfile.mkstemp(prefix="vllm_omni_control_reference_", suffix=suffix) + size = 0 + persisted = False + try: + with os.fdopen(fd, "wb") as output: + while chunk := await upload.read(min(1024 * 1024, max_bytes - size + 1)): + size += len(chunk) + if size > max_bytes: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail=f"control_reference exceeds the {max_bytes // (1024 * 1024)} MiB size limit.", + ) + output.write(chunk) + if size == 0: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="control_reference must not be empty.", + ) + persisted = True + return path + finally: + if not persisted: + with suppress(OSError): + os.unlink(path) + + +def _validate_control_upload( + handler: OmniOpenAIServingVideo, + request: VideoGenerationRequest, + control_reference: UploadFile | None, + control_type: str | None, +) -> str | None: + """Validate a generic uploaded control against model capabilities.""" + if control_reference is None: + if control_type is not None: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="control_type requires a control_reference upload.", + ) + return None + if control_type is None or not control_type.strip(): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="control_reference requires control_type.", + ) + + normalized_type = control_type.strip().lower() + supported_types = frozenset(getattr(handler, "supported_control_upload_types", ())) + if normalized_type not in supported_types: + supported = ", ".join(sorted(supported_types)) or "none" + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail=( + f"Control upload type '{normalized_type}' is not supported by this model. " + f"Supported control upload types: {supported}." + ), + ) + + existing = (request.extra_params or {}).get(normalized_type) + if existing is not None: + if not isinstance(existing, Mapping) and existing is not True: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail=f"extra_params.{normalized_type} must be an object when control_reference is uploaded.", + ) + if isinstance(existing, Mapping) and any(existing.get(key) is not None for key in ("control", "control_path")): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail=( + "Provide either control_reference or " + f"extra_params.{normalized_type}.control/control_path, not both." + ), + ) + return normalized_type + + +def _attach_control_upload( + request: VideoGenerationRequest, + control_type: str, + control_path: str | None = None, +) -> None: + """Declare an uploaded control and attach its persisted path when available.""" + extra_params = dict(request.extra_params or {}) + existing = extra_params.get(control_type) + control_params = dict(existing) if isinstance(existing, Mapping) else {} + if control_path is not None: + control_params["control_path"] = control_path + extra_params[control_type] = control_params + request.extra_params = extra_params + + def _reference_list(value: Any) -> list[Any]: if value is None: return [] @@ -3337,6 +3460,8 @@ async def _parse_video_form( prompt: str = Form(...), input_reference: UploadFile | None = File(default=None), input_references: list[UploadFile] | None = File(default=None), + control_reference: UploadFile | None = File(default=None), + control_type: str | None = Form(default=None), image_reference: str | None = Form(default=None), video_reference: str | None = Form(default=None), audio_reference: str | None = Form(default=None), @@ -3376,6 +3501,7 @@ async def _parse_video_form( ReferenceImage | None, ReferenceVideo | None, ReferenceAudio | None, + str | None, ]: """FastAPI dependency that parses video form data, validates inputs, resolves the handler, and decodes any reference image. @@ -3470,6 +3596,13 @@ async def _parse_video_form( detail=f"Video generation setup failed: {str(e)}", ) + normalized_control_type = _validate_control_upload(handler, request, control_reference, control_type) + if normalized_control_type is not None: + # Make the selected transfer mode visible while choosing the model's + # reference-video decode policy. The upload itself stays unpersisted + # until reference parsing succeeds, preserving failure cleanup. + _attach_control_upload(request, normalized_control_type) + supports_mixed_reference_inputs = bool(getattr(handler, "supports_mixed_reference_inputs", False)) if input_reference is not None: input_reference_bytes = await _read_upload_limited( @@ -3592,7 +3725,27 @@ async def _parse_video_form( cleanup_paths=cleanup_paths, ) - return request, handler, effective_model_name, reference_image, reference_video, reference_audio + control_path: str | None = None + if control_reference is not None and normalized_control_type is not None: + try: + control_path = await _persist_uploaded_control_reference( + control_reference, + max_bytes=CONTROL_REFERENCE_MAX_BYTES, + ) + _attach_control_upload(request, normalized_control_type, control_path) + except (asyncio.CancelledError, HTTPException, OSError, TypeError, ValueError): + _cleanup_video_references(reference_video, reference_audio, control_path) + raise + + return ( + request, + handler, + effective_model_name, + reference_image, + reference_video, + reference_audio, + control_path, + ) @router.post( @@ -3613,6 +3766,7 @@ async def create_video( ReferenceImage | None, ReferenceVideo | None, ReferenceAudio | None, + str | None, ] = Depends(_parse_video_form), ) -> VideoResponse: """Create an asynchronous video generation job. @@ -3620,7 +3774,15 @@ async def create_video( Accepts multipart form-data (see ``_parse_video_form`` for parameters), persists a queued job record, and starts generation in the background. """ - request, handler, effective_model_name, reference_image, reference_video, reference_audio = ctx + ( + request, + handler, + effective_model_name, + reference_image, + reference_video, + reference_audio, + control_path, + ) = ctx ref = video_response_from_request(effective_model_name, request) await VIDEO_STORE.upsert(ref.id, ref) task = asyncio.create_task( @@ -3631,6 +3793,7 @@ async def create_video( reference_image, reference_video, reference_audio, + control_path, app_state=raw_request.app.state, ) ) @@ -3656,6 +3819,7 @@ async def create_video_sync( ReferenceImage | None, ReferenceVideo | None, ReferenceAudio | None, + str | None, ] = Depends(_parse_video_form), ) -> Response: """Synchronous video generation endpoint. @@ -3667,7 +3831,15 @@ async def create_video_sync( Metadata is returned via response headers ``X-Request-Id``, ``X-Model``, and ``X-Inference-Time-S``. """ - request, handler, effective_model_name, reference_image, reference_video, reference_audio = ctx + ( + request, + handler, + effective_model_name, + reference_image, + reference_video, + reference_audio, + control_path, + ) = ctx request_id = f"video_sync-{random_uuid()}" raw_request.state.request_metadata = RequestResponseMetadata(request_id=request_id) started_at = time.perf_counter() @@ -3701,7 +3873,7 @@ async def create_video_sync( detail=f"Video generation failed: {str(exc)}", ) from exc finally: - _cleanup_video_references(reference_video, reference_audio) + _cleanup_video_references(reference_video, reference_audio, control_path) inference_time_s = time.perf_counter() - started_at return Response( diff --git a/vllm_omni/entrypoints/openai/serving_video.py b/vllm_omni/entrypoints/openai/serving_video.py index f3f4b7570..939d55ef8 100644 --- a/vllm_omni/entrypoints/openai/serving_video.py +++ b/vllm_omni/entrypoints/openai/serving_video.py @@ -182,6 +182,35 @@ class OmniOpenAIServingVideo: ) return capability is True or metadata_capability + @property + def supported_control_upload_types(self) -> frozenset[str]: + """Return multipart control types accepted by the active pipeline. + + Unknown pipelines deliberately return an empty set. This keeps the + generic video API isolated from model-specific controls unless a model + explicitly opts into the ``control_path`` contract in metadata. + """ + od_config = self._resolve_diffusion_od_config() + model_archs = [None if od_config is None else getattr(od_config, "model_class_name", None)] + for stage_config in self.stage_configs or (): + stage_get = ( + stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None) + ) + engine_args = stage_get("engine_args") or {} + model_archs.extend( + ( + stage_get("model_arch"), + engine_args.get("model_class_name") + if isinstance(engine_args, Mapping) + else getattr(engine_args, "model_class_name", None), + ) + ) + + supported: set[str] = set() + for model_arch in model_archs: + supported.update(get_diffusion_model_metadata(model_arch).supported_control_upload_types) + return frozenset(supported) + @classmethod def for_diffusion( cls,