[feat] Streaming diffusion video generation output (#3737)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
authored and GitHub committed 2026-06-23 09:57:53 +00:00
1 parent b84c99c122
commit 3901ad8165
42 files changed
+4322 -65

No files matched your search

+97 -1
View File
@@ -19,7 +19,7 @@ from collections.abc import Generator
from dataclasses import asdict, dataclass
from io import BytesIO
from pathlib import Path
from typing import Any, NamedTuple
from typing import Any, NamedTuple, cast
from urllib.parse import quote
import psutil
@@ -2168,6 +2168,102 @@ class OpenAIClientHandler:
timeout=timeout,
)
def send_streaming_video_diffusion_request(
self,
request_config: dict[str, Any],
request_num: int = 1,
*,
timeout_seconds: float = 600.0,
) -> list[DiffusionResponse]:
"""
Send a native ``/v1/realtime/video`` WebSocket request and return one
finalized MP4 artifact assembled from the streamed binary fragments.
"""
if request_num != 1:
raise NotImplementedError("Concurrent streaming video diffusion requests are not currently implemented")
response = asyncio.run(
self._send_streaming_video_diffusion_request_once(
request_config,
timeout_seconds=timeout_seconds,
)
)
assert_diffusion_response(response, request_config, run_level=self.run_level)
if response.e2e_latency is not None:
self._print_client_stat(f"[diffusion.stream] request#1 success in {response.e2e_latency:.3f}s")
else:
self._print_client_stat("[diffusion.stream] request#1 completed")
return [response]
async def _send_streaming_video_diffusion_request_once(
self,
request_config: dict[str, Any],
*,
timeout_seconds: float,
) -> DiffusionResponse:
form_data = request_config.get("form_data")
if not isinstance(form_data, dict):
raise ValueError("Video request_config must contain 'form_data'")
payload: dict[str, Any] = {
"type": "session.start",
**{key: value for key, value in form_data.items() if value is not None},
}
model = request_config.get("model")
if model is not None:
payload["model"] = model
payload.setdefault("format", "m4s")
fps = float(payload.get("fps") or 16)
stream_format = payload["format"]
url = self._build_ws_url("/v1/realtime/video")
result = DiffusionResponse()
chunks: list[bytes] = []
start_time = time.perf_counter()
deadline = start_time + timeout_seconds
import websockets
async with websockets.connect(url, max_size=None) as websocket:
await websocket.send(json.dumps(payload))
while True:
remaining = deadline - time.perf_counter()
if remaining <= 0:
raise TimeoutError(f"Streaming video request did not complete within {timeout_seconds}s")
message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
if isinstance(message, bytes):
chunks.append(message)
continue
msg = json.loads(message)
msg_type = msg.get("type")
if msg_type == "video.start":
stream_format = msg.get("format") or stream_format
continue
if msg_type == "session.done":
break
if msg_type == "error":
raise RuntimeError(str(msg.get("message", msg)))
from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
from vllm_omni.entrypoints.openai.video_api_utils import StreamingVideoFormat
streamed_bytes = b"".join(chunks)
if not streamed_bytes:
raise RuntimeError("Streaming video request completed without binary video chunks")
result.videos = [
finalize_streaming_video_bytes(
streamed_bytes,
input_format=cast(StreamingVideoFormat, stream_format),
fps=fps,
)
]
result.e2e_latency = time.perf_counter() - start_time
result.success = True
return result
def _wait_until_video_completed(
self, video_id: str, poll_interval_seconds: int = 2, timeout_seconds: int = 300
) -> None: