[feat] Streaming diffusion video generation output (#3737)
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> Signed-off-by: Zeyu Huang | 黃澤宇 <11222265+fhfuih@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
42 files changed
+4322
-65
No files matched your search
@@ -19,7 +19,7 @@ from collections.abc import Generator
|
||||
from dataclasses import asdict, dataclass
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
from typing import Any, NamedTuple, cast
|
||||
from urllib.parse import quote
|
||||
|
||||
import psutil
|
||||
@@ -2168,6 +2168,102 @@ class OpenAIClientHandler:
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
def send_streaming_video_diffusion_request(
|
||||
self,
|
||||
request_config: dict[str, Any],
|
||||
request_num: int = 1,
|
||||
*,
|
||||
timeout_seconds: float = 600.0,
|
||||
) -> list[DiffusionResponse]:
|
||||
"""
|
||||
Send a native ``/v1/realtime/video`` WebSocket request and return one
|
||||
finalized MP4 artifact assembled from the streamed binary fragments.
|
||||
"""
|
||||
if request_num != 1:
|
||||
raise NotImplementedError("Concurrent streaming video diffusion requests are not currently implemented")
|
||||
|
||||
response = asyncio.run(
|
||||
self._send_streaming_video_diffusion_request_once(
|
||||
request_config,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
)
|
||||
assert_diffusion_response(response, request_config, run_level=self.run_level)
|
||||
if response.e2e_latency is not None:
|
||||
self._print_client_stat(f"[diffusion.stream] request#1 success in {response.e2e_latency:.3f}s")
|
||||
else:
|
||||
self._print_client_stat("[diffusion.stream] request#1 completed")
|
||||
return [response]
|
||||
|
||||
async def _send_streaming_video_diffusion_request_once(
|
||||
self,
|
||||
request_config: dict[str, Any],
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
) -> DiffusionResponse:
|
||||
form_data = request_config.get("form_data")
|
||||
if not isinstance(form_data, dict):
|
||||
raise ValueError("Video request_config must contain 'form_data'")
|
||||
payload: dict[str, Any] = {
|
||||
"type": "session.start",
|
||||
**{key: value for key, value in form_data.items() if value is not None},
|
||||
}
|
||||
model = request_config.get("model")
|
||||
if model is not None:
|
||||
payload["model"] = model
|
||||
payload.setdefault("format", "m4s")
|
||||
|
||||
fps = float(payload.get("fps") or 16)
|
||||
stream_format = payload["format"]
|
||||
url = self._build_ws_url("/v1/realtime/video")
|
||||
|
||||
result = DiffusionResponse()
|
||||
chunks: list[bytes] = []
|
||||
start_time = time.perf_counter()
|
||||
deadline = start_time + timeout_seconds
|
||||
|
||||
import websockets
|
||||
|
||||
async with websockets.connect(url, max_size=None) as websocket:
|
||||
await websocket.send(json.dumps(payload))
|
||||
|
||||
while True:
|
||||
remaining = deadline - time.perf_counter()
|
||||
if remaining <= 0:
|
||||
raise TimeoutError(f"Streaming video request did not complete within {timeout_seconds}s")
|
||||
|
||||
message = await asyncio.wait_for(websocket.recv(), timeout=remaining)
|
||||
if isinstance(message, bytes):
|
||||
chunks.append(message)
|
||||
continue
|
||||
|
||||
msg = json.loads(message)
|
||||
msg_type = msg.get("type")
|
||||
if msg_type == "video.start":
|
||||
stream_format = msg.get("format") or stream_format
|
||||
continue
|
||||
if msg_type == "session.done":
|
||||
break
|
||||
if msg_type == "error":
|
||||
raise RuntimeError(str(msg.get("message", msg)))
|
||||
|
||||
from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes
|
||||
from vllm_omni.entrypoints.openai.video_api_utils import StreamingVideoFormat
|
||||
|
||||
streamed_bytes = b"".join(chunks)
|
||||
if not streamed_bytes:
|
||||
raise RuntimeError("Streaming video request completed without binary video chunks")
|
||||
result.videos = [
|
||||
finalize_streaming_video_bytes(
|
||||
streamed_bytes,
|
||||
input_format=cast(StreamingVideoFormat, stream_format),
|
||||
fps=fps,
|
||||
)
|
||||
]
|
||||
result.e2e_latency = time.perf_counter() - start_time
|
||||
result.success = True
|
||||
return result
|
||||
|
||||
def _wait_until_video_completed(
|
||||
self, video_id: str, poll_interval_seconds: int = 2, timeout_seconds: int = 300
|
||||
) -> None:
|
||||
|
||||
Reference in new issue
Block a user