[Frontend] Add midway prompt update for streaming video generation (#4652)

Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>

examples/online_serving/streaming_video_generation/README.md

@@ -21,13 +21,16 @@ The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=Tr
 | Direction | Message | Format | Description |
 | --- | --- | --- | --- |
 | Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. |
+| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt midway through generation. `event_id` is optional and `transition_chunks` defaults to the model setting. |
 | Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
+| Server to client | `video.chunk_metadata` | JSON text: `{"type":"video.chunk_metadata","request_id":"...","kind":"media","transport_chunk_index":0,"generation_chunk_index":0,"num_frames":9,"byte_length":1234,"started_event_ids":[],"active_event_ids":[],"completed_event_ids":[]}` | Precedes each binary frame and describes the immediately following payload. |
 | Server to client | Video chunk | Binary WebSocket frame | Fragmented MP4 (`m4s`) video bytes. |
 | Client to server | `session.stop` | JSON text: `{"type":"session.stop"}` | Requests cancellation of the active session. |
 | Client to server | `session.ping` | JSON text: `{"type":"session.ping"}` | Optional keepalive; refreshes the server stall clock. |
+| Server to client | `session.interaction.queued` | JSON text: `{"type":"session.interaction.queued","request_id":"...","event_id":"xxx"}` | Confirms the server queued the interaction for the active request. |
 | Server to client | `session.done` | JSON text: `{"type":"session.done","request_id":"...","chunks":3,"stopped":false}` | Ends a completed or stopped session. |
 | Server to client | `session.pong` | JSON text: `{"type":"session.pong"}` | Reply to `session.ping`. |
-| Server to client | `error` | JSON text: `{"type":"error","message":"..."}` | Reports invalid input, unsupported formats, generation failures, control-message errors, or stall timeout. |
+| Server to client | `error` | JSON text: `{"type":"error","message":"..."}` | Reports invalid input, unsupported formats, inactive requests, unsupported prompt updates, generation failures, control-message errors, or stall timeout. |
 
 During generation the client normally sends only `session.start` and then receives binary chunks; silence on the client socket is expected. The server closes the session with a stall error only when there is no engine progress and no `session.ping` for about 60 seconds.
 
@@ -59,6 +62,17 @@ python streaming_video_client.py \
 The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
 The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.
 
+Schedule midway prompt updates with `--prompt-updates`. Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
+
+```bash
+python streaming_video_client.py \
+  --prompt "A serene lakeside sunrise with mist over the water." \
+  --prompt-updates '[
+    {"at": 2.5, "prompt": "A sea turtle glides past the reeds"},
+    {"at": 5.0, "prompt": "Sunlight breaks through the morning mist", "transition_chunks": 2}
+  ]'
+```
+
 ## Run The Gradio Demo
 
 ```bash

examples/online_serving/streaming_video_generation/gradio_demo.py

@@ -14,8 +14,9 @@ from __future__ import annotations
 
 import argparse
 import json
+import math
 from pathlib import Path
-from typing import Any
+from typing import Any, TypedDict
 
 try:
     import gradio as gr
@@ -23,16 +24,49 @@ except ImportError:
     raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
 
 DEFAULT_PROMPT = "A vibrant tropical fish swimming gracefully among colorful coral reefs in a clear, turquoise ocean. The fish has bright blue and yellow scales with a small, distinctive orange spot on its side, its fins moving fluidly. The coral reefs are alive with a variety of marine life, including small schools of colorful fish and sea turtles gliding by. The water is crystal clear, allowing for a view of the sandy ocean floor below. The reef itself is adorned with a mix of hard and soft corals in shades of red, orange, and green. The photo captures the fish from a slightly elevated angle, emphasizing its lively movements and the vivid colors of its surroundings. A close-up shot with dynamic movement."
+DEFAULT_TRANSITION_CHUNKS = 3
+PROMPT_UPDATE_HEADERS = ["At (s)", "Prompt", "Transition Chunks"]
+PROMPT_UPDATE_VALUES = [
+    [
+        4,
+        (
+            "An underwater tornado appears and affects the ocean floor in a dramatic and chaotic scene. "
+            "The water is murky, swirling violently, carrying debris and marine life into the vortex. "
+            "The tropical fish on the scene all swim in panic, trying to avoid the powerful currents. "
+            "The camera remains stationary, capturing the intensity of the underwater tornado as it disrupts the serene ocean floor. "
+            "Close-up shot emphasizing the turbulent motion and destruction."
+        ),
+        3,
+    ],
+    [
+        11,
+        (
+            "The swirling underwater vortex now seizes a heavy, encrusted treasure chest, its lid flapping open as it is smashed onto the ocean floor. "
+            "Gold coins and silver trinkets spill out, glittering briefly in the murky water before being swept instantly into the violent funnel. "
+            "The heavy wooden box tumbles end over end, colliding with floating rocks and adding to the debris field. "
+            "Swirling sediment and bubbles surround the spilling fortune, highlighting the chaotic power of the storm as it ravages the seabed. "
+            "Close-up shot emphasizing the turbulent motion and destruction."
+        ),
+        3,
+    ],
+]
 VIDEO_STREAM_VIEW_HTML = Path(__file__).with_name("video-stream-view.html")
 VIDEO_STREAM_VIEW_JS = Path(__file__).with_name("video-stream-view.js")
 
 
+class ScheduledPromptUpdate(TypedDict):
+    at: float
+    prompt: str
+    transition_chunks: int
+
+
 def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
     if value is None:
         return
-    if isinstance(value, str) and not value.strip():
-        return
-    payload[key] = value
+    if isinstance(value, str):
+        payload[key] = value.strip()
+    else:
+        payload[key] = value
 
 
 def _optional_int(value: int | float | None) -> int | None:
@@ -43,11 +77,83 @@ def _optional_float(value: int | float | None) -> float | None:
     return None if value is None else float(value)
 
 
+def _normalize_prompt_update_row(row: list[Any] | tuple[Any, ...]) -> tuple[Any, Any, Any]:
+    values = []
+    for v in row[: len(PROMPT_UPDATE_HEADERS)]:
+        if isinstance(v, str):
+            values.append(v.strip() or None)
+        elif isinstance(v, float) and math.isnan(v):
+            values.append(None)
+        else:
+            values.append(v)
+    while len(values) < len(PROMPT_UPDATE_HEADERS):
+        values.append(None)
+    return values[0], values[1], values[2]  # pyright: ignore[reportIndexIssue]
+
+
+def _prompt_update_row_is_empty(row: list[Any] | tuple[Any, ...]) -> bool:
+    at, prompt, transition = _normalize_prompt_update_row(row)
+    return at is None and prompt is None and transition is None
+
+
+def _parse_prompt_update_row(row: list[Any] | tuple[Any, ...], *, index: int) -> ScheduledPromptUpdate:
+    at_val, prompt_val, transition_val = _normalize_prompt_update_row(row)
+    row_label = f"Prompt update row {index + 1}"
+
+    if at_val is None or prompt_val is None:
+        raise gr.Error(f"{row_label} requires both At (s) and Prompt.")
+
+    try:
+        at = float(at_val)
+    except (TypeError, ValueError) as exc:
+        raise gr.Error(f"{row_label}: At (s) must be a number.") from exc
+    if at < 0:
+        raise gr.Error(f"{row_label}: At (s) must be >= 0.")
+
+    prompt = str(prompt_val).strip()
+    if not prompt:
+        raise gr.Error(f"{row_label}: Prompt must be a non-empty string.")
+
+    transition_chunks = DEFAULT_TRANSITION_CHUNKS
+    if transition_val is not None:
+        try:
+            transition_chunks = int(transition_val)
+        except (TypeError, ValueError) as exc:
+            raise gr.Error(f"{row_label}: Transition Chunks must be an integer.") from exc
+        if transition_chunks < 0:
+            raise gr.Error(f"{row_label}: Transition Chunks must be >= 0.")
+
+    return {
+        "at": at,
+        "prompt": prompt,
+        "transition_chunks": transition_chunks,
+    }
+
+
+def _parse_prompt_updates_dataframe(rows: list[list[Any]] | None) -> list[ScheduledPromptUpdate]:
+    if not rows:
+        return []
+
+    # Single row: empty or complete.
+    if len(rows) == 1:
+        row = rows[0]
+        if _prompt_update_row_is_empty(row):
+            return []
+        return [_parse_prompt_update_row(row, index=0)]
+
+    updates: list[ScheduledPromptUpdate] = []
+    for index, row in enumerate(rows):
+        if _prompt_update_row_is_empty(row):
+            raise gr.Error(f"Prompt update row {index + 1} cannot be empty when multiple rows are present.")
+        updates.append(_parse_prompt_update_row(row, index=index))
+    return sorted(updates, key=lambda update: update["at"])
+
+
 def _build_session_start(
     *,
     model: str,
     prompt: str,
-    negative_prompt: str,
+    negative_prompt: str | None,
     width: int | float | None,
     height: int | float | None,
     fps: int | float | None,
@@ -101,6 +207,7 @@ def build_browser_stream_config(
     model: str,
     prompt: str,
     negative_prompt: str,
+    include_negative_prompt: bool,
     width: int | float | None,
     height: int | float | None,
     fps: int | float,
@@ -118,6 +225,7 @@ def build_browser_stream_config(
     flow_shift: int | float | None,
     include_true_cfg_scale: bool,
     true_cfg_scale: int | float | None,
+    prompt_updates_df: list[list[Any]] | None,
 ) -> tuple[str, Any]:
     """Build the browser-side WebSocket request config for the HTML/MSE player."""
     if not model.strip():
@@ -128,7 +236,7 @@ def build_browser_stream_config(
     payload = _build_session_start(
         model=model.strip(),
         prompt=prompt.strip(),
-        negative_prompt=negative_prompt,
+        negative_prompt=negative_prompt if include_negative_prompt else None,
         width=width,
         height=height,
         fps=fps,
@@ -141,15 +249,21 @@ def build_browser_stream_config(
         true_cfg_scale=true_cfg_scale if include_true_cfg_scale else None,
         seed=seed if include_seed else None,
     )
+    prompt_updates = _parse_prompt_updates_dataframe(prompt_updates_df)
     config = {
         "url": f"ws://{host}:{int(port)}/v1/realtime/video",
         "payload": payload,
     }
+    if prompt_updates:
+        config["prompt_updates"] = prompt_updates
     return json.dumps(config, ensure_ascii=False), gr.update(value="Streaming...", interactive=False)
 
 
 def create_demo() -> gr.Blocks:
-    with gr.Blocks(title="Streaming Video Generation") as demo:
+    with gr.Blocks(
+        title="Streaming Video Generation",
+        css="textarea:disabled, input:disabled { color: var(--body-text-color-subdued); -webkit-text-fill-color: var(--body-text-color-subdued); background-color: var(--background-fill-secondary); }",
+    ) as demo:
         gr.Markdown("# Streaming Video Generation")
         gr.Markdown(
             "Connects to `WS /v1/realtime/video` in the browser and appends fMP4 chunks "
@@ -162,7 +276,6 @@ def create_demo() -> gr.Blocks:
                 port = gr.Number(label="Server Port", value=8000, precision=0)
                 model = gr.Textbox(label="Model", value="BestWishYsh/Helios-Distilled")
                 prompt = gr.Textbox(label="Prompt", value=DEFAULT_PROMPT, lines=3)
-                negative_prompt = gr.Textbox(label="Negative Prompt", value="", lines=2)
 
                 with gr.Row():
                     width = gr.Number(label="Width", value=640, precision=0)
@@ -170,49 +283,66 @@ def create_demo() -> gr.Blocks:
 
                 with gr.Row():
                     fps = gr.Number(label="FPS", value=16, precision=0)
-                    num_frames = gr.Number(label="Num Frames", value=132, precision=0)
+                    num_frames = gr.Number(label="Num Frames", value=429, precision=0)
                     num_inference_steps = gr.Number(label="Num Inference Steps", value=50, precision=0)
 
-                with gr.Row():
-                    with gr.Column():
-                        guidance_scale = gr.Number(
-                            label="Guidance Scale",
-                            value=1.0,
-                            interactive=True,
-                            scale=4,
-                        )
-                        include_guidance_scale = gr.Checkbox(label="Send", value=True, scale=1, min_width=70)
-                    with gr.Column():
-                        guidance_scale_2 = gr.Number(
-                            label="Guidance Scale 2",
-                            value=None,
-                            interactive=False,
-                            scale=4,
-                        )
-                        include_guidance_scale_2 = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
-                    with gr.Column():
-                        true_cfg_scale = gr.Number(
-                            label="True CFG Scale",
-                            value=None,
-                            interactive=False,
-                            scale=4,
-                        )
-                        include_true_cfg_scale = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
-                    # I want it to wrap after 3, but it auto wrap after 2. I'll rather fully delegate to auto wrapping.
-                    with gr.Column():
-                        boundary_ratio = gr.Number(
-                            label="Boundary Ratio",
-                            value=None,
-                            interactive=False,
-                            scale=4,
-                        )
-                        include_boundary_ratio = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
-                    with gr.Column():
-                        flow_shift = gr.Number(label="Flow Shift", value=None, interactive=False, scale=4)
-                        include_flow_shift = gr.Checkbox(label="Send", value=False, scale=1, min_width=70)
-                    with gr.Column():
-                        seed = gr.Number(label="Seed", value=42, precision=0, interactive=True, scale=4)
-                        include_seed = gr.Checkbox(label="Send", value=True, scale=1, min_width=70)
+                with gr.Accordion("Advanced Sampling", open=False):
+                    with gr.Row():
+                        with gr.Column():
+                            negative_prompt = gr.Textbox(label="Negative Prompt", value="", lines=1, interactive=False)
+                            include_negative_prompt = gr.Checkbox(label="Send", value=False)
+                    with gr.Row():
+                        with gr.Column(scale=0, min_width=150):
+                            guidance_scale = gr.Number(
+                                label="Guidance Scale",
+                                value=1.0,
+                                interactive=True,
+                            )
+                            include_guidance_scale = gr.Checkbox(label="Send", value=True)
+                        with gr.Column(scale=0, min_width=150):
+                            guidance_scale_2 = gr.Number(
+                                label="Guidance Scale 2",
+                                value=None,
+                                interactive=False,
+                            )
+                            include_guidance_scale_2 = gr.Checkbox(label="Send", value=False)
+                        with gr.Column(scale=0, min_width=150):
+                            true_cfg_scale = gr.Number(
+                                label="True CFG Scale",
+                                value=None,
+                                interactive=False,
+                            )
+                            include_true_cfg_scale = gr.Checkbox(label="Send", value=False)
+                        with gr.Column(scale=0, min_width=150):
+                            boundary_ratio = gr.Number(
+                                label="Boundary Ratio",
+                                value=None,
+                                interactive=False,
+                            )
+                            include_boundary_ratio = gr.Checkbox(label="Send", value=False)
+                        with gr.Column(scale=0, min_width=150):
+                            flow_shift = gr.Number(label="Flow Shift", value=None, interactive=False)
+                            include_flow_shift = gr.Checkbox(label="Send", value=False)
+                        with gr.Column(scale=0, min_width=150):
+                            seed = gr.Number(label="Seed", value=42, precision=0, interactive=True)
+                            include_seed = gr.Checkbox(label="Send", value=True)
+
+                with gr.Accordion("Prompt Updates", open=True):
+                    gr.Markdown(
+                        "Schedule midway text `session.interaction` messages after `video.start`. "
+                        "With one row, leave it fully empty or fill both **At (s)** and **Prompt**. "
+                        "With multiple rows, every row must be complete."
+                    )
+                    prompt_updates_df = gr.Dataframe(
+                        headers=PROMPT_UPDATE_HEADERS,
+                        datatype=["number", "str", "number"],
+                        type="array",
+                        row_count=1,
+                        row_limits=(1, None),
+                        column_count=3,
+                        value=PROMPT_UPDATE_VALUES,
+                        interactive=True,
+                    )
 
                 with gr.Row():
                     start_button = gr.Button("Start", variant="primary", elem_id="streaming-video-start")
@@ -231,6 +361,7 @@ def create_demo() -> gr.Blocks:
             model,
             prompt,
             negative_prompt,
+            include_negative_prompt,
             width,
             height,
             fps,
@@ -248,8 +379,10 @@ def create_demo() -> gr.Blocks:
             flow_shift,
             include_true_cfg_scale,
             true_cfg_scale,
+            prompt_updates_df,
         ]
         for toggle, field in [
+            (include_negative_prompt, negative_prompt),
             (include_guidance_scale, guidance_scale),
             (include_guidance_scale_2, guidance_scale_2),
             (include_seed, seed),

examples/online_serving/streaming_video_generation/streaming_video_client.py

@@ -13,10 +13,12 @@ Requirements:
 
 import argparse
 import asyncio
+import contextlib
 import io
 import json
 import os
 import time
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
@@ -39,17 +41,69 @@ HELIOS_DISTILLED_EXTRA_PARAMS = {
     "is_amplify_first_chunk": True,
 }
 
+DEFAULT_TRANSITION_CHUNKS = 3
 
-def _json_object(value: str, *, argument_name: str) -> dict[str, Any]:
+
+@dataclass(frozen=True)
+class ScheduledPromptUpdate:
+    """Client-side schedule entry for a midway text ``session.interaction``."""
+
+    at: float
+    prompt: str
+    transition_chunks: int
+
+
+def _parse_extra_params(value: str) -> dict[str, Any]:
     try:
         parsed = json.loads(value)
     except json.JSONDecodeError as exc:
-        raise argparse.ArgumentTypeError(f"{argument_name} must be valid JSON: {exc}") from exc
+        raise argparse.ArgumentTypeError(f"--extra-params must be valid JSON: {exc}") from exc
     if not isinstance(parsed, dict):
-        raise argparse.ArgumentTypeError(f"{argument_name} must be a JSON object")
+        raise argparse.ArgumentTypeError("--extra-params must be a JSON object")
     return parsed
 
 
+def _parse_prompt_updates(value: str) -> list[ScheduledPromptUpdate]:
+    try:
+        parsed = json.loads(value)
+    except json.JSONDecodeError as exc:
+        raise argparse.ArgumentTypeError(f"--prompt-updates must be valid JSON: {exc}") from exc
+    if not isinstance(parsed, list):
+        raise argparse.ArgumentTypeError("--prompt-updates must be a JSON array")
+
+    updates: list[ScheduledPromptUpdate] = []
+    for index, item in enumerate(parsed):
+        try:
+            at = float(item["at"])
+            prompt = item["prompt"]
+        except (KeyError, TypeError, ValueError) as exc:
+            raise argparse.ArgumentTypeError(
+                f"--prompt-updates[{index}] must have valid 'at' and 'prompt' fields"
+            ) from exc
+        if at < 0:
+            raise argparse.ArgumentTypeError(f"--prompt-updates[{index}].at must be >= 0")
+        if not isinstance(prompt, str) or not prompt.strip():
+            raise argparse.ArgumentTypeError(f"--prompt-updates[{index}].prompt must be a non-empty string")
+
+        try:
+            transition_chunks = item.get("transition_chunks", DEFAULT_TRANSITION_CHUNKS)
+            transition_chunks = int(transition_chunks)
+        except (TypeError, ValueError) as exc:
+            raise argparse.ArgumentTypeError(f"--prompt-updates[{index}].transition_chunks must be an integer") from exc
+        if transition_chunks < 0:
+            raise argparse.ArgumentTypeError(f"--prompt-updates[{index}].transition_chunks must be >= 0")
+
+        updates.append(
+            ScheduledPromptUpdate(
+                at=at,
+                prompt=prompt,
+                transition_chunks=transition_chunks,
+            )
+        )
+
+    return sorted(updates, key=lambda update: update.at)
+
+
 def _maybe_set(payload: dict[str, Any], key: str, value: Any) -> None:
     if value is not None:
         payload[key] = value
@@ -88,6 +142,31 @@ def build_session_start(args: argparse.Namespace) -> dict[str, Any]:
     return payload
 
 
+async def _run_prompt_update_scheduler(
+    websocket: Any,
+    updates: list[ScheduledPromptUpdate],
+    *,
+    video_started_at: float,
+    send_lock: asyncio.Lock,
+) -> None:
+    for update in updates:
+        delay = (video_started_at + update.at) - time.perf_counter()
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+        payload = {
+            "type": "session.interaction",
+            "interaction": {
+                "event_id": f"scheduled-{update.at:.3f}",
+                "event": {"prompt": update.prompt},
+                "transition_chunks": update.transition_chunks,
+            },
+        }
+        async with send_lock:
+            await websocket.send(json.dumps(payload, ensure_ascii=False))
+        print(f"Sent session.interaction at t={update.at:.2f}s: {json.dumps(payload, ensure_ascii=False)}")
+
+
 def _count_decoded_frames(video_bytes: bytes, *, stream_format: str) -> int | None:
     """Return the number of decodable video frames in a streamed byte sequence, if known."""
     if not video_bytes:
@@ -144,11 +223,15 @@ async def stream_video(args: argparse.Namespace) -> None:
     stream_format = "m4s"
     done = False
     ws = None
+    send_lock = asyncio.Lock()
+    prompt_update_task: asyncio.Task[None] | None = None
+    pending_chunk_metadata: dict[str, Any] | None = None
 
     try:
         async with connect(url, max_size=None) as websocket:
             ws = websocket
-            await websocket.send(json.dumps(payload))
+            async with send_lock:
+                await websocket.send(json.dumps(payload))
             print(f"Connected: {url}")
             print(f"Sent session.start: {json.dumps(payload, ensure_ascii=False)}")
             started_at = time.perf_counter()
@@ -159,6 +242,9 @@ async def stream_video(args: argparse.Namespace) -> None:
                 now = time.perf_counter()
 
                 if isinstance(message, bytes):
+                    # Use chunk metadata just received before video chunks
+                    metadata = pending_chunk_metadata or {}
+                    pending_chunk_metadata = None
                     chunks.append(message)
                     chunk_bytes = len(message)
                     chunk_elapsed = now - last_chunk_at
@@ -176,8 +262,11 @@ async def stream_video(args: argparse.Namespace) -> None:
                     total_elapsed = now - started_at
                     print(
                         f"[chunk {len(chunks):03d}] "
+                        f"kind={metadata.get('kind', 'media')} "
                         f"bytes={chunk_bytes} "
-                        f"frames={chunk_frames} "
+                        f"frames={metadata.get('num_frames', chunk_frames)} "
+                        f"generation_chunk={metadata.get('generation_chunk_index')} "
+                        f"active_interactions={metadata.get('active_event_ids', [])} "
                         f"elapsed={chunk_elapsed:.2f}s "
                         f"total_bytes={total_bytes} "
                         f"total_frames={total_frames} "
@@ -190,9 +279,26 @@ async def stream_video(args: argparse.Namespace) -> None:
                 if msg_type == "video.start":
                     stream_format = msg.get("format") or stream_format
                     print(f"Video session started: request_id={msg.get('request_id')} format={msg.get('format')}")
+                    if args.prompt_updates and prompt_update_task is None:
+                        video_started_at = time.perf_counter()
+                        prompt_update_task = asyncio.create_task(
+                            _run_prompt_update_scheduler(
+                                websocket,
+                                args.prompt_updates,
+                                video_started_at=video_started_at,
+                                send_lock=send_lock,
+                            )
+                        )
+                elif msg_type == "video.chunk_metadata":
+                    # Save the metadata for the incoming video chunk
+                    pending_chunk_metadata = msg
+                elif msg_type == "session.interaction.queued":
+                    print(f"Interaction queued by server: event_id={msg.get('event_id')}")
                 elif msg_type == "session.done":
                     print(f"Session complete: {json.dumps(msg, ensure_ascii=False)}")
                     done = True
+                    if prompt_update_task is not None:
+                        prompt_update_task.cancel()
                     break
                 elif msg_type == "error":
                     print(f"ERROR: {msg.get('message', msg)}")
@@ -202,12 +308,17 @@ async def stream_video(args: argparse.Namespace) -> None:
         print("\nInterrupted. Asking server to stop the active session...")
         if ws is not None:
             try:
-                await ws.send(json.dumps({"type": "session.stop"}))
+                async with send_lock:
+                    await ws.send(json.dumps({"type": "session.stop"}))
                 await ws.close(code=1000, reason="client interrupted")
             except Exception:
                 pass
         raise
     finally:
+        if prompt_update_task is not None:
+            prompt_update_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await prompt_update_task
         print(
             "Saving video... (May take a while. Remuxing concatenated chunks into one progressive MP4 file with total duration metadata in file header)"
         )
@@ -232,7 +343,15 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--prompt",
-        default="A vibrant tropical fish swimming gracefully among colorful coral reefs in a clear, turquoise ocean. The fish has bright blue and yellow scales with a small, distinctive orange spot on its side, its fins moving fluidly. The coral reefs are alive with a variety of marine life, including small schools of colorful fish and sea turtles gliding by. The water is crystal clear, allowing for a view of the sandy ocean floor below. The reef itself is adorned with a mix of hard and soft corals in shades of red, orange, and green. The photo captures the fish from a slightly elevated angle, emphasizing its lively movements and the vivid colors of its surroundings. A close-up shot with dynamic movement.",
+        default=(
+            "A vibrant tropical fish swimming gracefully among colorful coral reefs in a clear, turquoise ocean. "
+            "The fish has bright blue and yellow scales with a small, distinctive orange spot on its side, its fins moving fluidly. "
+            "The coral reefs are alive with a variety of marine life, including small schools of colorful fish and sea turtles gliding by. "
+            "The water is crystal clear, allowing for a view of the sandy ocean floor below. "
+            "The reef itself is adorned with a mix of hard and soft corals in shades of red, orange, and green. "
+            "The photo captures the fish from a slightly elevated angle, emphasizing its lively movements and the vivid colors of its surroundings. "
+            "A close-up shot with dynamic movement."
+        ),
         help="Text prompt for video generation",
     )
     parser.add_argument("--output", default="streaming_video_output.mp4", help="Path for the received video bytes")
@@ -241,7 +360,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--size", default=None, help="Video size as WIDTHxHEIGHT; alternative to width/height")
     parser.add_argument("--seconds", default=None, help="Clip duration in seconds")
     parser.add_argument("--fps", type=int, default=16, help="Frames per second")
-    parser.add_argument("--num-frames", type=int, default=99, help="Number of generated frames")
+    parser.add_argument("--num-frames", type=int, default=429, help="Number of generated frames")
 
     parser.add_argument("--negative-prompt", default=None, help="Negative prompt")
     parser.add_argument("--num-inference-steps", type=int, default=None, help="Number of diffusion steps")
@@ -260,10 +379,44 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--extra-params",
-        type=lambda value: _json_object(value, argument_name="--extra-params"),
+        type=_parse_extra_params,
         default=None,
         help="JSON object merged into extra_params; overrides preset keys on conflict.",
     )
+    parser.add_argument(
+        "--prompt-updates",
+        type=_parse_prompt_updates,
+        default=json.dumps(
+            [
+                {
+                    "at": 4,
+                    "prompt": (
+                        "An underwater tornado appears and affects the ocean floor in a dramatic and chaotic scene. "
+                        "The water is murky, swirling violently, carrying debris and marine life into the vortex. "
+                        "The tropical fish on the scene all swim in panic, trying to avoid the powerful currents. "
+                        "The camera remains stationary, capturing the intensity of the underwater tornado as it disrupts the serene ocean floor. "
+                        "Close-up shot emphasizing the turbulent motion and destruction."
+                    ),
+                },
+                {
+                    "at": 11,
+                    "prompt": (
+                        "The swirling underwater vortex now seizes a heavy, encrusted treasure chest, its lid flapping open as it is smashed onto the ocean floor. "
+                        "Gold coins and silver trinkets spill out, glittering briefly in the murky water before being swept instantly into the violent funnel. "
+                        "The heavy wooden box tumbles end over end, colliding with floating rocks and adding to the debris field. "
+                        "Swirling sediment and bubbles surround the spilling fortune, highlighting the chaotic power of the storm as it ravages the seabed. "
+                        "Close-up shot emphasizing the turbulent motion and destruction."
+                    ),
+                    # "transition_chunks": 3,
+                },
+            ]
+        ),
+        help=(
+            "JSON array of scheduled prompt updates. Each object requires "
+            "'at' (seconds after video.start) and 'prompt'. Optional "
+            f"'transition_chunks' defaults to {DEFAULT_TRANSITION_CHUNKS}."
+        ),
+    )
     return parser.parse_args()
 
 

examples/online_serving/streaming_video_generation/video-stream-view.js

@@ -14,6 +14,7 @@
   let totalBytes = 0;
   let done = false;
   let objectUrl = null;
+  let promptUpdateTimers = [];
 
   function el(id) {
     return document.getElementById(id);
@@ -73,7 +74,50 @@
     } catch (_) { }
   }
 
+  function clearPromptUpdateTimers() {
+    for (const timer of promptUpdateTimers) {
+      clearTimeout(timer);
+    }
+    promptUpdateTimers = [];
+  }
+
+  function schedulePromptUpdates(updates) {
+    clearPromptUpdateTimers();
+    if (!Array.isArray(updates) || updates.length === 0) return;
+
+    const videoStartedAt = performance.now();
+    for (const update of updates) {
+      const atSeconds = Number(update.at);
+      if (!Number.isFinite(atSeconds) || atSeconds < 0) continue;
+
+      const delayMs = Math.max(0, atSeconds * 1000 - (performance.now() - videoStartedAt));
+      const transitionChunks = Number.isFinite(Number(update.transition_chunks))
+        ? Number(update.transition_chunks)
+        : 3;
+
+      const timer = setTimeout(() => {
+        if (!ws || ws.readyState !== WebSocket.OPEN) return;
+        const payload = {
+          type: "session.interaction",
+          interaction: {
+            event_id: "scheduled-" + atSeconds.toFixed(3),
+            event: { prompt: update.prompt },
+            transition_chunks: transitionChunks,
+          },
+        };
+        try {
+          ws.send(JSON.stringify(payload));
+          log("Sent session.interaction at t=" + atSeconds.toFixed(2) + "s: " + JSON.stringify(payload));
+        } catch (err) {
+          log("Failed to send session.interaction: " + err.message);
+        }
+      }, delayMs);
+      promptUpdateTimers.push(timer);
+    }
+  }
+
   function stopCurrent() {
+    clearPromptUpdateTimers();
     if (ws && ws.readyState === WebSocket.OPEN) {
       try { ws.send(JSON.stringify({ type: "session.stop" })); } catch (_) { }
       try { ws.close(); } catch (_) { }
@@ -148,6 +192,11 @@
           const msg = JSON.parse(event.data);
           if (msg.type === "video.start") {
             log("Video session started: request_id=" + (msg.request_id || "") + " format=" + (msg.format || ""));
+            schedulePromptUpdates(config.prompt_updates);
+          } else if (msg.type === "video.chunk_metadata") {
+            log("Chunk metadata: " + JSON.stringify(msg));
+          } else if (msg.type === "session.interaction.queued") {
+            log("Interaction queued by server: event_id=" + (msg.event_id || ""));
           } else if (msg.type === "session.done") {
             done = true;
             setStatus("Done");