vllm-omni/examples/online_serving/streaming_video_generation/video-stream-view.js

/**
 * This file and the corresponding HTML file are needed to display a streamable video view in the Gradio demo.
 * Gradio's built-in video player does not support modern M4S streaming format.
 */
(function () {
  const playerId = "vllm-streaming-video-player";
  const statusId = "vllm-streaming-video-status";
  const logId = "vllm-streaming-video-log";
  let ws = null;
  let mediaSource = null;
  let sourceBuffer = null;
  let queue = [];
  let chunkCount = 0;
  let totalBytes = 0;
  let done = false;
  let objectUrl = null;
  let promptUpdateTimers = [];

  function el(id) {
    return document.getElementById(id);
  }

  function setStatus(text) {
    const node = el(statusId);
    if (node) node.textContent = text;
  }

  function log(line) {
    const node = el(logId);
    if (!node) return;
    node.textContent += "\n" + line;
    node.scrollTop = node.scrollHeight;
  }

  function setStartButton(disabled, label) {
    const btn = document.querySelector("#streaming-video-start button") || document.getElementById("streaming-video-start");
    if (!btn) return;
    btn.disabled = disabled;
    btn.textContent = label;
  }

  function chooseMime() {
    const candidates = [
      'video/mp4; codecs="avc1.42E01E"',
      'video/mp4; codecs="avc1.4D401F"',
      'video/mp4; codecs="avc1.64001F"',
      "video/mp4"
    ];
    for (const mime of candidates) {
      if (window.MediaSource && MediaSource.isTypeSupported(mime)) return mime;
    }
    return null;
  }

  function pump() {
    if (!sourceBuffer || sourceBuffer.updating || queue.length === 0) return;
    try {
      sourceBuffer.appendBuffer(queue.shift());
    } catch (err) {
      done = true;
      setStatus("SourceBuffer error");
      setStartButton(false, "Restart");
      log("SourceBuffer append error: " + err.message);
      finishStream();
    }
  }

  function finishStream() {
    if (!mediaSource || mediaSource.readyState !== "open") return;
    if (sourceBuffer && sourceBuffer.updating) return;
    if (queue.length > 0) return;
    try {
      mediaSource.endOfStream();
    } catch (_) { }
  }

  function clearPromptUpdateTimers() {
    for (const timer of promptUpdateTimers) {
      clearTimeout(timer);
    }
    promptUpdateTimers = [];
  }

  function schedulePromptUpdates(updates) {
    clearPromptUpdateTimers();
    if (!Array.isArray(updates) || updates.length === 0) return;

    const videoStartedAt = performance.now();
    for (const update of updates) {
      const atSeconds = Number(update.at);
      if (!Number.isFinite(atSeconds) || atSeconds < 0) continue;

      const delayMs = Math.max(0, atSeconds * 1000 - (performance.now() - videoStartedAt));
      const transitionChunks = Number.isFinite(Number(update.transition_chunks))
        ? Number(update.transition_chunks)
        : 3;

      const timer = setTimeout(() => {
        if (!ws || ws.readyState !== WebSocket.OPEN) return;
        const payload = {
          type: "session.interaction",
          interaction: {
            event_id: "scheduled-" + atSeconds.toFixed(3),
            event: { prompt: update.prompt },
            transition_chunks: transitionChunks,
          },
        };
        try {
          ws.send(JSON.stringify(payload));
          log("Sent session.interaction at t=" + atSeconds.toFixed(2) + "s: " + JSON.stringify(payload));
        } catch (err) {
          log("Failed to send session.interaction: " + err.message);
        }
      }, delayMs);
      promptUpdateTimers.push(timer);
    }
  }

  function stopCurrent() {
    clearPromptUpdateTimers();
    if (ws && ws.readyState === WebSocket.OPEN) {
      try { ws.send(JSON.stringify({ type: "session.stop" })); } catch (_) { }
      try { ws.close(); } catch (_) { }
    }
    ws = null;
    sourceBuffer = null;
    mediaSource = null;
    queue = [];
    if (objectUrl) URL.revokeObjectURL(objectUrl);
    objectUrl = null;
  }

  window.vllmStreamingVideoStart = function (configJson) {
    stopCurrent();
    chunkCount = 0;
    totalBytes = 0;
    done = false;
    const logNode = el(logId);
    if (logNode) logNode.textContent = "Starting...";
    setStatus("Starting...");

    let config;
    try {
      config = JSON.parse(configJson);
    } catch (err) {
      setStatus("Invalid request");
      setStartButton(false, "Restart");
      log("Invalid request config: " + err.message);
      return configJson;
    }

    const mime = chooseMime();
    if (!mime) {
      setStatus("MSE unsupported");
      setStartButton(false, "Restart");
      log("This browser does not support MP4 Media Source Extensions playback.");
      return configJson;
    }

    const video = el(playerId);
    if (!video) return configJson;
    mediaSource = new MediaSource();
    objectUrl = URL.createObjectURL(mediaSource);
    video.src = objectUrl;

    mediaSource.addEventListener("sourceopen", () => {
      try {
        sourceBuffer = mediaSource.addSourceBuffer(mime);
        sourceBuffer.mode = "segments";
        sourceBuffer.addEventListener("updateend", () => {
          pump();
          if (done) finishStream();
        });
      } catch (err) {
        setStatus("SourceBuffer error");
        setStartButton(false, "Restart");
        log("SourceBuffer error: " + err.message);
        return;
      }

      ws = new WebSocket(config.url);
      ws.binaryType = "arraybuffer";
      ws.onopen = () => {
        setStatus("Streaming...");
        setStartButton(true, "Streaming...");
        log("Connected: " + config.url);
        ws.send(JSON.stringify(config.payload));
        log("Sent session.start: " + JSON.stringify(config.payload));
      };
      ws.onmessage = (event) => {
        if (typeof event.data === "string") {
          const msg = JSON.parse(event.data);
          if (msg.type === "video.start") {
            log("Video session started: request_id=" + (msg.request_id || "") + " format=" + (msg.format || ""));
            schedulePromptUpdates(config.prompt_updates);
          } else if (msg.type === "video.chunk_metadata") {
            log("Chunk metadata: " + JSON.stringify(msg));
          } else if (msg.type === "session.interaction.queued") {
            log("Interaction queued by server: event_id=" + (msg.event_id || ""));
          } else if (msg.type === "session.done") {
            done = true;
            setStatus("Done");
            setStartButton(false, "Restart");
            log("Session complete: " + JSON.stringify(msg));
            finishStream();
          } else if (msg.type === "error") {
            done = true;
            setStatus("Error");
            setStartButton(false, "Restart");
            log("ERROR: " + (msg.message || JSON.stringify(msg)));
            finishStream();
          } else {
            log("Control message: " + JSON.stringify(msg));
          }
          return;
        }

        const data = event.data;
        queue.push(data);
        chunkCount += 1;
        totalBytes += data.byteLength;
        log(`[chunk ${String(chunkCount).padStart(3, "0")}] bytes=${data.byteLength} total_bytes=${totalBytes}`);
        pump();
      };
      ws.onerror = () => {
        done = true;
        setStatus("Error");
        setStartButton(false, "Restart");
        log("ERROR: WebSocket error");
        finishStream();
      };
      ws.onclose = () => {
        if (!done) {
          done = true;
          setStatus("Closed");
          setStartButton(false, "Restart");
          log("WebSocket closed before session.done");
          finishStream();
        }
      };
    }, { once: true });

    return configJson;
  };

  console.log("video-stream-view.html loaded");
})();