vllm-omni/examples/online_serving/minicpmo/realtime_web/app/static/app.js

(() => {
  'use strict';

  const config = window.FULL_DUPLEX_CONFIG || {};
  const callButton = document.getElementById('callButton');
  const muteButton = document.getElementById('muteButton');
  const cameraButton = document.getElementById('cameraButton');
  const cameraPreview = document.getElementById('cameraPreview');
  const promptPreset = document.getElementById('promptPreset');
  const systemPromptInput = document.getElementById('systemPrompt');
  const connectionState = document.getElementById('connectionState');
  const modelState = document.getElementById('modelState');
  const playbackState = document.getElementById('playbackState');
  const sessionTimer = document.getElementById('sessionTimer');
  const meterFill = document.getElementById('meterFill');
  const conversation = document.getElementById('conversation');
  const emptyConversation = document.getElementById('emptyConversation');
  const eventLog = document.getElementById('eventLog');
  const eventCount = document.getElementById('eventCount');
  const runtimeDetail = document.getElementById('runtimeDetail');
  const clearLogButton = document.getElementById('clearLogButton');

  const INPUT_RATE = 16000;
  const OUTPUT_RATE = 24000;
  const SEND_INTERVAL_MS = 200;
  const ECHO_GUARD_MS = 300;
  const INITIAL_PLAYBACK_BUFFER_MS = 400;
  const SESSION_CLOSE_TIMEOUT_MS = 1000;

  // Default prompts mirroring the official MiniCPM-o-Demo presets
  // (assets/presets/{omni,audio_duplex}/*.yaml).
  const PROMPT_PRESETS = {
    omni: 'Streaming Omni Conversation.',
    chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
      + '请用高自然度的方式和用户聊天。你处于双工模式，可以一边听、一边说。'
      + '你是由面壁智能开发的人工智能助手：面壁小钢炮。',
    english_call: 'Replicate the tone and style from the input audio. Your task is to be '
      + 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
      + 'seriously and in a high quality. Please chat with the user in a high naturalness '
      + 'style. You are in duplex mode, where you can listen and speak at the same time.',
  };

  let socket = null;
  let mediaStream = null;
  let captureContext = null;
  let captureNode = null;
  let playbackContext = null;
  let playbackNode = null;
  let sendTimer = null;
  let clockTimer = null;
  let startedAt = 0;
  let running = false;
  let muted = false;
  let assistantActive = false;
  let captureRate = INPUT_RATE;
  let cameraStream = null;
  let cameraTimer = null;
  let cameraPendingFrame = null;
  const cameraCanvas = document.createElement('canvas');
  let playbackRate = OUTPUT_RATE;
  let pendingCapture = [];
  let currentResponseId = null;
  let responseHasAudio = false;
  let logCount = 0;
  let liveUserTurn = null;
  let liveAssistantTurn = null;
  let sessionCloseResolver = null;

  function staticAssetUrl(path) {
    const version = String(config.appVersion || '').trim();
    return version ? `${path}?v=${encodeURIComponent(version)}` : path;
  }

  if (promptPreset && systemPromptInput) {
    promptPreset.addEventListener('change', () => {
      const preset = PROMPT_PRESETS[promptPreset.value];
      if (preset !== undefined) systemPromptInput.value = preset;
    });
    systemPromptInput.addEventListener('input', () => {
      promptPreset.value = 'custom';
    });
  }

  function realtimeUrl() {
    const url = new URL(config.realtimePath, window.location.href);
    url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
    url.searchParams.set('duplex', '1');
    url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
    url.searchParams.set('minicpmo45_native_duplex', '1');
    url.searchParams.set('autostart', '0');
    return url.toString();
  }

  function setConnection(label, kind) {
    connectionState.textContent = label;
    connectionState.className = `status status-${kind}`;
  }

  function setModel(label) {
    modelState.textContent = label;
  }

  function setPlayback(label) {
    playbackState.textContent = label;
  }

  function compactEvent(event) {
    const fields = [];
    const payload = event.event || event;
    const responseId = responseIdOf(event);
    if (responseId) fields.push(`response=${responseId}`);
    if (event.item_id) fields.push(`item=${event.item_id}`);
    if (event.code) fields.push(`code=${event.code}`);
    if (event.response && event.response.status) fields.push(`status=${event.response.status}`);
    if (payload.played_ms !== undefined) fields.push(`played=${payload.played_ms}ms`);
    if (payload.committed_ms !== undefined) fields.push(`committed=${payload.committed_ms}ms`);
    if (payload.history_committed !== undefined) fields.push(`history=${payload.history_committed}`);
    return fields.join(' ');
  }

  function appendLog(message, error = false) {
    const time = new Date().toLocaleTimeString([], { hour12: false });
    const line = document.createElement('span');
    if (error) line.className = 'log-error';
    line.textContent = `${time}  ${message}\n`;
    eventLog.appendChild(line);
    eventLog.scrollTop = eventLog.scrollHeight;
    logCount += 1;
    eventCount.textContent = `${logCount} ${logCount === 1 ? 'event' : 'events'}`;
  }

  function appendEventLog(event) {
    const detail = compactEvent(event);
    appendLog(`${event.type || 'unknown'}${detail ? `  ${detail}` : ''}`, event.type === 'error');
  }

  function responseIdOf(event) {
    return event.response_id || (event.response && event.response.id) || null;
  }

  function ensureTurn(role) {
    const existing = role === 'user' ? liveUserTurn : liveAssistantTurn;
    if (existing) return existing;
    if (emptyConversation) emptyConversation.remove();
    const row = document.createElement('div');
    row.className = `turn turn-${role} turn-live`;
    const label = document.createElement('div');
    label.className = 'turn-role';
    label.textContent = role === 'user' ? 'You' : 'Assistant';
    const text = document.createElement('div');
    text.className = 'turn-text';
    row.append(label, text);
    conversation.appendChild(row);
    conversation.scrollTop = conversation.scrollHeight;
    const turn = { row, text, value: '' };
    if (role === 'user') liveUserTurn = turn;
    else liveAssistantTurn = turn;
    return turn;
  }

  function addTranscript(role, delta) {
    if (!delta) return;
    const turn = ensureTurn(role);
    turn.value += delta;
    turn.text.textContent = turn.value;
    conversation.scrollTop = conversation.scrollHeight;
  }

  function finishTranscript(role, finalText = '') {
    const turn = role === 'user' ? liveUserTurn : liveAssistantTurn;
    if (!turn && !finalText) return;
    const current = turn || ensureTurn(role);
    if (finalText) {
      current.value = finalText;
      current.text.textContent = finalText;
    }
    current.row.classList.remove('turn-live');
    if (role === 'user') liveUserTurn = null;
    else liveAssistantTurn = null;
  }

  function bytesToBase64(bytes) {
    let binary = '';
    const chunkSize = 0x8000;
    for (let offset = 0; offset < bytes.length; offset += chunkSize) {
      binary += String.fromCharCode(...bytes.subarray(offset, offset + chunkSize));
    }
    return btoa(binary);
  }

  function int16ToBase64(pcm) {
    return bytesToBase64(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength));
  }

  function base64ToBytes(encoded) {
    const binary = atob(encoded);
    const bytes = new Uint8Array(binary.length);
    for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index);
    return bytes;
  }

  function resampleInt16(input, sourceRate, targetRate) {
    if (sourceRate === targetRate) return input;
    const ratio = sourceRate / targetRate;
    const output = new Int16Array(Math.floor(input.length / ratio));
    for (let index = 0; index < output.length; index += 1) {
      const start = Math.floor(index * ratio);
      const end = Math.max(start + 1, Math.min(input.length, Math.floor((index + 1) * ratio)));
      let sum = 0;
      for (let source = start; source < end; source += 1) sum += input[source];
      output[index] = sum / (end - start);
    }
    return output;
  }

  async function decodeAudioDelta(event) {
    const encoded = event.delta || (event.response && event.response.audio);
    if (!encoded) return null;
    const bytes = base64ToBytes(encoded);
    const format = String(event.format || event.audio_format || 'pcm16').toLowerCase();
    const sourceRate = Number(event.sample_rate_hz || event.sample_rate || OUTPUT_RATE);
    if (format.includes('f32')) {
      const floats = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
      const pcm = new Int16Array(floats.length);
      for (let index = 0; index < floats.length; index += 1) {
        const sample = Math.max(-1, Math.min(1, floats[index]));
        pcm[index] = sample < 0 ? sample * 32768 : sample * 32767;
      }
      return { pcm, sourceRate };
    }
    if (format.includes('wav')) {
      const decoded = await playbackContext.decodeAudioData(bytes.buffer.slice(0));
      const channel = decoded.getChannelData(0);
      const pcm = new Int16Array(channel.length);
      for (let index = 0; index < channel.length; index += 1) {
        const sample = Math.max(-1, Math.min(1, channel[index]));
        pcm[index] = sample < 0 ? sample * 32768 : sample * 32767;
      }
      return { pcm, sourceRate: decoded.sampleRate };
    }
    return {
      pcm: new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 2)),
      sourceRate,
    };
  }

  function updateMeter(pcm) {
    let peak = 0;
    for (let index = 0; index < pcm.length; index += 8) peak = Math.max(peak, Math.abs(pcm[index]));
    meterFill.style.width = `${Math.min(100, (peak / 32768) * 150).toFixed(0)}%`;
  }

  function microphoneUploadEnabled() {
    return running && !muted;
  }

  function flushCapture() {
    if (!socket || socket.readyState !== WebSocket.OPEN || pendingCapture.length === 0) return;
    if (!microphoneUploadEnabled()) {
      pendingCapture = [];
      return;
    }
    const length = pendingCapture.reduce((total, chunk) => total + chunk.length, 0);
    const merged = new Int16Array(length);
    let offset = 0;
    for (const chunk of pendingCapture) {
      merged.set(chunk, offset);
      offset += chunk.length;
    }
    pendingCapture = [];
    const pcm = resampleInt16(merged, captureRate, INPUT_RATE);
    const appendEvent = {
      type: 'input_audio_buffer.append',
      audio: int16ToBase64(pcm),
      format: 'pcm16',
      sample_rate_hz: INPUT_RATE,
    };
    // Omni duplex: ~1 fps camera frame rides the audio append (official
    // MiniCPM-o-Demo contract: one base64 JPEG per ~1 s chunk).
    if (cameraPendingFrame) {
      appendEvent.video_frames = [cameraPendingFrame];
      cameraPendingFrame = null;
    }
    socket.send(JSON.stringify(appendEvent));
  }

  function beginAssistant(responseId) {
    currentResponseId = responseId || currentResponseId;
    responseHasAudio = false;
    assistantActive = true;
    setModel('Speaking');
  }

  function feedPlayback(decoded, responseId) {
    if (!decoded || !decoded.pcm || decoded.pcm.length === 0 || !playbackNode) return;
    const pcm = resampleInt16(decoded.pcm, decoded.sourceRate, playbackRate);
    responseHasAudio = true;
    assistantActive = true;
    setPlayback('Buffering');
    playbackNode.port.postMessage({
      type: 'audio',
      pcm,
      responseId: responseId || currentResponseId,
      initialBufferMs: INITIAL_PLAYBACK_BUFFER_MS,
    }, [pcm.buffer]);
  }

  function requestPlaybackDrain(responseId) {
    if (!playbackNode) return;
    playbackNode.port.postMessage({ type: 'drain', responseId: responseId || currentResponseId });
  }

  function sendPlaybackAck(responseId, playedMs) {
    if (!responseId || !socket || socket.readyState !== WebSocket.OPEN || playedMs <= 0) {
      if (!responseId && playedMs > 0) appendLog('playback ack skipped: missing response id', true);
      return;
    }
    socket.send(JSON.stringify({
      type: 'playback.ack',
      response_id: responseId,
      item_id: `item_${responseId}`,
      played_ms: playedMs,
      committed_ms: playedMs,
    }));
  }

  function playbackDrained(message) {
    const responseId = message.responseId || currentResponseId;
    sendPlaybackAck(responseId, Number(message.playedMs) || 0);
    setPlayback('Idle');
    if (message.underrunMs > 0) {
      appendLog(`playback underrun ${message.underrunMs} ms`);
    }
    window.setTimeout(() => {
      assistantActive = false;
      currentResponseId = null;
      responseHasAudio = false;
      if (running) setModel('Listening');
    }, ECHO_GUARD_MS);
  }

  function handleEvent(event) {
    appendEventLog(event);
    const responseId = responseIdOf(event);
    switch (event.type) {
      case 'session.created':
      case 'session.updated':
        setConnection('Connected', 'online');
        setModel('Listening');
        break;
      case 'response.listen':
        assistantActive = false;
        setModel('Listening');
        break;
      case 'response.created':
      case 'response.speak':
        beginAssistant(responseId);
        break;
      case 'response.audio.delta':
        currentResponseId = responseId || currentResponseId;
        assistantActive = true;
        setModel('Speaking');
        decodeAudioDelta(event)
          .then((decoded) => feedPlayback(decoded, responseId))
          .catch((error) => appendLog(`audio decode failed: ${error.message || error}`, true));
        break;
      case 'response.audio.done':
        requestPlaybackDrain(responseId);
        break;
      case 'response.audio_transcript.delta':
        addTranscript('assistant', event.delta || '');
        break;
      case 'response.audio_transcript.done':
        finishTranscript('assistant', event.transcript || '');
        break;
      case 'conversation.item.input_audio_transcription.delta':
        addTranscript('user', event.delta || '');
        break;
      case 'conversation.item.input_audio_transcription.completed':
        finishTranscript('user', event.transcript || '');
        break;
      case 'response.done':
        finishTranscript('assistant');
        if (!responseHasAudio) requestPlaybackDrain(responseId);
        break;
      case 'playback.acknowledged':
        {
          const acknowledgement = event.event || event;
          runtimeDetail.textContent = `Playback committed ${acknowledgement.committed_ms || 0} ms`;
        }
        break;
      case 'session.closed':
        if (sessionCloseResolver) sessionCloseResolver();
        sessionCloseResolver = null;
        break;
      case 'error':
        setConnection('Error', 'error');
        runtimeDetail.textContent = String(event.error || event.code || 'Server error');
        break;
      default:
        break;
    }
  }

  async function openPlayback() {
    playbackContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: OUTPUT_RATE });
    playbackRate = playbackContext.sampleRate;
    await playbackContext.audioWorklet.addModule(staticAssetUrl('static/playback_worklet.js'));
    playbackNode = new AudioWorkletNode(playbackContext, 'fullduplex-pcm-playback');
    playbackNode.port.onmessage = (message) => {
      if (message.data.type === 'playback-started') setPlayback('Playing');
      else if (message.data.type === 'playback-drained') playbackDrained(message.data);
      else if (message.data.type === 'playback-underrun') {
        runtimeDetail.textContent = `Playback underrun ${message.data.underrunMs || 0} ms`;
      }
    };
    playbackNode.connect(playbackContext.destination);
    await playbackContext.resume();
  }

  async function openCapture() {
    mediaStream = await navigator.mediaDevices.getUserMedia({
      audio: {
        channelCount: 1,
        echoCancellation: true,
        noiseSuppression: true,
        autoGainControl: true,
        sampleRate: { ideal: INPUT_RATE },
      },
    });
    try {
      captureContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: INPUT_RATE });
    } catch (_error) {
      captureContext = new (window.AudioContext || window.webkitAudioContext)();
    }
    captureRate = captureContext.sampleRate;
    await captureContext.audioWorklet.addModule(staticAssetUrl('static/pcm_worklet.js'));
    const source = captureContext.createMediaStreamSource(mediaStream);
    captureNode = new AudioWorkletNode(captureContext, 'fullduplex-pcm-capture');
    captureNode.port.onmessage = (message) => {
      const pcm = new Int16Array(message.data);
      updateMeter(pcm);
      if (microphoneUploadEnabled()) pendingCapture.push(pcm);
    };
    const silentSink = captureContext.createGain();
    silentSink.gain.value = 0;
    source.connect(captureNode);
    captureNode.connect(silentSink).connect(captureContext.destination);
    await captureContext.resume();
  }

  function openSocket() {
    return new Promise((resolve, reject) => {
      const url = realtimeUrl();
      socket = new WebSocket(url);
      let settled = false;
      socket.onopen = () => {
        settled = true;
        const extraBody = {
          auto_response: true,
          minicpmo45_native_duplex: true,
        };
        const session = {
          modalities: ['audio', 'text'],
          voice: 'default',
          extra_body: extraBody,
        };
        // Reference voice for TTS cloning, provided by the server via
        // --ref-audio (mirrors the official demo's default ref audio).
        if (config.refAudio) session.ref_audio = config.refAudio;
        const instructions = systemPromptInput ? systemPromptInput.value.trim() : '';
        if (instructions) session.instructions = instructions;
        socket.send(JSON.stringify({ type: 'session.update', session }));
        runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
        appendLog(`websocket open  ${url}`);
        resolve();
      };
      socket.onmessage = (message) => {
        if (typeof message.data !== 'string') return;
        try {
          handleEvent(JSON.parse(message.data));
        } catch (error) {
          appendLog(`invalid server event: ${error.message || error}`, true);
        }
      };
      socket.onerror = () => {
        if (!settled) {
          settled = true;
          reject(new Error(`WebSocket connection failed: ${url}`));
        }
      };
      socket.onclose = (event) => {
        appendLog(`websocket closed  code=${event.code}`);
        if (running) {
          setConnection('Disconnected', 'error');
          stopSession();
        }
      };
    });
  }

  function formatElapsed(seconds) {
    const minutes = Math.floor(seconds / 60);
    return `${String(minutes).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`;
  }

  function startClock() {
    startedAt = Date.now();
    clockTimer = window.setInterval(() => {
      sessionTimer.textContent = formatElapsed(Math.floor((Date.now() - startedAt) / 1000));
    }, 1000);
  }

  async function startSession() {
    if (running) return;
    callButton.disabled = true;
    setConnection('Connecting', 'connecting');
    runtimeDetail.textContent = 'Requesting microphone access';
    try {
      await openPlayback();
      await openCapture();
      await openSocket();
      running = true;
      muted = false;
      assistantActive = false;
      sendTimer = window.setInterval(flushCapture, SEND_INTERVAL_MS);
      startClock();
      callButton.textContent = 'End session';
      callButton.classList.add('is-active');
      muteButton.disabled = false;
      cameraButton.disabled = false;
      setConnection('Connected', 'online');
      setModel('Listening');
      appendLog('session started');
    } catch (error) {
      appendLog(`start failed: ${error.message || error}`, true);
      setConnection('Error', 'error');
      runtimeDetail.textContent = String(error.message || error);
      await stopSession();
    } finally {
      callButton.disabled = false;
    }
  }

  async function startCamera() {
    if (cameraStream) return;
    cameraStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false });
    cameraPreview.srcObject = cameraStream;
    cameraPreview.style.display = '';
    await cameraPreview.play().catch(() => {});
    // Official omni-duplex cadence: one JPEG (quality 0.7) per ~1 s chunk,
    // no client-side resize (the server normalizes at scale_resolution=448).
    cameraTimer = window.setInterval(() => {
      if (!cameraStream || cameraPreview.videoWidth === 0) return;
      cameraCanvas.width = cameraPreview.videoWidth;
      cameraCanvas.height = cameraPreview.videoHeight;
      cameraCanvas.getContext('2d').drawImage(cameraPreview, 0, 0);
      cameraPendingFrame = cameraCanvas.toDataURL('image/jpeg', 0.7).split(',')[1];
    }, 1000);
    cameraButton.textContent = 'Camera off';
    cameraButton.classList.add('is-active');
    appendLog('camera on (1 fps omni frames)');
  }

  function stopCamera() {
    if (cameraTimer !== null) clearInterval(cameraTimer);
    cameraTimer = null;
    if (cameraStream) {
      for (const track of cameraStream.getTracks()) track.stop();
    }
    cameraStream = null;
    cameraPendingFrame = null;
    cameraPreview.srcObject = null;
    cameraPreview.style.display = 'none';
    cameraButton.textContent = 'Camera';
    cameraButton.classList.remove('is-active');
  }

  cameraButton.addEventListener('click', () => {
    if (cameraStream) {
      stopCamera();
      appendLog('camera off');
      return;
    }
    startCamera().catch((error) => appendLog(`camera failed: ${error.message || error}`, true));
  });

  function waitForSessionClosed(targetSocket, timeoutMs) {
    if (!targetSocket || targetSocket.readyState !== WebSocket.OPEN) return Promise.resolve();
    return new Promise((resolve) => {
      let done = false;
      const finish = () => {
        if (done) return;
        done = true;
        if (sessionCloseResolver === finish) sessionCloseResolver = null;
        resolve();
      };
      sessionCloseResolver = finish;
      window.setTimeout(finish, timeoutMs);
    });
  }

  async function stopSession({ terminal = true } = {}) {
    running = false;
    assistantActive = false;
    pendingCapture = [];
    if (sendTimer !== null) clearInterval(sendTimer);
    if (clockTimer !== null) clearInterval(clockTimer);
    sendTimer = null;
    clockTimer = null;
    if (socket) {
      const closingSocket = socket;
      socket = null;
      closingSocket.onclose = null;
      if (terminal && closingSocket.readyState === WebSocket.OPEN) {
        const closed = waitForSessionClosed(closingSocket, SESSION_CLOSE_TIMEOUT_MS);
        closingSocket.send(JSON.stringify({ type: 'session.close' }));
        await closed;
      }
      closingSocket.close(1000, 'client stop');
    }
    if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
    if (mediaStream) {
      for (const track of mediaStream.getTracks()) track.stop();
    }
    mediaStream = null;
    stopCamera();
    cameraButton.disabled = true;
    if (captureContext) await captureContext.close().catch(() => {});
    if (playbackContext) await playbackContext.close().catch(() => {});
    captureContext = null;
    captureNode = null;
    playbackContext = null;
    playbackNode = null;
    currentResponseId = null;
    responseHasAudio = false;
    meterFill.style.width = '0%';
    sessionTimer.textContent = '00:00';
    callButton.textContent = 'Start session';
    callButton.classList.remove('is-active');
    muteButton.textContent = 'Mute';
    muteButton.classList.remove('is-active');
    muteButton.disabled = true;
    setConnection('Offline', 'offline');
    setModel('Idle');
    setPlayback('Idle');
    if (runtimeDetail.textContent.startsWith('Playback committed')) return;
    if (!runtimeDetail.textContent.startsWith('start failed')) runtimeDetail.textContent = 'No active connection';
  }

  function toggleMute() {
    if (!running) return;
    muted = !muted;
    pendingCapture = [];
    muteButton.textContent = muted ? 'Unmute' : 'Mute';
    muteButton.classList.toggle('is-active', muted);
    appendLog(muted ? 'microphone muted' : 'microphone unmuted');
  }

  callButton.addEventListener('click', () => {
    if (running) stopSession();
    else startSession();
  });
  muteButton.addEventListener('click', toggleMute);
  clearLogButton.addEventListener('click', () => {
    eventLog.textContent = '';
    logCount = 0;
    eventCount.textContent = '0 events';
  });
  window.addEventListener('beforeunload', () => { stopSession({ terminal: false }); });
})();