[Bugfix] Fix Qwen3-Omni realtime playback interruption and conversation history (#7791)

Signed-off-by: psv666 <2693925048@qq.com>

examples/online_serving/realtime_web/app/static/app.js

@@ -79,6 +79,13 @@
   let echoTimer = null;
   let sessionGeneration = 0;
   let audioChain = Promise.resolve();
+  // Retain interrupted IDs until startSession() resets the session. Queued
+  // events and pending audio decodes can outlive response.done, so removing
+  // IDs there could resume interrupted playback. This set grows with the
+  // interruptions in a call; sessionGeneration rejects old-session decodes
+  // after the set is cleared for a new session.
+  const interruptedResponses = new Set();
+  const pendingEvents = new Set();
   let assistantTextChannel = null;
   let connectionReady = false;
   let turnCounter = 0;
@@ -389,8 +396,9 @@
     currentResponseId = action.responseId || currentResponseId || `turn-${turnCounter}`;
     setModel('Speaking');
     const generation = sessionGeneration;
+    const responseId = currentResponseId;
     const decoded = await decodeAudioDelta(action.event);
-    if (generation === sessionGeneration) feedPlayback(decoded, currentResponseId);
+    if (generation === sessionGeneration && !interruptedResponses.has(responseId)) feedPlayback(decoded, responseId);
   }
 
   function handleTranscriptEvent(action) {
@@ -410,19 +418,32 @@
     } else finishTranscript(action.role, action.text);
   }
 
-  async function handleEvent(event) {
+  async function handleEvent(event, action = profile.mapEvent(event)) {
     appendEventLog(event);
-    const action = profile.mapEvent(event);
-    const responseId = action.responseId;
+    const responseId = action.responseId || responseIdOf(event);
+    if (responseId && interruptedResponses.has(responseId) && action.kind !== 'interrupt') return;
     switch (action.kind) {
       case 'connected':
         setConnection('Connected', 'online');
         if (!assistantActive) setModel(profile.waiting);
         break;
-      case 'interrupt':
+      case 'interrupt': {
+        if (responseId) interruptedResponses.add(responseId);
+        // A late cancellation belongs to its own response, not the newer one
+        // currently playing. Speech-start events interrupt the local playback.
+        if (responseId && responseId !== currentResponseId) break;
+        if (currentResponseId) interruptedResponses.add(currentResponseId);
         sessionGeneration += 1;
-        // A cancelled response never reaches 'done', so nothing else would
-        // disarm the turn watchdog it left armed.
+        audioChain = Promise.resolve();
+        // Move queued events off the interrupted decode, preserving any newer
+        // response already received. Its chunks must not be lost with the old one.
+        const queued = [...pendingEvents];
+        pendingEvents.clear();
+        for (const pending of queued) enqueueEvent(pending.event, pending.action, pending.source);
+        clearTimeout(echoTimer);
+        echoTimer = null;
+        // Local playback completes immediately on interruption, including
+        // when the server already completed generation.
         clearTimeout(turnTimeout);
         if (playbackNode) playbackNode.port.postMessage({ type: 'clear' });
         responseComplete = true;
@@ -435,6 +456,7 @@
         setPlayback('Idle');
         setModel(profile.waiting);
         break;
+      }
       case 'listen':
         assistantActive = false;
         setModel(profile.waiting);
@@ -538,6 +560,18 @@
     await captureContext.resume();
   }
 
+  function enqueueEvent(event, action, source) {
+    const pending = { event, action, source };
+    pendingEvents.add(pending);
+    const generation = sessionGeneration;
+    audioChain = audioChain.then(() => {
+      if (!pendingEvents.delete(pending)) return;
+      if (socket === source || action.kind === 'closed') return handleEvent(event, action);
+    }).catch((error) => {
+      if (generation === sessionGeneration) return failSession(`Server event failed: ${error.message}`);
+    });
+  }
+
   function openSocket() {
     return new Promise((resolve, reject) => {
       const url = realtimeUrl();
@@ -591,11 +625,13 @@
           runtimeDetail.textContent = `${captureRate} Hz capture / ${playbackRate} Hz playback`;
           resolve();
         }
-        // Serialize decoding with terminal events: a drain must never overtake
-        // an asynchronously decoded WAV chunk.
-        audioChain = audioChain.then(() => {
-          if (socket === current || action.kind === 'closed') return handleEvent(event);
-        }).catch((error) => failSession(`Server event failed: ${error.message}`));
+        // Stop the speaker immediately, even while an old WAV is decoding.
+        // Ordinary completion still waits for its preceding audio chunks.
+        if (action.kind === 'interrupt') {
+          void handleEvent(event, action);
+          return;
+        }
+        enqueueEvent(event, action, current);
       };
       current.onerror = () => rejectOnce(`WebSocket connection failed. ${profile.connectionHint}`);
       current.onclose = (event) => {
@@ -628,6 +664,8 @@
     try {
       sessionGeneration += 1;
       audioChain = Promise.resolve();
+      interruptedResponses.clear();
+      pendingEvents.clear();
       await openPlayback();
       await openCapture();
       await openSocket();
@@ -731,6 +769,7 @@
 
   async function cleanupSession({ terminal = true } = {}) {
     sessionGeneration += 1;
+    pendingEvents.clear();
     connectionReady = false;
     clearTimeout(echoTimer);
     clearTimeout(turnTimeout);

examples/online_serving/realtime_web/app/static/profiles/qwen3_turn.js

@@ -3,6 +3,7 @@
   const profiles = root.OmniRealtimeProfiles;
   profiles['qwen3-turn'] = (config) => {
     const vad = config.adapter === 'vad';
+    let interruptOnSpeech = vad;
     let imageSeq = 0;
     let imageItems = [];
     // The engine refuses a ninth image (or 4 MiB of them) for the whole
@@ -31,6 +32,7 @@
       },
       initialMessages(config, instructions) {
         imageItems = [];
+        interruptOnSpeech = vad;
         if (!vad) return [
           { type: 'session.update', model: config.model },
           { type: 'input_audio_buffer.commit', final: false },
@@ -70,6 +72,20 @@
       ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
         item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
       mapEvent(event) {
+        if (event.type === 'session.created' || event.type === 'session.updated') {
+          const input = event.session?.audio?.input;
+          const turnDetection = input && 'turn_detection' in input
+            ? input.turn_detection : event.session?.turn_detection;
+          if (turnDetection !== undefined) {
+            interruptOnSpeech = vad && turnDetection?.type === 'server_vad'
+              && turnDetection.interrupt_response !== false;
+          }
+        }
+        // Generation can finish long before the speaker drains its queue. In
+        // that case the server has no active response left to cancel for us.
+        if (interruptOnSpeech && event.type === 'input_audio_buffer.speech_started') {
+          return { kind: 'interrupt' };
+        }
         if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
         if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
           // On the shipped STT route transcription.* is model-generated text,
@@ -81,7 +97,7 @@
         }
         if (vad && (event.type === 'output_audio_buffer.cleared' ||
             (event.type === 'response.done' && event.response?.status === 'cancelled'))) {
-          return { kind: 'interrupt' };
+          return { kind: 'interrupt', responseId: event.response_id || event.response?.id || null };
         }
         if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
         if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };