[Core] Generalized turn-based /v1/realtime implementation (#8339)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Codex <noreply@openai.com>

tests/examples/test_minicpmo_realtime_web_server.py

@@ -108,10 +108,10 @@ def test_build_app_exposes_realtime_websocket_and_static_assets():
 
 
 def test_qwen_config_does_not_inject_reference_voice():
-    client = TestClient(build_app(profile="qwen3-turn", adapter="vad", model="Qwen/Qwen3-Omni-30B-A3B-Instruct"))
+    client = TestClient(build_app(profile="qwen3-turn", turn_mode="vad", model="Qwen/Qwen3-Omni-30B-A3B-Instruct"))
     page = client.get("/").text
     assert '"profile": "qwen3-turn"' in page
-    assert '"adapter": "vad"' in page
+    assert '"turnMode": "vad"' in page
     assert '"refAudio": null' in page
     for name in ("common", "minicpm_native", "qwen3_turn", "aura_ptt"):
         assert client.get(f"/static/profiles/{name}.js").status_code == 200

tests/examples/test_realtime_web_profiles.cjs

@@ -28,40 +28,57 @@ test('MiniCPM keeps native flags, reference voice, camera, acknowledgements and
   assert.equal(native.halfDuplex, false);
 });
 
-test('STT uses the shipped commit sequence and audio payload, including each new turn', () => {
-  const stt = profiles['qwen3-turn'](config);
-  for (let turn = 0; turn < 2; turn++) {
-    assert.deepEqual(plain(stt.initialMessages(config, 'ignored')), [
-      { type: 'session.update', model: config.model },
-      { type: 'input_audio_buffer.commit', final: false },
-    ]);
-    assert.deepEqual(plain(stt.commitMessages()), [{ type: 'input_audio_buffer.commit', final: true }]);
-  }
-  assert.equal(stt.reconnectEachTurn, true);
-  assert.equal(stt.mapEvent({ type: 'response.output_audio.delta', audio: 'PCM' }).event.delta, 'PCM');
-  assert.equal(stt.mapEvent({ type: 'response.output_audio.done' }).kind, 'done');
-  assert.equal(stt.mapEvent({ type: 'transcription.delta', delta: 'hello' }).role, 'assistant');
+test('manual turns use standard Realtime events and keep one session', () => {
+  const manual = profiles['qwen3-turn'](config);
+  const [update] = plain(manual.initialMessages(config, 'help'));
+  assert.equal(update.type, 'session.update');
+  assert.deepEqual(update.session, {
+    type: 'realtime',
+    model: config.model,
+    audio: { input: {
+      format: { type: 'audio/pcm', rate: 24000 },
+      turn_detection: null,
+    } },
+    instructions: 'help',
+  });
+  assert.equal(manual.readyEvent, 'session.updated');
+  assert.equal(manual.inputSampleRate, 24000);
+  assert.deepEqual(plain(manual.append('PCM')), { type: 'input_audio_buffer.append', audio: 'PCM' });
+  assert.deepEqual(plain(manual.commitMessages()), [
+    { type: 'input_audio_buffer.commit' },
+    { type: 'response.create' },
+  ]);
+  assert.equal(manual.reconnectEachTurn, undefined);
+  assert.equal(manual.mapEvent({ type: 'response.output_audio.delta', delta: 'PCM' }).event.delta, 'PCM');
+  assert.equal(manual.mapEvent({ type: 'response.output_audio.done' }).kind, 'drain');
+  assert.equal(manual.mapEvent({ type: 'response.done' }).kind, 'done');
+  assert.equal(manual.mapEvent({ type: 'response.output_audio_transcript.delta', delta: 'hello' }).role, 'assistant');
+  assert.equal(manual.mapEvent({ type: 'input_audio_buffer.cleared' }).kind, 'ignore');
+  assert.equal(manual.mapEvent({ type: 'transcription.delta', delta: 'hello' }).kind, 'ignore');
 });
 
 test('Qwen enables camera and playback ACK only for the duplex VAD profile', () => {
-  for (const adapter of ['stt', 'vad']) {
-    const p = profiles['qwen3-turn']({ ...config, adapter });
+  for (const turnMode of ['manual', 'vad']) {
+    const p = profiles['qwen3-turn']({ ...config, turnMode });
     const url = new URL(p.url({ ...config, realtimePath: 'wss://backend/v1/realtime?native_duplex=1&minicpmo45_native_duplex=1' }, 'http://localhost/'));
     assert.equal(url.protocol, 'wss:');
-    assert.equal(url.searchParams.get('duplex'), adapter === 'vad' ? '1' : '0');
+    assert.equal(url.searchParams.get('duplex'), turnMode === 'vad' ? '1' : '0');
     assert.equal(url.searchParams.has('native_duplex'), false);
     assert.equal(url.searchParams.has('minicpmo45_native_duplex'), false);
     assert.equal(p.append('PCM', 'JPEG').video_frames, undefined);
-    assert.equal(plain(p.imageMessages('JPEG')).length, adapter === 'vad' ? 1 : 0);
-    assert.equal(p.ack('r', 100)?.type || null, adapter === 'vad' ? 'playback.ack' : null);
-    assert.equal(p.camera, adapter === 'vad');
+    assert.equal(plain(p.imageMessages('JPEG')).length, turnMode === 'vad' ? 1 : 0);
+    assert.equal(p.ack('r', 100)?.type || null, turnMode === 'vad' ? 'playback.ack' : null);
+    assert.equal(p.camera, turnMode === 'vad');
     assert.equal(p.mapEvent({ type: 'response.listen' }).kind, 'ignore');
   }
 });
 
 test('VAD uses nested format and interruptible endpoint detection', () => {
-  const vad = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
+  const vad = profiles['qwen3-turn']({ ...config, turnMode: 'vad' });
   const [update] = vad.initialMessages(config, 'help');
+  assert.equal(update.session.type, undefined);
+  assert.equal(update.session.overlap_policy, 'barge_in_on_speech');
+  assert.equal(update.session.instructions, 'help');
   assert.deepEqual(plain(update.session.audio.input), {
     format: { type: 'audio/pcm', rate: 24000 },
     turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
@@ -78,7 +95,7 @@ test('VAD uses nested format and interruptible endpoint detection', () => {
   assert.match(vad.mapEvent({ type: 'error', code: 'server_vad_initialization_failed' }).message, /Silero/);
 });
 
-function shell(profileName, adapter = 'stt', options = {}) {
+function shell(profileName, turnMode = 'manual', options = {}) {
   const elements = new Map();
   class Element {
     constructor() { this.style = {}; this.listeners = {}; this.children = []; this.value = ''; this.textContent = ''; this.classList = { add() {}, remove() {}, toggle() {} }; }
@@ -103,7 +120,7 @@ function shell(profileName, adapter = 'stt', options = {}) {
       this.readyState = 1; this.sent = []; sockets.push(this);
       queueMicrotask(() => {
         this.onopen?.();
-        if (!options.silent) this.onmessage?.({ data: JSON.stringify(options.error || { type: adapter === 'stt' && profileName === 'qwen3-turn' ? 'session.created' : 'session.updated' }) });
+        if (!options.silent) this.onmessage?.({ data: JSON.stringify(options.error || { type: 'session.updated' }) });
       });
     }
     send(value) { this.sent.push(JSON.parse(value)); }
@@ -141,7 +158,7 @@ function shell(profileName, adapter = 'stt', options = {}) {
     URL, document, AudioContext, AudioWorkletNode, WebSocket: Socket,
     navigator: { mediaDevices: { getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }) } },
     location: { href: 'https://localhost/' },
-    OMNI_REALTIME_CONFIG: { ...config, profile: profileName, adapter },
+    OMNI_REALTIME_CONFIG: { ...config, profile: profileName, turnMode },
     addEventListener() {},
     setTimeout(fn, ms) { const id = ++timerId; timers.set(id, { fn, ms }); return id; },
     clearTimeout(id) { timers.delete(id); },
@@ -174,26 +191,53 @@ function shell(profileName, adapter = 'stt', options = {}) {
   };
 }
 
-test('shared shell STT sends final commit and starts a fresh second turn only after playback', async () => {
+test('shared shell sends standard manual turns and keeps the Realtime session open', async () => {
   const app = shell('qwen3-turn');
   await app.ui.startSession();
   assert.equal(app.ui.microphoneUploadEnabled(), true);
+  assert.deepEqual(app.sockets[0].sent[0], {
+    type: 'session.update',
+    session: {
+      type: 'realtime',
+      model: config.model,
+      audio: { input: {
+        format: { type: 'audio/pcm', rate: 24000 },
+        turn_detection: null,
+      } },
+      instructions: 'You are a helpful assistant. Answer clearly and concisely.',
+    },
+  });
   app.ui.capture(); app.ui.flushCapture();
+  const append = app.sockets[0].sent.at(-1);
+  assert.equal(append.type, 'input_audio_buffer.append');
+  assert.equal(typeof append.audio, 'string');
+  assert.equal(append.format, undefined);
+  assert.equal(append.sample_rate_hz, undefined);
   app.elements.get('sendTurnButton').listeners.click();
-  assert.equal(app.sockets[0].sent.at(-1).final, true);
+  assert.deepEqual(app.sockets[0].sent.slice(-2), [
+    { type: 'input_audio_buffer.commit' },
+    { type: 'response.create' },
+  ]);
   assert.equal(app.ui.microphoneUploadEnabled(), false);
-  await app.ui.handleEvent({ type: 'response.output_audio.delta', audio: 'AAAAAA==' });
+  await app.ui.handleEvent({ type: 'response.created', response: { id: 'turn-1' } });
+  await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'turn-1' });
   await app.ui.handleEvent({ type: 'response.output_audio.done' });
-  await app.echo();
-  assert.equal(app.sockets.length, 1, 'wait for actual speaker drain');
+  await app.ui.handleEvent({ type: 'response.done', response: { id: 'turn-1', status: 'completed' } });
   app.ui.playbackDrained({ responseId: 'turn-1', playedMs: 1 });
   await app.echo();
-  assert.equal(app.sockets.length, 2);
-  assert.equal(app.sockets[1].sent[1].final, false);
+  assert.equal(app.sockets.length, 1, 'manual turns preserve the same Realtime session');
   assert.equal(app.ui.microphoneUploadEnabled(), true);
   assert.equal(app.elements.get('cameraButton').hidden, true);
   assert.equal(app.sockets.flatMap(s => s.sent).some(e => e.type === 'playback.ack'), false);
   assert.equal(app.elements.get('pttButton').hidden, true);
+
+  app.ui.capture(); app.ui.flushCapture();
+  app.elements.get('sendTurnButton').listeners.click();
+  assert.deepEqual(app.sockets[0].sent.slice(-2), [
+    { type: 'input_audio_buffer.commit' },
+    { type: 'response.create' },
+  ]);
+  assert.equal(app.sockets.length, 1);
   await app.ui.stopSession({ terminal: false });
 });
 
@@ -360,7 +404,7 @@ test('Qwen VAD interruption clears playback while continuing microphone upload',
 
 
 test('Qwen runtime errors do not suggest changing a working deployment', () => {
-  const p = profiles['qwen3-turn']({ adapter: 'vad' });
+  const p = profiles['qwen3-turn']({ turnMode: 'vad' });
   const message = 'playback.ack arrived after a later user input was committed.';
   assert.equal(p.mapEvent({ type: 'error', code: 'playback_ack_too_late', error: message }).message, message);
 });
@@ -403,7 +447,7 @@ test('late interruption cursor acknowledges old response without finishing the n
 
 
 test('Qwen camera frames ride the OpenAI image interface, not the audio append', () => {
-  const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
+  const p = profiles['qwen3-turn']({ ...config, turnMode: 'vad' });
   assert.deepEqual(plain(p.append('PCM', 'JPEG')), { type: 'input_audio_buffer.append', audio: 'PCM' });
 
   const [created] = plain(p.imageMessages('JPEG'));
@@ -415,7 +459,7 @@ test('Qwen camera frames ride the OpenAI image interface, not the audio append',
 });
 
 test('the camera retires its oldest image instead of exhausting the session budget', () => {
-  const p = profiles['qwen3-turn']({ ...config, adapter: 'vad' });
+  const p = profiles['qwen3-turn']({ ...config, turnMode: 'vad' });
   const ids = [];
   for (let i = 0; i < 8; i++) {
     const messages = plain(p.imageMessages('JPEG'));
@@ -543,7 +587,7 @@ test('AURA shell holds speech until PTT and opens one vision turn per two frames
 });
 
 test('AURA hold-to-talk stops local playback and does not cancel the response', async () => {
-  const app = shell('aura-ptt', 'stt', { realPlayback: true });
+  const app = shell('aura-ptt', 'manual', { realPlayback: true });
   await app.ui.startSession();
   receive(app, { type: 'response.created', response: { id: 'old' } });
   receive(app, audioChunk('old'));
@@ -679,7 +723,7 @@ test('late cancellation of an old Qwen response does not clear a newer response'
 });
 
 test('Qwen speech interruption follows accepted turn detection; MiniCPM mapping stays unchanged', () => {
-  const qwen = profiles['qwen3-turn']({ adapter: 'vad' });
+  const qwen = profiles['qwen3-turn']({ turnMode: 'vad' });
   const speech = { type: 'input_audio_buffer.speech_started', item_id: 'next' };
   assert.equal(qwen.mapEvent(speech).kind, 'interrupt');
   qwen.mapEvent({ type: 'session.updated', session: { audio: { input: { turn_detection: {
@@ -692,7 +736,7 @@ test('Qwen speech interruption follows accepted turn detection; MiniCPM mapping
     type: 'server_vad', interrupt_response: true,
   } } } } });
   assert.equal(qwen.mapEvent(speech).kind, 'interrupt');
-  assert.equal(profiles['qwen3-turn']({ adapter: 'stt' }).mapEvent(speech).kind, 'ignore');
+  assert.equal(profiles['qwen3-turn']({ turnMode: 'manual' }).mapEvent(speech).kind, 'ignore');
   assert.equal(profiles['minicpm-native']().mapEvent(speech).kind, 'ignore');
 });