[Core] Generalized turn-based /v1/realtime implementation (#8339)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Nick CaoandCodex authored and GitHub committed 2026-10-06 22:07:31 +08:00
1 parent 25fb257667
commit 65699fe94c
16 files changed
+716 -799

No files matched your search

@@ -2,24 +2,24 @@
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['qwen3-turn'] = (config) => {
const vad = config.adapter === 'vad';
const vad = config.turnMode === 'vad';
let interruptOnSpeech = vad;
let imageSeq = 0;
let imageItems = [];
// The engine refuses a ninth image (or 4 MiB of them) for the whole
// session, so the camera track retires its own oldest frame instead.
const MAX_IMAGE_ITEMS = 8;
const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
const hint = 'This Qwen deployment needs session_mode: duplex with Server VAD enabled.';
return {
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based Realtime call',
policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
description: vad
? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Conversation history stays on the Realtime session. No barge-in or camera input.',
waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
inputSampleRate: vad ? 24000 : 16000,
readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad,
inputSampleRate: 24000,
readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
url(config, location) {
const url = profiles.url(config, location);
@@ -33,22 +33,17 @@
initialMessages(config, instructions) {
imageItems = [];
interruptOnSpeech = vad;
if (!vad) return [
{ type: 'session.update', model: config.model },
{ type: 'input_audio_buffer.commit', final: false },
];
const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
const session = { model: config.model, audio: { input: {
format: { type: 'audio/pcm', rate: 24000 },
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true },
turn_detection: vad ? { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true } : null,
} } };
if (vad) session.overlap_policy = 'barge_in_on_speech';
else session.type = 'realtime';
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio) {
return vad ? { type: 'input_audio_buffer.append', audio }
: { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
},
append: (audio) => ({ type: 'input_audio_buffer.append', audio }),
// Qwen is a turn model: a frame is a picture added to the conversation,
// not a track interleaved into the audio. That is exactly the OpenAI
// Realtime image interface, so the camera sends conversation items
@@ -68,7 +63,10 @@
} });
return messages;
},
commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
commitMessages: () => vad ? [] : [
{ type: 'input_audio_buffer.commit' },
{ type: 'response.create' },
],
ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
mapEvent(event) {
@@ -87,30 +85,28 @@
return { kind: 'interrupt' };
}
if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
// On the shipped STT route transcription.* is model-generated text,
// not a separate ASR transcript of the user. See realtime_connection.py.
if (event.type === 'response.output_text.delta') {
return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
}
if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
if (event.type === 'response.output_text.done') {
return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
}
if (vad && (event.type === 'output_audio_buffer.cleared' ||
(event.type === 'response.done' && event.response?.status === 'cancelled'))) {
return { kind: 'interrupt', responseId: event.response_id || event.response?.id || null };
}
if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
if (vad && event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
if (!vad && event.type === 'input_audio_buffer.cleared') return { kind: 'ignore' };
if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
const action = profiles.event(event);
if (action.kind === 'error') {
if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact and restart the UI.';
else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
}
return action;
},
connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
connectionHint: vad ? hint : 'Check the Qwen turn-based /v1/realtime deployment and model name.',
};
};
})(globalThis);