[Frontend] Add a shared realtime web UI for MiniCPM-o and Qwen3-Omni (#7222) (#7585)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-17 23:32:31 +08:00
1 parent eb1abbadf8
commit 882bf8bc95
53 files changed
+3438 -633

No files matched your search

@@ -0,0 +1,56 @@
/* Shared wire helpers. Profiles translate server events into shell actions. */
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles = root.OmniRealtimeProfiles || {};
profiles.url = (config, location) => {
const url = new URL(config.realtimePath || 'v1/realtime', location);
if (url.protocol === 'http:') url.protocol = 'ws:';
if (url.protocol === 'https:') url.protocol = 'wss:';
return url;
};
/* Errors that leave the session unusable, so the shell has to tear it down.
Everything else costs one operation --- a rejected image, a refused
session.update, a late playback ack --- and the call carries on.
'runtime_append_failed' is deliberately absent: the engine reports it and
keeps the session, unlike 'runtime_append_task_failed' which kills it. */
profiles.fatalErrorCodes = new Set([
'unknown_session',
'session_closed',
'resource_exhausted',
'unsupported',
'unsupported_turn_detection',
'server_vad_unavailable',
'runtime_append_task_failed',
'internal_error',
]);
profiles.event = (event) => {
const responseId = event.response_id || event.response?.id || null;
const action = (kind, extra = {}) => ({ kind, responseId, ...extra });
switch (event.type) {
case 'session.created': case 'session.updated': return action('connected');
case 'response.created': return action('begin');
case 'response.audio.delta': case 'response.output_audio.delta':
return action('audio', { event: { ...event, delta: event.delta || event.audio || event.response?.audio } });
case 'response.audio.done': case 'response.output_audio.done': return action('drain');
case 'response.audio_transcript.delta': case 'response.output_audio_transcript.delta':
return action('text', { role: 'assistant', channel: 'audio', text: event.delta || '' });
case 'response.audio_transcript.done': case 'response.output_audio_transcript.done':
return action('text-final', { role: 'assistant', channel: 'audio', text: event.transcript || '' });
case 'conversation.item.input_audio_transcription.delta':
return action('text', { role: 'user', text: event.delta || '' });
case 'conversation.item.input_audio_transcription.completed':
return action('text-final', { role: 'user', text: event.transcript || '' });
case 'response.done': return action('done');
case 'session.closed': return action('closed');
case 'error': {
const code = event.code || event.error?.code;
return action('error', {
code,
fatal: profiles.fatalErrorCodes.has(code),
message: typeof event.error === 'string' ? event.error : event.error?.message || event.message || code || 'Server error',
});
}
default: return action('ignore');
}
};
})(globalThis);
@@ -0,0 +1,55 @@
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['minicpm-native'] = () => ({
title: 'MiniCPM-o 4.5 Voice', eyebrow: 'Experimental full-duplex runtime',
policy: 'Model-controlled listen / speak',
description: 'Speak naturally. The model can listen and speak at the same time. Camera frames are sent with your audio.',
waiting: 'Listening', camera: true, playbackAck: true, clientCommit: false,
halfDuplex: false, closeSession: true, reconnectEachTurn: false,
readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
presets: {
omni: 'Streaming Omni Conversation.',
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
+ '请用高自然度的方式和用户聊天。你处于双工模式,可以一边听、一边说。'
+ '你是由面壁智能开发的人工智能助手:面壁小钢炮。',
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
},
url(config, location) {
const url = profiles.url(config, location);
url.searchParams.set('duplex', '1');
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
url.searchParams.set('autostart', '0');
return url.toString();
},
initialMessages(config, instructions) {
const session = { modalities: ['audio', 'text'], voice: 'default',
extra_body: { auto_response: true } };
if (config.refAudio) session.ref_audio = config.refAudio;
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio, frame) {
const event = { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
if (frame) event.video_frames = [frame];
return event;
},
commitMessages: () => [],
ack(responseId, playedMs) {
return { type: 'playback.ack', response_id: responseId, item_id: `item_${responseId}`,
played_ms: playedMs, committed_ms: playedMs };
},
mapEvent(event) {
if (event.type === 'response.listen') return { kind: 'listen' };
if (event.type === 'response.speak') return { kind: 'begin', responseId: event.response_id || event.response?.id };
if (event.type === 'playback.acknowledged') {
return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
}
return profiles.event(event);
},
connectionHint: 'Check the MiniCPM duplex deployment and reference voice configuration.',
});
})(globalThis);
@@ -0,0 +1,100 @@
(function (root) {
'use strict';
const profiles = root.OmniRealtimeProfiles;
profiles['qwen3-turn'] = (config) => {
const vad = config.adapter === 'vad';
let imageSeq = 0;
let imageItems = [];
// The engine refuses a ninth image (or 4 MiB of them) for the whole
// session, so the camera track retires its own oldest frame instead.
const MAX_IMAGE_ITEMS = 8;
const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
return {
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
description: vad
? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
inputSampleRate: vad ? 24000 : 16000,
readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
url(config, location) {
const url = profiles.url(config, location);
for (const key of ['native_duplex', 'minicpmo45_native_duplex', 'autostart']) url.searchParams.delete(key);
// Explicit opt-out matches the local file client, including servers
// that auto-select the duplex endpoint when the query is omitted.
url.searchParams.set('duplex', vad ? '1' : '0');
url.searchParams.set('model', config.model || 'Qwen/Qwen3-Omni-30B-A3B-Instruct');
return url.toString();
},
initialMessages(config, instructions) {
imageItems = [];
if (!vad) return [
{ type: 'session.update', model: config.model },
{ type: 'input_audio_buffer.commit', final: false },
];
const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
format: { type: 'audio/pcm', rate: 24000 },
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
silence_duration_ms: 500, create_response: true, interrupt_response: true },
} } };
if (instructions) session.instructions = instructions;
return [{ type: 'session.update', session }];
},
append(audio) {
return vad ? { type: 'input_audio_buffer.append', audio }
: { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
},
// Qwen is a turn model: a frame is a picture added to the conversation,
// not a track interleaved into the audio. That is exactly the OpenAI
// Realtime image interface, so the camera sends conversation items
// rather than an extension field on the append.
imageMessages(frame) {
if (!vad) return [];
const messages = [];
if (imageItems.length >= MAX_IMAGE_ITEMS) {
// Retire before creating: the engine counts what is already stored.
messages.push({ type: 'conversation.item.delete', item_id: imageItems.shift() });
}
const id = `camera_${++imageSeq}`;
imageItems.push(id);
messages.push({ type: 'conversation.item.create', item: {
id, type: 'message', role: 'user',
content: [{ type: 'input_image', image_url: `data:image/jpeg;base64,${frame}` }],
} });
return messages;
},
commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
mapEvent(event) {
if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
// On the shipped STT route transcription.* is model-generated text,
// not a separate ASR transcript of the user. See realtime_connection.py.
return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
}
if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
}
if (vad && (event.type === 'output_audio_buffer.cleared' ||
(event.type === 'response.done' && event.response?.status === 'cancelled'))) {
return { kind: 'interrupt' };
}
if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
const action = profiles.event(event);
if (action.kind === 'error') {
if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
}
return action;
},
connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
};
};
})(globalThis);