Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
eb1abbadf8
commit
882bf8bc95
53 files changed
+3438
-633
No files matched your search
@@ -0,0 +1,56 @@
|
||||
/* Shared wire helpers. Profiles translate server events into shell actions. */
|
||||
(function (root) {
|
||||
'use strict';
|
||||
const profiles = root.OmniRealtimeProfiles = root.OmniRealtimeProfiles || {};
|
||||
profiles.url = (config, location) => {
|
||||
const url = new URL(config.realtimePath || 'v1/realtime', location);
|
||||
if (url.protocol === 'http:') url.protocol = 'ws:';
|
||||
if (url.protocol === 'https:') url.protocol = 'wss:';
|
||||
return url;
|
||||
};
|
||||
/* Errors that leave the session unusable, so the shell has to tear it down.
|
||||
Everything else costs one operation --- a rejected image, a refused
|
||||
session.update, a late playback ack --- and the call carries on.
|
||||
'runtime_append_failed' is deliberately absent: the engine reports it and
|
||||
keeps the session, unlike 'runtime_append_task_failed' which kills it. */
|
||||
profiles.fatalErrorCodes = new Set([
|
||||
'unknown_session',
|
||||
'session_closed',
|
||||
'resource_exhausted',
|
||||
'unsupported',
|
||||
'unsupported_turn_detection',
|
||||
'server_vad_unavailable',
|
||||
'runtime_append_task_failed',
|
||||
'internal_error',
|
||||
]);
|
||||
profiles.event = (event) => {
|
||||
const responseId = event.response_id || event.response?.id || null;
|
||||
const action = (kind, extra = {}) => ({ kind, responseId, ...extra });
|
||||
switch (event.type) {
|
||||
case 'session.created': case 'session.updated': return action('connected');
|
||||
case 'response.created': return action('begin');
|
||||
case 'response.audio.delta': case 'response.output_audio.delta':
|
||||
return action('audio', { event: { ...event, delta: event.delta || event.audio || event.response?.audio } });
|
||||
case 'response.audio.done': case 'response.output_audio.done': return action('drain');
|
||||
case 'response.audio_transcript.delta': case 'response.output_audio_transcript.delta':
|
||||
return action('text', { role: 'assistant', channel: 'audio', text: event.delta || '' });
|
||||
case 'response.audio_transcript.done': case 'response.output_audio_transcript.done':
|
||||
return action('text-final', { role: 'assistant', channel: 'audio', text: event.transcript || '' });
|
||||
case 'conversation.item.input_audio_transcription.delta':
|
||||
return action('text', { role: 'user', text: event.delta || '' });
|
||||
case 'conversation.item.input_audio_transcription.completed':
|
||||
return action('text-final', { role: 'user', text: event.transcript || '' });
|
||||
case 'response.done': return action('done');
|
||||
case 'session.closed': return action('closed');
|
||||
case 'error': {
|
||||
const code = event.code || event.error?.code;
|
||||
return action('error', {
|
||||
code,
|
||||
fatal: profiles.fatalErrorCodes.has(code),
|
||||
message: typeof event.error === 'string' ? event.error : event.error?.message || event.message || code || 'Server error',
|
||||
});
|
||||
}
|
||||
default: return action('ignore');
|
||||
}
|
||||
};
|
||||
})(globalThis);
|
||||
@@ -0,0 +1,55 @@
|
||||
(function (root) {
|
||||
'use strict';
|
||||
const profiles = root.OmniRealtimeProfiles;
|
||||
profiles['minicpm-native'] = () => ({
|
||||
title: 'MiniCPM-o 4.5 Voice', eyebrow: 'Experimental full-duplex runtime',
|
||||
policy: 'Model-controlled listen / speak',
|
||||
description: 'Speak naturally. The model can listen and speak at the same time. Camera frames are sent with your audio.',
|
||||
waiting: 'Listening', camera: true, playbackAck: true, clientCommit: false,
|
||||
halfDuplex: false, closeSession: true, reconnectEachTurn: false,
|
||||
readyEvent: 'session.updated', instructions: true, sendIntervalMs: 200,
|
||||
presets: {
|
||||
omni: 'Streaming Omni Conversation.',
|
||||
chinese_call: '扮演一个具有以上声音特征的助手。请认真、高质量地回复用户的问题。'
|
||||
+ '请用高自然度的方式和用户聊天。你处于双工模式,可以一边听、一边说。'
|
||||
+ '你是由面壁智能开发的人工智能助手:面壁小钢炮。',
|
||||
english_call: 'Replicate the tone and style from the input audio. Your task is to be '
|
||||
+ 'a helpful assistant using this voice pattern. Please answer the user\'s questions '
|
||||
+ 'seriously and in a high quality. Please chat with the user in a high naturalness '
|
||||
+ 'style. You are in duplex mode, where you can listen and speak at the same time.',
|
||||
},
|
||||
url(config, location) {
|
||||
const url = profiles.url(config, location);
|
||||
url.searchParams.set('duplex', '1');
|
||||
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
|
||||
url.searchParams.set('autostart', '0');
|
||||
return url.toString();
|
||||
},
|
||||
initialMessages(config, instructions) {
|
||||
const session = { modalities: ['audio', 'text'], voice: 'default',
|
||||
extra_body: { auto_response: true } };
|
||||
if (config.refAudio) session.ref_audio = config.refAudio;
|
||||
if (instructions) session.instructions = instructions;
|
||||
return [{ type: 'session.update', session }];
|
||||
},
|
||||
append(audio, frame) {
|
||||
const event = { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
|
||||
if (frame) event.video_frames = [frame];
|
||||
return event;
|
||||
},
|
||||
commitMessages: () => [],
|
||||
ack(responseId, playedMs) {
|
||||
return { type: 'playback.ack', response_id: responseId, item_id: `item_${responseId}`,
|
||||
played_ms: playedMs, committed_ms: playedMs };
|
||||
},
|
||||
mapEvent(event) {
|
||||
if (event.type === 'response.listen') return { kind: 'listen' };
|
||||
if (event.type === 'response.speak') return { kind: 'begin', responseId: event.response_id || event.response?.id };
|
||||
if (event.type === 'playback.acknowledged') {
|
||||
return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
|
||||
}
|
||||
return profiles.event(event);
|
||||
},
|
||||
connectionHint: 'Check the MiniCPM duplex deployment and reference voice configuration.',
|
||||
});
|
||||
})(globalThis);
|
||||
@@ -0,0 +1,100 @@
|
||||
(function (root) {
|
||||
'use strict';
|
||||
const profiles = root.OmniRealtimeProfiles;
|
||||
profiles['qwen3-turn'] = (config) => {
|
||||
const vad = config.adapter === 'vad';
|
||||
let imageSeq = 0;
|
||||
let imageItems = [];
|
||||
// The engine refuses a ninth image (or 4 MiB of them) for the whole
|
||||
// session, so the camera track retires its own oldest frame instead.
|
||||
const MAX_IMAGE_ITEMS = 8;
|
||||
const hint = 'This Qwen deploy needs session_mode: duplex and Server VAD support; use --stt or enable Server VAD.';
|
||||
return {
|
||||
title: 'Qwen3-Omni Voice', eyebrow: vad ? 'Engine-owned duplex call' : 'Turn-based realtime call',
|
||||
policy: vad ? 'Server VAD • interruptible responses' : 'Send turn • one response at a time',
|
||||
description: vad
|
||||
? 'Pause to send your turn; speak again to interrupt. Microphone upload continues during playback. Requires the Qwen duplex plugin and Silero. Optional camera frames accompany each spoken turn.'
|
||||
: 'Speak, then press Send turn. Microphone upload pauses while the model answers. Each turn uses a fresh connection; conversation history is not carried between turns. No barge-in or camera input.',
|
||||
waiting: 'Waiting for you', camera: vad, cameraMaxDimension: 448, playbackAck: vad, clientCommit: !vad,
|
||||
waitForResponseDone: true, halfDuplex: !vad, deduplicateTranscript: true, closeSession: vad, reconnectEachTurn: !vad,
|
||||
inputSampleRate: vad ? 24000 : 16000,
|
||||
readyEvent: vad ? 'session.updated' : 'session.created', instructions: vad, sendIntervalMs: 200,
|
||||
presets: { assistant: 'You are a helpful assistant. Answer clearly and concisely.' },
|
||||
url(config, location) {
|
||||
const url = profiles.url(config, location);
|
||||
for (const key of ['native_duplex', 'minicpmo45_native_duplex', 'autostart']) url.searchParams.delete(key);
|
||||
// Explicit opt-out matches the local file client, including servers
|
||||
// that auto-select the duplex endpoint when the query is omitted.
|
||||
url.searchParams.set('duplex', vad ? '1' : '0');
|
||||
url.searchParams.set('model', config.model || 'Qwen/Qwen3-Omni-30B-A3B-Instruct');
|
||||
return url.toString();
|
||||
},
|
||||
initialMessages(config, instructions) {
|
||||
imageItems = [];
|
||||
if (!vad) return [
|
||||
{ type: 'session.update', model: config.model },
|
||||
{ type: 'input_audio_buffer.commit', final: false },
|
||||
];
|
||||
const session = { model: config.model, overlap_policy: 'barge_in_on_speech', audio: { input: {
|
||||
format: { type: 'audio/pcm', rate: 24000 },
|
||||
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
|
||||
silence_duration_ms: 500, create_response: true, interrupt_response: true },
|
||||
} } };
|
||||
if (instructions) session.instructions = instructions;
|
||||
return [{ type: 'session.update', session }];
|
||||
},
|
||||
append(audio) {
|
||||
return vad ? { type: 'input_audio_buffer.append', audio }
|
||||
: { type: 'input_audio_buffer.append', audio, format: 'pcm16', sample_rate_hz: 16000 };
|
||||
},
|
||||
// Qwen is a turn model: a frame is a picture added to the conversation,
|
||||
// not a track interleaved into the audio. That is exactly the OpenAI
|
||||
// Realtime image interface, so the camera sends conversation items
|
||||
// rather than an extension field on the append.
|
||||
imageMessages(frame) {
|
||||
if (!vad) return [];
|
||||
const messages = [];
|
||||
if (imageItems.length >= MAX_IMAGE_ITEMS) {
|
||||
// Retire before creating: the engine counts what is already stored.
|
||||
messages.push({ type: 'conversation.item.delete', item_id: imageItems.shift() });
|
||||
}
|
||||
const id = `camera_${++imageSeq}`;
|
||||
imageItems.push(id);
|
||||
messages.push({ type: 'conversation.item.create', item: {
|
||||
id, type: 'message', role: 'user',
|
||||
content: [{ type: 'input_image', image_url: `data:image/jpeg;base64,${frame}` }],
|
||||
} });
|
||||
return messages;
|
||||
},
|
||||
commitMessages: () => vad ? [] : [{ type: 'input_audio_buffer.commit', final: true }],
|
||||
ack: (responseId, playedMs) => vad ? { type: 'playback.ack', response_id: responseId,
|
||||
item_id: `item_${responseId}`, played_ms: playedMs, committed_ms: playedMs } : null,
|
||||
mapEvent(event) {
|
||||
if (event.type === 'playback.acknowledged') return { kind: 'ack', committedMs: (event.event || event).committed_ms || 0 };
|
||||
if (event.type === 'response.output_text.delta' || event.type === 'transcription.delta') {
|
||||
// On the shipped STT route transcription.* is model-generated text,
|
||||
// not a separate ASR transcript of the user. See realtime_connection.py.
|
||||
return { kind: 'text', role: 'assistant', channel: 'text', text: event.delta || '' };
|
||||
}
|
||||
if (event.type === 'response.output_text.done' || event.type === 'transcription.done') {
|
||||
return { kind: 'text-final', role: 'assistant', channel: 'text', text: event.text || '' };
|
||||
}
|
||||
if (vad && (event.type === 'output_audio_buffer.cleared' ||
|
||||
(event.type === 'response.done' && event.response?.status === 'cancelled'))) {
|
||||
return { kind: 'interrupt' };
|
||||
}
|
||||
if (event.type === 'input_audio_buffer.committed') return { kind: 'begin' };
|
||||
if (event.type === 'input_audio_buffer.cleared') return { kind: 'backpressure', message: 'Input cleared. Model is answering; please wait.' };
|
||||
if (!vad && event.type === 'response.output_audio.done') return { kind: 'done' };
|
||||
const action = profiles.event(event);
|
||||
if (action.kind === 'error') {
|
||||
if (action.code === 'input_backpressure') return { ...action, kind: 'backpressure', message: 'Input buffer is full; please repeat dropped speech after the response.' };
|
||||
if (action.code === 'server_vad_initialization_failed') action.message = 'Server VAD could not load Silero. Configure the server artifact or restart this UI with --stt.';
|
||||
else if (vad && action.code === 'unsupported') action.message += ` ${hint}`;
|
||||
}
|
||||
return action;
|
||||
},
|
||||
connectionHint: vad ? hint : 'Check the Qwen /v1/realtime STT endpoint and model name.',
|
||||
};
|
||||
};
|
||||
})(globalThis);
|
||||
Reference in new issue
Block a user