Nick CaoandCodex
65699fe94c
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Codex <noreply@openai.com>
794 lines
38 KiB
JavaScript
794 lines
38 KiB
JavaScript
const assert = require('node:assert/strict');
|
|
const { test } = require('node:test');
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const vm = require('node:vm');
|
|
const root = path.resolve(__dirname, '../../examples/online_serving/realtime_web/app/static');
|
|
const context = vm.createContext({ URL });
|
|
for (const name of ['common', 'minicpm_native', 'qwen3_turn', 'aura_ptt']) {
|
|
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), context);
|
|
}
|
|
const profiles = context.OmniRealtimeProfiles;
|
|
const plain = (value) => JSON.parse(JSON.stringify(value));
|
|
const config = { model: 'Qwen/Qwen3-Omni-30B-A3B-Instruct', realtimePath: 'v1/realtime' };
|
|
|
|
test('MiniCPM keeps native flags, reference voice, camera, acknowledgements and no commits', () => {
|
|
const native = profiles['minicpm-native']();
|
|
const url = new URL(native.url({ ...config, model: 'MiniCPM' }, 'https://localhost/ui/'));
|
|
assert.equal(url.protocol, 'wss:');
|
|
assert.equal(url.searchParams.get('duplex'), '1');
|
|
assert.equal(url.searchParams.has('native_duplex'), false);
|
|
const [update] = native.initialMessages({ refAudio: 'data:audio/wav;base64,AA==' }, 'Prompt');
|
|
assert.equal(update.session.ref_audio, 'data:audio/wav;base64,AA==');
|
|
assert.equal(update.session.extra_body.native_duplex, undefined);
|
|
assert.equal(update.session.instructions, 'Prompt');
|
|
assert.deepEqual(plain(native.commitMessages()), []);
|
|
assert.equal(native.append('PCM', 'JPEG').video_frames[0], 'JPEG');
|
|
assert.equal(native.ack('r1', 200).type, 'playback.ack');
|
|
assert.equal(native.halfDuplex, false);
|
|
});
|
|
|
|
test('manual turns use standard Realtime events and keep one session', () => {
|
|
const manual = profiles['qwen3-turn'](config);
|
|
const [update] = plain(manual.initialMessages(config, 'help'));
|
|
assert.equal(update.type, 'session.update');
|
|
assert.deepEqual(update.session, {
|
|
type: 'realtime',
|
|
model: config.model,
|
|
audio: { input: {
|
|
format: { type: 'audio/pcm', rate: 24000 },
|
|
turn_detection: null,
|
|
} },
|
|
instructions: 'help',
|
|
});
|
|
assert.equal(manual.readyEvent, 'session.updated');
|
|
assert.equal(manual.inputSampleRate, 24000);
|
|
assert.deepEqual(plain(manual.append('PCM')), { type: 'input_audio_buffer.append', audio: 'PCM' });
|
|
assert.deepEqual(plain(manual.commitMessages()), [
|
|
{ type: 'input_audio_buffer.commit' },
|
|
{ type: 'response.create' },
|
|
]);
|
|
assert.equal(manual.reconnectEachTurn, undefined);
|
|
assert.equal(manual.mapEvent({ type: 'response.output_audio.delta', delta: 'PCM' }).event.delta, 'PCM');
|
|
assert.equal(manual.mapEvent({ type: 'response.output_audio.done' }).kind, 'drain');
|
|
assert.equal(manual.mapEvent({ type: 'response.done' }).kind, 'done');
|
|
assert.equal(manual.mapEvent({ type: 'response.output_audio_transcript.delta', delta: 'hello' }).role, 'assistant');
|
|
assert.equal(manual.mapEvent({ type: 'input_audio_buffer.cleared' }).kind, 'ignore');
|
|
assert.equal(manual.mapEvent({ type: 'transcription.delta', delta: 'hello' }).kind, 'ignore');
|
|
});
|
|
|
|
test('Qwen enables camera and playback ACK only for the duplex VAD profile', () => {
|
|
for (const turnMode of ['manual', 'vad']) {
|
|
const p = profiles['qwen3-turn']({ ...config, turnMode });
|
|
const url = new URL(p.url({ ...config, realtimePath: 'wss://backend/v1/realtime?native_duplex=1&minicpmo45_native_duplex=1' }, 'http://localhost/'));
|
|
assert.equal(url.protocol, 'wss:');
|
|
assert.equal(url.searchParams.get('duplex'), turnMode === 'vad' ? '1' : '0');
|
|
assert.equal(url.searchParams.has('native_duplex'), false);
|
|
assert.equal(url.searchParams.has('minicpmo45_native_duplex'), false);
|
|
assert.equal(p.append('PCM', 'JPEG').video_frames, undefined);
|
|
assert.equal(plain(p.imageMessages('JPEG')).length, turnMode === 'vad' ? 1 : 0);
|
|
assert.equal(p.ack('r', 100)?.type || null, turnMode === 'vad' ? 'playback.ack' : null);
|
|
assert.equal(p.camera, turnMode === 'vad');
|
|
assert.equal(p.mapEvent({ type: 'response.listen' }).kind, 'ignore');
|
|
}
|
|
});
|
|
|
|
test('VAD uses nested format and interruptible endpoint detection', () => {
|
|
const vad = profiles['qwen3-turn']({ ...config, turnMode: 'vad' });
|
|
const [update] = vad.initialMessages(config, 'help');
|
|
assert.equal(update.session.type, undefined);
|
|
assert.equal(update.session.overlap_policy, 'barge_in_on_speech');
|
|
assert.equal(update.session.instructions, 'help');
|
|
assert.deepEqual(plain(update.session.audio.input), {
|
|
format: { type: 'audio/pcm', rate: 24000 },
|
|
turn_detection: { type: 'server_vad', threshold: 0.5, prefix_padding_ms: 300,
|
|
silence_duration_ms: 500, create_response: true, interrupt_response: true },
|
|
});
|
|
assert.equal(update.session.extra_body, undefined);
|
|
assert.deepEqual(plain(vad.commitMessages()), []);
|
|
assert.equal(vad.mapEvent({ type: 'response.audio.done' }).kind, 'drain');
|
|
assert.equal(vad.mapEvent({ type: 'response.done' }).kind, 'done');
|
|
assert.equal(vad.mapEvent({ type: 'response.output_text.delta', delta: 'hello' }).text, 'hello');
|
|
assert.equal(vad.mapEvent({ type: 'error', error: { code: 'input_backpressure' } }).kind, 'backpressure');
|
|
assert.equal(vad.mapEvent({ type: 'input_audio_buffer.cleared' }).kind, 'backpressure');
|
|
assert.match(vad.mapEvent({ type: 'error', code: 'unsupported', error: 'Unavailable' }).message, /session_mode: duplex/);
|
|
assert.match(vad.mapEvent({ type: 'error', code: 'server_vad_initialization_failed' }).message, /Silero/);
|
|
});
|
|
|
|
function shell(profileName, turnMode = 'manual', options = {}) {
|
|
const elements = new Map();
|
|
class Element {
|
|
constructor() { this.style = {}; this.listeners = {}; this.children = []; this.value = ''; this.textContent = ''; this.classList = { add() {}, remove() {}, toggle() {} }; }
|
|
addEventListener(name, callback) { this.listeners[name] = callback; }
|
|
closest() { return null; }
|
|
append(...children) { this.children.push(...children); }
|
|
appendChild(child) { this.children.push(child); }
|
|
replaceChildren() { this.children = []; }
|
|
remove() {}
|
|
}
|
|
const document = {
|
|
getElementById(id) { if (!elements.has(id)) elements.set(id, new Element()); return elements.get(id); },
|
|
createElement() { return new Element(); },
|
|
};
|
|
const timers = new Map();
|
|
let timerId = 0;
|
|
const sockets = [];
|
|
const nodes = [];
|
|
class Socket {
|
|
static OPEN = 1;
|
|
constructor() {
|
|
this.readyState = 1; this.sent = []; sockets.push(this);
|
|
queueMicrotask(() => {
|
|
this.onopen?.();
|
|
if (!options.silent) this.onmessage?.({ data: JSON.stringify(options.error || { type: 'session.updated' }) });
|
|
});
|
|
}
|
|
send(value) { this.sent.push(JSON.parse(value)); }
|
|
close() { this.readyState = 3; this.onclose?.({ code: 1000 }); }
|
|
}
|
|
class AudioContext {
|
|
constructor(options) { this.sampleRate = options.sampleRate; this.audioWorklet = { addModule: async () => {} }; this.destination = {}; }
|
|
createMediaStreamSource() { return { connect() {} }; }
|
|
createGain() { return { gain: {}, connect() {} }; }
|
|
decodeAudioData(bytes) { return options.decodeAudioData(bytes); }
|
|
async resume() {}
|
|
async close() {}
|
|
}
|
|
class AudioWorkletNode {
|
|
constructor(_context, name) {
|
|
this.name = name; this.sent = []; nodes.push(this);
|
|
this.port = { postMessage: (message) => { this.sent.push(message); this.player?.handleMessage(message); } };
|
|
if (options.realPlayback && name === 'fullduplex-pcm-playback') {
|
|
let Playback;
|
|
const node = this;
|
|
const worker = vm.createContext({
|
|
sampleRate: 24000,
|
|
AudioWorkletProcessor: class {
|
|
constructor() { this.port = { postMessage: message => queueMicrotask(() => node.port.onmessage?.({ data: message })) }; }
|
|
},
|
|
registerProcessor: (_name, cls) => { Playback = cls; },
|
|
});
|
|
vm.runInContext(fs.readFileSync(path.join(root, 'playback_worklet.js'), 'utf8'), worker);
|
|
this.player = new Playback();
|
|
}
|
|
}
|
|
connect(target) { return target; }
|
|
}
|
|
const ctx = vm.createContext({
|
|
URL, document, AudioContext, AudioWorkletNode, WebSocket: Socket,
|
|
navigator: { mediaDevices: { getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }) } },
|
|
location: { href: 'https://localhost/' },
|
|
OMNI_REALTIME_CONFIG: { ...config, profile: profileName, turnMode },
|
|
addEventListener() {},
|
|
setTimeout(fn, ms) { const id = ++timerId; timers.set(id, { fn, ms }); return id; },
|
|
clearTimeout(id) { timers.delete(id); },
|
|
setInterval() { return ++timerId; }, clearInterval() {},
|
|
btoa: (value) => Buffer.from(value, 'binary').toString('base64'),
|
|
atob: (value) => Buffer.from(value, 'base64').toString('binary'),
|
|
});
|
|
ctx.window = ctx;
|
|
for (const name of ['common', 'minicpm_native', 'qwen3_turn', 'aura_ptt']) {
|
|
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), ctx);
|
|
}
|
|
// Expose closure controls only in the test VM; production has no test API.
|
|
const source = fs.readFileSync(path.join(root, 'app.js'), 'utf8').replace(/\}\)\(\);\s*$/, `
|
|
globalThis.testUI = { startSession, stopSession, handleEvent, playbackDrained,
|
|
microphoneUploadEnabled, flushCapture, setPttHeld, awaitQueue: () => audioChain,
|
|
capture() { pendingCapture.push(new Int16Array([100, 200])); },
|
|
setPendingFrame(frame) { cameraPendingFrame = frame; if (profile.stickyCamera) cameraLastFrame = frame; },
|
|
state() { return { running, connectionReady, assistantActive, pttHeld }; }
|
|
};
|
|
})();`);
|
|
vm.runInContext(source, ctx);
|
|
return { ui: ctx.testUI, elements, sockets, nodes,
|
|
timeout() { for (const timer of timers.values()) if (timer.ms === 15000) timer.fn(); },
|
|
async echo() {
|
|
for (const [id, timer] of timers) if (timer.ms === 300) { timers.delete(id); await timer.fn(); }
|
|
},
|
|
async turnWatchdog() {
|
|
for (const [id, timer] of timers) if (timer.ms === 120000) { timers.delete(id); await timer.fn(); }
|
|
},
|
|
};
|
|
}
|
|
|
|
test('shared shell sends standard manual turns and keeps the Realtime session open', async () => {
|
|
const app = shell('qwen3-turn');
|
|
await app.ui.startSession();
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
assert.deepEqual(app.sockets[0].sent[0], {
|
|
type: 'session.update',
|
|
session: {
|
|
type: 'realtime',
|
|
model: config.model,
|
|
audio: { input: {
|
|
format: { type: 'audio/pcm', rate: 24000 },
|
|
turn_detection: null,
|
|
} },
|
|
instructions: 'You are a helpful assistant. Answer clearly and concisely.',
|
|
},
|
|
});
|
|
app.ui.capture(); app.ui.flushCapture();
|
|
const append = app.sockets[0].sent.at(-1);
|
|
assert.equal(append.type, 'input_audio_buffer.append');
|
|
assert.equal(typeof append.audio, 'string');
|
|
assert.equal(append.format, undefined);
|
|
assert.equal(append.sample_rate_hz, undefined);
|
|
app.elements.get('sendTurnButton').listeners.click();
|
|
assert.deepEqual(app.sockets[0].sent.slice(-2), [
|
|
{ type: 'input_audio_buffer.commit' },
|
|
{ type: 'response.create' },
|
|
]);
|
|
assert.equal(app.ui.microphoneUploadEnabled(), false);
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'turn-1' } });
|
|
await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'turn-1' });
|
|
await app.ui.handleEvent({ type: 'response.output_audio.done' });
|
|
await app.ui.handleEvent({ type: 'response.done', response: { id: 'turn-1', status: 'completed' } });
|
|
app.ui.playbackDrained({ responseId: 'turn-1', playedMs: 1 });
|
|
await app.echo();
|
|
assert.equal(app.sockets.length, 1, 'manual turns preserve the same Realtime session');
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
assert.equal(app.elements.get('cameraButton').hidden, true);
|
|
assert.equal(app.sockets.flatMap(s => s.sent).some(e => e.type === 'playback.ack'), false);
|
|
assert.equal(app.elements.get('pttButton').hidden, true);
|
|
|
|
app.ui.capture(); app.ui.flushCapture();
|
|
app.elements.get('sendTurnButton').listeners.click();
|
|
assert.deepEqual(app.sockets[0].sent.slice(-2), [
|
|
{ type: 'input_audio_buffer.commit' },
|
|
{ type: 'response.create' },
|
|
]);
|
|
assert.equal(app.sockets.length, 1);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('VAD keeps uploading while generating and waits for response.done after audio drain', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
|
|
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
|
|
await app.ui.handleEvent({ type: 'response.audio.done', response_id: 'r1' });
|
|
app.ui.playbackDrained({ responseId: 'r1', playedMs: 1 });
|
|
await app.echo();
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true, 'continuous capture remains enabled');
|
|
assert.equal(app.ui.state().assistantActive, true, 'audio drain must wait for response.done');
|
|
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
|
|
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1' } });
|
|
await app.echo();
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
assert.equal(app.sockets.length, 1, 'VAD keeps its session');
|
|
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('a rejected operation is reported without hanging up the call', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
await app.ui.handleEvent({ type: 'error', code: 'invalid_image', error: 'input_image requires a JPEG or PNG base64 data URL' });
|
|
assert.equal(app.ui.state().running, true, 'one refused image must not end the session');
|
|
assert.equal(app.ui.state().connectionReady, true);
|
|
assert.match(app.elements.get('runtimeDetail').textContent, /JPEG or PNG/);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('a session-level error still tears the call down', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
await app.ui.handleEvent({ type: 'error', code: 'unknown_session', error: 'Unknown or closed duplex session' });
|
|
assert.equal(app.ui.state().running, false);
|
|
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
|
});
|
|
|
|
test('an interrupted response disarms the turn watchdog it left armed', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
|
|
await app.ui.handleEvent({ type: 'error', code: 'input_backpressure' });
|
|
await app.ui.handleEvent({ type: 'response.done', response: { id: 'r1', status: 'cancelled' } });
|
|
await app.turnWatchdog();
|
|
assert.equal(app.ui.state().running, true, 'barge-in must not time the session out two minutes later');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('MiniCPM retains microphone upload while speaking and acknowledges speaker drain', async () => {
|
|
const app = shell('minicpm-native');
|
|
await app.ui.startSession();
|
|
assert.equal(app.elements.get('pttButton').hidden, true);
|
|
assert.equal(app.elements.get('sendTurnButton').hidden, true);
|
|
await app.ui.handleEvent({ type: 'response.speak', response_id: 'native-r1' });
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'native-r1' });
|
|
app.ui.playbackDrained({ responseId: 'native-r1', playedMs: 100 });
|
|
assert.equal(app.sockets[0].sent.at(-1).type, 'playback.ack');
|
|
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.commit'), false);
|
|
await app.ui.stopSession({ terminal: false });
|
|
await app.echo();
|
|
assert.equal(app.ui.state().running, false);
|
|
});
|
|
|
|
test('VAD handshake errors remain visible after resource cleanup', async () => {
|
|
const app = shell('qwen3-turn', 'vad', { error: { type: 'error', code: 'unsupported', error: 'Realtime API is not available' } });
|
|
await app.ui.startSession();
|
|
assert.equal(app.ui.state().running, false);
|
|
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
|
assert.match(app.elements.get('runtimeDetail').textContent, /session_mode: duplex/);
|
|
assert.equal(app.sockets[0].readyState, 3);
|
|
});
|
|
|
|
test('missing session acknowledgement fails instead of leaving Start disabled forever', async () => {
|
|
const app = shell('qwen3-turn', 'vad', { silent: true });
|
|
const starting = app.ui.startSession();
|
|
for (let i = 0; i < 20; i++) await Promise.resolve();
|
|
app.timeout();
|
|
await starting;
|
|
assert.equal(app.ui.state().running, false);
|
|
assert.equal(app.elements.get('callButton').disabled, false);
|
|
assert.match(app.elements.get('runtimeDetail').textContent, /handshake timed out/);
|
|
});
|
|
|
|
test('Qwen does not duplicate text and audio-transcript channels', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: '' });
|
|
await app.ui.handleEvent({ type: 'response.output_text.delta', delta: 'Hello' });
|
|
await app.ui.handleEvent({ type: 'response.audio_transcript.delta', delta: 'Hello' });
|
|
await app.ui.handleEvent({ type: 'response.output_text.done', text: 'Hello' });
|
|
await app.ui.handleEvent({ type: 'response.done' });
|
|
const turns = app.elements.get('conversation').children;
|
|
assert.equal(turns.length, 1);
|
|
assert.equal(turns[0].children[1].textContent, 'Hello');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
const flushTasks = async () => { for (let i = 0; i < 40; i++) await Promise.resolve(); };
|
|
|
|
test('closing socket accepts only session.closed and completes without the fallback timer', async () => {
|
|
const app = shell('minicpm-native');
|
|
await app.ui.startSession();
|
|
const old = app.sockets[0];
|
|
let stopped = false;
|
|
const stopping = app.ui.stopSession().then(() => { stopped = true; });
|
|
assert.equal(old.sent.at(-1).type, 'session.close');
|
|
old.onmessage({ data: JSON.stringify({ type: 'session.updated' }) });
|
|
await flushTasks();
|
|
assert.equal(stopped, false);
|
|
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
|
|
await flushTasks();
|
|
assert.equal(stopped, true, 'must finish without firing the timeout');
|
|
await stopping;
|
|
await app.ui.startSession();
|
|
old.onmessage({ data: JSON.stringify({ type: 'session.closed' }) });
|
|
old.onmessage({ data: '{broken' });
|
|
await flushTasks();
|
|
assert.equal(app.ui.state().running, true, 'stale socket cannot affect a new session');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('malformed JSON before readiness rejects the handshake and cleans up', async () => {
|
|
const app = shell('qwen3-turn', 'vad', { silent: true });
|
|
const starting = app.ui.startSession();
|
|
await flushTasks();
|
|
app.sockets[0].onmessage({ data: '{broken' });
|
|
await starting;
|
|
assert.equal(app.ui.state().running, false);
|
|
assert.equal(app.sockets[0].readyState, 3);
|
|
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
|
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
|
|
});
|
|
|
|
test('malformed JSON after readiness fails the session and releases resources', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
app.sockets[0].onmessage({ data: '{broken' });
|
|
await flushTasks();
|
|
assert.equal(app.ui.state().running, false);
|
|
assert.equal(app.ui.microphoneUploadEnabled(), false);
|
|
assert.equal(app.sockets[0].readyState, 3);
|
|
assert.equal(app.elements.get('connectionState').textContent, 'Error');
|
|
assert.match(app.elements.get('runtimeDetail').textContent, /Invalid server event/);
|
|
});
|
|
|
|
|
|
test('Qwen VAD interruption clears playback while continuing microphone upload', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'r1' } });
|
|
await app.ui.handleEvent({ type: 'response.audio.delta', delta: 'AAAAAA==', response_id: 'r1' });
|
|
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'r1' });
|
|
assert.equal(app.ui.state().assistantActive, false);
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
assert.ok(app.nodes.some(node => node.sent.some(message => message.type === 'clear')));
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
|
|
test('Qwen runtime errors do not suggest changing a working deployment', () => {
|
|
const p = profiles['qwen3-turn']({ turnMode: 'vad' });
|
|
const message = 'playback.ack arrived after a later user input was committed.';
|
|
assert.equal(p.mapEvent({ type: 'error', code: 'playback_ack_too_late', error: message }).message, message);
|
|
});
|
|
|
|
test('interrupted playback reports its cursor before clear resets it', () => {
|
|
let Playback;
|
|
const messages = [];
|
|
const ctx = vm.createContext({
|
|
sampleRate: 24000,
|
|
AudioWorkletProcessor: class { constructor() { this.port = { postMessage: m => messages.push(m) }; } },
|
|
registerProcessor: (_name, cls) => { Playback = cls; },
|
|
});
|
|
vm.runInContext(fs.readFileSync(path.join(root, 'playback_worklet.js'), 'utf8'), ctx);
|
|
const player = new Playback();
|
|
player.handleMessage({ type: 'audio', responseId: 'old', pcm: new Int16Array(24000), initialBufferMs: 0 });
|
|
player.process([], [[new Float32Array(2400)]]);
|
|
player.handleMessage({ type: 'clear' });
|
|
const report = messages.find(m => m.type === 'playback-stopped');
|
|
assert.equal(report.responseId, 'old');
|
|
assert.equal(report.playedMs, 100);
|
|
assert.equal(player.playedFrames, 0);
|
|
assert.equal(player.bufferedFrames(), 0);
|
|
player.handleMessage({ type: 'clear' });
|
|
assert.equal(messages.filter(m => m.type === 'playback-stopped').length, 1);
|
|
});
|
|
|
|
test('late interruption cursor acknowledges old response without finishing the new response', async () => {
|
|
const app = shell('qwen3-turn', 'vad');
|
|
await app.ui.startSession();
|
|
const player = app.nodes.find(n => n.name === 'fullduplex-pcm-playback');
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'old' } });
|
|
await app.ui.handleEvent({ type: 'output_audio_buffer.cleared', response_id: 'old' });
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'new' } });
|
|
player.port.onmessage({ data: { type: 'playback-stopped', responseId: 'old', playedMs: 100 } });
|
|
const ack = app.sockets[0].sent.find(m => m.type === 'playback.ack');
|
|
assert.equal(ack.response_id, 'old');
|
|
assert.equal(ack.played_ms, 100);
|
|
assert.equal(app.ui.state().assistantActive, true);
|
|
});
|
|
|
|
|
|
test('Qwen camera frames ride the OpenAI image interface, not the audio append', () => {
|
|
const p = profiles['qwen3-turn']({ ...config, turnMode: 'vad' });
|
|
assert.deepEqual(plain(p.append('PCM', 'JPEG')), { type: 'input_audio_buffer.append', audio: 'PCM' });
|
|
|
|
const [created] = plain(p.imageMessages('JPEG'));
|
|
assert.equal(created.type, 'conversation.item.create');
|
|
assert.equal(created.item.role, 'user');
|
|
assert.deepEqual(created.item.content, [
|
|
{ type: 'input_image', image_url: 'data:image/jpeg;base64,JPEG' },
|
|
]);
|
|
});
|
|
|
|
test('the camera retires its oldest image instead of exhausting the session budget', () => {
|
|
const p = profiles['qwen3-turn']({ ...config, turnMode: 'vad' });
|
|
const ids = [];
|
|
for (let i = 0; i < 8; i++) {
|
|
const messages = plain(p.imageMessages('JPEG'));
|
|
assert.equal(messages.length, 1, 'nothing to retire while under the budget');
|
|
ids.push(messages[0].item.id);
|
|
}
|
|
// The engine counts what it already stores, so the delete has to lead.
|
|
const ninth = plain(p.imageMessages('JPEG'));
|
|
assert.deepEqual(ninth[0], { type: 'conversation.item.delete', item_id: ids[0] });
|
|
assert.equal(ninth[1].type, 'conversation.item.create');
|
|
assert.equal(new Set(ids).size, 8, 'item ids must be distinct to be deletable');
|
|
|
|
// A new call reopens the budget; stale ids from the old one must not linger.
|
|
assert.equal(plain(p.initialMessages(config, 'x')).length > 0, true);
|
|
assert.equal(plain(p.imageMessages('JPEG')).length, 1);
|
|
});
|
|
|
|
test('AURA PTT profile sets duplex, is_speech append, commit on release, and shell flags', () => {
|
|
const aura = profiles['aura-ptt']();
|
|
const url = new URL(aura.url({ ...config, model: 'aurateam/AURA' }, 'http://localhost/'));
|
|
assert.equal(url.searchParams.get('duplex'), '1');
|
|
assert.equal(url.searchParams.get('model'), 'aurateam/AURA');
|
|
assert.equal(aura.pushToTalk, true);
|
|
assert.equal(aura.clientCommit, false);
|
|
assert.equal(aura.camera, true);
|
|
assert.equal(aura.stickyCamera, true);
|
|
assert.equal(aura.cameraIntervalMs, 500);
|
|
assert.equal(aura.visionFollowWhileSpeaking, true);
|
|
assert.equal(aura.halfDuplex, false);
|
|
assert.equal(aura.playbackAck, true);
|
|
assert.equal(aura.append('PCM', 'JPEG', { isSpeech: true }).is_speech, true);
|
|
assert.equal(aura.append('PCM', 'JPEG', { isSpeech: false }).is_speech, false);
|
|
assert.equal(aura.append('PCM', 'JPEG', { isSpeech: false }).video_frames[0], 'JPEG');
|
|
assert.deepEqual(plain(aura.commitMessages()), [
|
|
{ type: 'input_audio_buffer.commit', create_response: true },
|
|
]);
|
|
assert.equal(aura.ack('r1', 50).type, 'playback.ack');
|
|
const [update] = aura.initialMessages({}, 'Be brief');
|
|
assert.equal(update.session.instructions, 'Be brief');
|
|
assert.equal(update.session.extra_body.auto_response, true);
|
|
});
|
|
|
|
test('AURA does not open a second bubble for the same sentence after response.done', async () => {
|
|
const app = shell('aura-ptt');
|
|
await app.ui.startSession();
|
|
const sentence = '你好，我看到一个戴眼镜、穿黑T恤的男生正对着镜头说话呢。';
|
|
await app.ui.handleEvent({
|
|
type: 'response.output_audio_transcript.delta', response_id: 'r1', delta: sentence,
|
|
});
|
|
await app.ui.handleEvent({ type: 'response.done', response_id: 'r1' });
|
|
await app.ui.handleEvent({
|
|
type: 'response.output_audio_transcript.done', response_id: 'r2', transcript: sentence,
|
|
});
|
|
const turns = [...app.elements.get('conversation').children].filter((row) => row.children.length);
|
|
assert.equal(turns.length, 1);
|
|
assert.equal(turns[0].children[1].textContent, sentence);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
function visionAppends(app) {
|
|
return app.sockets[0].sent.filter(e => e.type === 'input_audio_buffer.append' && e.is_speech === false);
|
|
}
|
|
|
|
test('AURA shell holds speech until PTT and opens one vision turn per two frames', async () => {
|
|
const app = shell('aura-ptt');
|
|
await app.ui.startSession();
|
|
assert.equal(app.elements.get('pttButton').hidden, false);
|
|
assert.equal(app.elements.get('sendTurnButton').hidden, true);
|
|
assert.equal(app.elements.get('cameraButton').hidden, false);
|
|
assert.equal(app.ui.microphoneUploadEnabled(), false);
|
|
|
|
app.ui.capture();
|
|
app.ui.flushCapture();
|
|
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.append'), false);
|
|
|
|
app.ui.setPendingFrame('IDLE1');
|
|
app.ui.flushCapture();
|
|
assert.equal(visionAppends(app).length, 0, 'one frame must not open a turn');
|
|
app.ui.setPendingFrame('IDLE2');
|
|
app.ui.flushCapture();
|
|
const idle = visionAppends(app).at(-1);
|
|
assert.equal(idle.is_speech, false);
|
|
assert.deepEqual(idle.video_frames, ['IDLE1', 'IDLE2'], 'two frames gate the turn and both go out as one clip');
|
|
assert.equal(app.sockets[0].sent.at(-1).type, 'input_audio_buffer.commit');
|
|
assert.equal(app.ui.state().assistantActive, false);
|
|
|
|
app.ui.setPttHeld(true);
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
app.ui.setPendingFrame('FRAME1');
|
|
app.ui.capture();
|
|
app.ui.flushCapture();
|
|
const speech = app.sockets[0].sent.find(e => e.type === 'input_audio_buffer.append' && e.is_speech === true);
|
|
assert.equal(speech.video_frames[0], 'FRAME1');
|
|
assert.equal(visionAppends(app).length, 1, 'speech must not wait for a second frame');
|
|
|
|
app.ui.setPttHeld(false);
|
|
assert.equal(app.sockets[0].sent.at(-1).type, 'input_audio_buffer.commit');
|
|
assert.equal(app.ui.state().pttHeld, false);
|
|
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'aura-r1' } });
|
|
app.ui.setPendingFrame('LOCKED1');
|
|
app.ui.flushCapture();
|
|
app.ui.setPendingFrame('LOCKED2');
|
|
app.ui.flushCapture();
|
|
assert.equal(visionAppends(app).length, 1, 'open text turn blocks the next vision commit');
|
|
|
|
await app.ui.handleEvent({
|
|
type: 'response.output_audio_transcript.done', response_id: 'aura-r1', transcript: '看到了。',
|
|
});
|
|
assert.equal(app.ui.state().assistantActive, true, 'text end must not wait for playback');
|
|
app.ui.flushCapture();
|
|
assert.deepEqual(visionAppends(app).at(-1).video_frames, ['LOCKED1', 'LOCKED2']);
|
|
assert.equal(app.sockets[0].sent.at(-1).type, 'input_audio_buffer.commit');
|
|
|
|
await app.ui.handleEvent({ type: 'response.created', response: { id: 'aura-r2' } });
|
|
app.ui.setPendingFrame('SILENT1');
|
|
app.ui.flushCapture();
|
|
app.ui.setPendingFrame('SILENT2');
|
|
app.ui.flushCapture();
|
|
assert.equal(visionAppends(app).length, 2);
|
|
await app.ui.handleEvent({ type: 'response.listen', response_id: 'aura-r2' });
|
|
app.ui.flushCapture();
|
|
assert.deepEqual(visionAppends(app).at(-1).video_frames, ['SILENT1', 'SILENT2']);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('AURA hold-to-talk stops local playback and does not cancel the response', async () => {
|
|
const app = shell('aura-ptt', 'manual', { realPlayback: true });
|
|
await app.ui.startSession();
|
|
receive(app, { type: 'response.created', response: { id: 'old' } });
|
|
receive(app, audioChunk('old'));
|
|
await app.ui.awaitQueue();
|
|
const player = app.nodes.find(node => node.name === 'fullduplex-pcm-playback').player;
|
|
for (let index = 0; index < 75; index++) renderPlayback(player);
|
|
assert.ok(renderPlayback(player).some(sample => sample !== 0));
|
|
const sentBefore = app.sockets[0].sent.length;
|
|
app.ui.setPttHeld(true);
|
|
assertSilent(player);
|
|
const extra = app.sockets[0].sent.slice(sentBefore);
|
|
assert.ok(extra.every(event => event.type === 'playback.ack'), 'hold must not abort the server response');
|
|
receive(app, audioChunk('old'));
|
|
await app.ui.awaitQueue();
|
|
assertSilent(player);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('AURA sticky camera reuses the last frame on speech without a fresh pending', async () => {
|
|
const app = shell('aura-ptt');
|
|
await app.ui.startSession();
|
|
app.ui.setPendingFrame('STICKY');
|
|
app.ui.setPttHeld(true);
|
|
app.ui.capture();
|
|
app.ui.flushCapture();
|
|
assert.equal(app.sockets[0].sent.at(-1).video_frames[0], 'STICKY');
|
|
// Pending cleared; sticky last frame still available for the next speech flush.
|
|
app.ui.capture();
|
|
app.ui.flushCapture();
|
|
assert.equal(app.sockets[0].sent.at(-1).video_frames[0], 'STICKY');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
// Exercise the real WebSocket dispatch queue together with the actual playback
|
|
// processor: sending a clear message alone does not prove the speaker is silent.
|
|
function receive(app, event) {
|
|
app.sockets[0].onmessage({ data: JSON.stringify(event) });
|
|
}
|
|
|
|
function audioChunk(responseId, extra = {}) {
|
|
const pcm = new Int16Array(24000).fill(12000);
|
|
return { type: 'response.output_audio.delta', response_id: responseId,
|
|
delta: Buffer.from(pcm.buffer).toString('base64'), format: 'pcm16', sample_rate_hz: 24000, ...extra };
|
|
}
|
|
|
|
function renderPlayback(player) {
|
|
const output = new Float32Array(128);
|
|
player.process([], [[output]]);
|
|
return output;
|
|
}
|
|
|
|
async function playingQwen(options = {}) {
|
|
const app = shell('qwen3-turn', 'vad', { realPlayback: true, ...options });
|
|
await app.ui.startSession();
|
|
receive(app, { type: 'response.created', response: { id: 'old' } });
|
|
receive(app, audioChunk('old'));
|
|
await app.ui.awaitQueue();
|
|
app.player = app.nodes.find(node => node.name === 'fullduplex-pcm-playback').player;
|
|
// Advance through the initial 400 ms playback buffer.
|
|
for (let index = 0; index < 75; index++) renderPlayback(app.player);
|
|
assert.ok(renderPlayback(app.player).some(sample => sample !== 0));
|
|
return app;
|
|
}
|
|
|
|
function assertSilent(player) {
|
|
assert.equal(player.bufferedFrames(), 0, 'interrupted audio must leave the playback queue');
|
|
assert.ok(renderPlayback(player).every(sample => sample === 0), 'speaker must output silence');
|
|
}
|
|
|
|
for (const terminal of [false, true]) {
|
|
test(`Qwen speech interrupts actual playback after generation completed=${terminal}`, async () => {
|
|
const app = await playingQwen();
|
|
if (terminal) {
|
|
receive(app, { type: 'response.output_audio.done', response_id: 'old' });
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'completed' } });
|
|
await app.ui.awaitQueue();
|
|
}
|
|
receive(app, { type: 'input_audio_buffer.speech_started', item_id: 'next', audio_start_ms: 1000 });
|
|
await app.ui.awaitQueue();
|
|
assertSilent(app.player);
|
|
assert.equal(app.ui.microphoneUploadEnabled(), true);
|
|
const ack = app.sockets[0].sent.find(event => event.type === 'playback.ack');
|
|
assert.equal(ack.response_id, 'old');
|
|
assert.equal(ack.played_ms, 5, 'ack only the 128 frames actually played, not all queued audio');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
}
|
|
|
|
test('Qwen cancellation preempts pending decode and does not delay the next response', async () => {
|
|
let releaseDecode;
|
|
const app = await playingQwen({ decodeAudioData: () => new Promise(resolve => { releaseDecode = resolve; }) });
|
|
receive(app, audioChunk('old', { format: 'wav' }));
|
|
await flushTasks();
|
|
assert.equal(typeof releaseDecode, 'function');
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'cancelled' } });
|
|
await flushTasks();
|
|
assertSilent(app.player);
|
|
receive(app, { type: 'response.created', response: { id: 'new' } });
|
|
receive(app, audioChunk('new'));
|
|
await flushTasks();
|
|
assert.equal(app.player.activeResponseId, 'new');
|
|
assert.equal(app.player.bufferedFrames(), 24000);
|
|
releaseDecode({ sampleRate: 24000, getChannelData: () => new Float32Array(24000).fill(0.4) });
|
|
await flushTasks();
|
|
assert.equal(app.player.bufferedFrames(), 24000, 'old decoded audio must not enter the new response');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('Qwen interruption rejects queued and late audio for the cancelled response', async () => {
|
|
const app = await playingQwen();
|
|
receive(app, audioChunk('old'));
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'cancelled' } });
|
|
receive(app, audioChunk('old'));
|
|
await app.ui.awaitQueue();
|
|
assertSilent(app.player);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('late cancellation of an old Qwen response does not clear a newer response', async () => {
|
|
const app = await playingQwen();
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'cancelled' } });
|
|
await app.ui.awaitQueue();
|
|
receive(app, { type: 'response.created', response: { id: 'new' } });
|
|
receive(app, audioChunk('new'));
|
|
await app.ui.awaitQueue();
|
|
receive(app, { type: 'output_audio_buffer.cleared', response_id: 'old' });
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'cancelled' } });
|
|
await app.ui.awaitQueue();
|
|
assert.equal(app.player.activeResponseId, 'new');
|
|
assert.equal(app.player.bufferedFrames(), 24000);
|
|
assert.equal(app.ui.state().assistantActive, true);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('Qwen speech interruption follows accepted turn detection; MiniCPM mapping stays unchanged', () => {
|
|
const qwen = profiles['qwen3-turn']({ turnMode: 'vad' });
|
|
const speech = { type: 'input_audio_buffer.speech_started', item_id: 'next' };
|
|
assert.equal(qwen.mapEvent(speech).kind, 'interrupt');
|
|
qwen.mapEvent({ type: 'session.updated', session: { audio: { input: { turn_detection: {
|
|
type: 'server_vad', interrupt_response: false,
|
|
} } } } });
|
|
assert.equal(qwen.mapEvent(speech).kind, 'ignore');
|
|
qwen.mapEvent({ type: 'session.updated', session: { audio: { input: { turn_detection: null } } } });
|
|
assert.equal(qwen.mapEvent(speech).kind, 'ignore');
|
|
qwen.mapEvent({ type: 'session.updated', session: { audio: { input: { turn_detection: {
|
|
type: 'server_vad', interrupt_response: true,
|
|
} } } } });
|
|
assert.equal(qwen.mapEvent(speech).kind, 'interrupt');
|
|
assert.equal(profiles['qwen3-turn']({ turnMode: 'manual' }).mapEvent(speech).kind, 'ignore');
|
|
assert.equal(profiles['minicpm-native']().mapEvent(speech).kind, 'ignore');
|
|
});
|
|
|
|
test('late old-response controls cannot invalidate a newer pending decode', async () => {
|
|
let releaseDecode;
|
|
const app = await playingQwen({ decodeAudioData: () => new Promise(resolve => { releaseDecode = resolve; }) });
|
|
receive(app, { type: 'input_audio_buffer.speech_started', item_id: 'next', audio_start_ms: 1000 });
|
|
await app.ui.awaitQueue();
|
|
receive(app, { type: 'response.created', response: { id: 'new' } });
|
|
receive(app, audioChunk('new', { format: 'wav' }));
|
|
await flushTasks();
|
|
assert.equal(typeof releaseDecode, 'function');
|
|
receive(app, { type: 'output_audio_buffer.cleared', response_id: 'old' });
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'cancelled' } });
|
|
releaseDecode({ sampleRate: 24000, getChannelData: () => new Float32Array(24000).fill(0.4) });
|
|
await app.ui.awaitQueue();
|
|
assert.equal(app.player.activeResponseId, 'new');
|
|
assert.equal(app.player.bufferedFrames(), 24000);
|
|
assert.equal(app.ui.state().assistantActive, true);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('normal completion still waits for decoding before draining playback', async () => {
|
|
let releaseDecode;
|
|
const app = await playingQwen({ decodeAudioData: () => new Promise(resolve => { releaseDecode = resolve; }) });
|
|
receive(app, audioChunk('old', { format: 'wav' }));
|
|
await flushTasks();
|
|
receive(app, { type: 'response.output_audio.done', response_id: 'old' });
|
|
receive(app, { type: 'response.done', response: { id: 'old', status: 'completed' } });
|
|
await flushTasks();
|
|
assert.equal(app.player.drain, null);
|
|
releaseDecode({ sampleRate: 24000, getChannelData: () => new Float32Array(24000).fill(0.4) });
|
|
await app.ui.awaitQueue();
|
|
assert.equal(app.player.bufferedFrames(), 47872);
|
|
assert.equal(app.player.drain.responseId, 'old');
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|
|
|
|
test('interrupting an old decode preserves a newer response already waiting in the event queue', async () => {
|
|
let releaseDecode;
|
|
const app = await playingQwen({ decodeAudioData: () => new Promise(resolve => { releaseDecode = resolve; }) });
|
|
receive(app, audioChunk('old', { format: 'wav' }));
|
|
await flushTasks();
|
|
receive(app, { type: 'response.created', response: { id: 'new' } });
|
|
receive(app, audioChunk('new'));
|
|
receive(app, { type: 'output_audio_buffer.cleared', response_id: 'old' });
|
|
await flushTasks();
|
|
assert.equal(app.player.activeResponseId, 'new');
|
|
assert.equal(app.player.bufferedFrames(), 24000);
|
|
releaseDecode({ sampleRate: 24000, getChannelData: () => new Float32Array(24000).fill(0.4) });
|
|
await flushTasks();
|
|
assert.equal(app.player.bufferedFrames(), 24000);
|
|
await app.ui.stopSession({ terminal: false });
|
|
});
|