[Model] Integrate AURA into Full-Duplex Runtime (#7633)

This commit is contained in:
NumberWan authored and GitHub committed 2026-09-21 09:26:54 -07:00
1 parent d397a2f547
commit 62b15142bd
77 files changed
+7695 -457

No files matched your search

+153 -4
View File
@@ -5,7 +5,7 @@ const path = require('node:path');
const vm = require('node:vm');
const root = path.resolve(__dirname, '../../examples/online_serving/realtime_web/app/static');
const context = vm.createContext({ URL });
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
for (const name of ['common', 'minicpm_native', 'qwen3_turn', 'aura_ptt']) {
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), context);
}
const profiles = context.OmniRealtimeProfiles;
@@ -83,6 +83,7 @@ function shell(profileName, adapter = 'stt', options = {}) {
class Element {
constructor() { this.style = {}; this.listeners = {}; this.children = []; this.value = ''; this.textContent = ''; this.classList = { add() {}, remove() {}, toggle() {} }; }
addEventListener(name, callback) { this.listeners[name] = callback; }
closest() { return null; }
append(...children) { this.children.push(...children); }
appendChild(child) { this.children.push(child); }
replaceChildren() { this.children = []; }
@@ -149,15 +150,16 @@ function shell(profileName, adapter = 'stt', options = {}) {
atob: (value) => Buffer.from(value, 'base64').toString('binary'),
});
ctx.window = ctx;
for (const name of ['common', 'minicpm_native', 'qwen3_turn']) {
for (const name of ['common', 'minicpm_native', 'qwen3_turn', 'aura_ptt']) {
vm.runInContext(fs.readFileSync(path.join(root, `profiles/${name}.js`), 'utf8'), ctx);
}
// Expose closure controls only in the test VM; production has no test API.
const source = fs.readFileSync(path.join(root, 'app.js'), 'utf8').replace(/\}\)\(\);\s*$/, `
globalThis.testUI = { startSession, stopSession, handleEvent, playbackDrained,
microphoneUploadEnabled, flushCapture, awaitQueue: () => audioChain,
microphoneUploadEnabled, flushCapture, setPttHeld, awaitQueue: () => audioChain,
capture() { pendingCapture.push(new Int16Array([100, 200])); },
state() { return { running, connectionReady, assistantActive }; }
setPendingFrame(frame) { cameraPendingFrame = frame; if (profile.stickyCamera) cameraLastFrame = frame; },
state() { return { running, connectionReady, assistantActive, pttHeld }; }
};
})();`);
vm.runInContext(source, ctx);
@@ -191,6 +193,7 @@ test('shared shell STT sends final commit and starts a fresh second turn only af
assert.equal(app.ui.microphoneUploadEnabled(), true);
assert.equal(app.elements.get('cameraButton').hidden, true);
assert.equal(app.sockets.flatMap(s => s.sent).some(e => e.type === 'playback.ack'), false);
assert.equal(app.elements.get('pttButton').hidden, true);
await app.ui.stopSession({ terminal: false });
});
@@ -247,6 +250,8 @@ test('an interrupted response disarms the turn watchdog it left armed', async ()
test('MiniCPM retains microphone upload while speaking and acknowledges speaker drain', async () => {
const app = shell('minicpm-native');
await app.ui.startSession();
assert.equal(app.elements.get('pttButton').hidden, true);
assert.equal(app.elements.get('sendTurnButton').hidden, true);
await app.ui.handleEvent({ type: 'response.speak', response_id: 'native-r1' });
assert.equal(app.ui.microphoneUploadEnabled(), true);
await app.ui.handleEvent({ type: 'response.output_audio.delta', delta: 'AAAAAA==', response_id: 'native-r1' });
@@ -428,6 +433,150 @@ test('the camera retires its oldest image instead of exhausting the session budg
assert.equal(plain(p.imageMessages('JPEG')).length, 1);
});
test('AURA PTT profile sets duplex, is_speech append, commit on release, and shell flags', () => {
const aura = profiles['aura-ptt']();
const url = new URL(aura.url({ ...config, model: 'aurateam/AURA' }, 'http://localhost/'));
assert.equal(url.searchParams.get('duplex'), '1');
assert.equal(url.searchParams.get('model'), 'aurateam/AURA');
assert.equal(aura.pushToTalk, true);
assert.equal(aura.clientCommit, false);
assert.equal(aura.camera, true);
assert.equal(aura.stickyCamera, true);
assert.equal(aura.cameraIntervalMs, 500);
assert.equal(aura.visionFollowWhileSpeaking, true);
assert.equal(aura.halfDuplex, false);
assert.equal(aura.playbackAck, true);
assert.equal(aura.append('PCM', 'JPEG', { isSpeech: true }).is_speech, true);
assert.equal(aura.append('PCM', 'JPEG', { isSpeech: false }).is_speech, false);
assert.equal(aura.append('PCM', 'JPEG', { isSpeech: false }).video_frames[0], 'JPEG');
assert.deepEqual(plain(aura.commitMessages()), [
{ type: 'input_audio_buffer.commit', create_response: true },
]);
assert.equal(aura.ack('r1', 50).type, 'playback.ack');
const [update] = aura.initialMessages({}, 'Be brief');
assert.equal(update.session.instructions, 'Be brief');
assert.equal(update.session.extra_body.auto_response, true);
});
test('AURA does not open a second bubble for the same sentence after response.done', async () => {
const app = shell('aura-ptt');
await app.ui.startSession();
const sentence = '你好,我看到一个戴眼镜、穿黑T恤的男生正对着镜头说话呢。';
await app.ui.handleEvent({
type: 'response.output_audio_transcript.delta', response_id: 'r1', delta: sentence,
});
await app.ui.handleEvent({ type: 'response.done', response_id: 'r1' });
await app.ui.handleEvent({
type: 'response.output_audio_transcript.done', response_id: 'r2', transcript: sentence,
});
const turns = [...app.elements.get('conversation').children].filter((row) => row.children.length);
assert.equal(turns.length, 1);
assert.equal(turns[0].children[1].textContent, sentence);
await app.ui.stopSession({ terminal: false });
});
function visionAppends(app) {
return app.sockets[0].sent.filter(e => e.type === 'input_audio_buffer.append' && e.is_speech === false);
}
test('AURA shell holds speech until PTT and opens one vision turn per two frames', async () => {
const app = shell('aura-ptt');
await app.ui.startSession();
assert.equal(app.elements.get('pttButton').hidden, false);
assert.equal(app.elements.get('sendTurnButton').hidden, true);
assert.equal(app.elements.get('cameraButton').hidden, false);
assert.equal(app.ui.microphoneUploadEnabled(), false);
app.ui.capture();
app.ui.flushCapture();
assert.equal(app.sockets[0].sent.some(e => e.type === 'input_audio_buffer.append'), false);
app.ui.setPendingFrame('IDLE1');
app.ui.flushCapture();
assert.equal(visionAppends(app).length, 0, 'one frame must not open a turn');
app.ui.setPendingFrame('IDLE2');
app.ui.flushCapture();
const idle = visionAppends(app).at(-1);
assert.equal(idle.is_speech, false);
assert.deepEqual(idle.video_frames, ['IDLE1', 'IDLE2'], 'two frames gate the turn and both go out as one clip');
assert.equal(app.sockets[0].sent.at(-1).type, 'input_audio_buffer.commit');
assert.equal(app.ui.state().assistantActive, false);
app.ui.setPttHeld(true);
assert.equal(app.ui.microphoneUploadEnabled(), true);
app.ui.setPendingFrame('FRAME1');
app.ui.capture();
app.ui.flushCapture();
const speech = app.sockets[0].sent.find(e => e.type === 'input_audio_buffer.append' && e.is_speech === true);
assert.equal(speech.video_frames[0], 'FRAME1');
assert.equal(visionAppends(app).length, 1, 'speech must not wait for a second frame');
app.ui.setPttHeld(false);
assert.equal(app.sockets[0].sent.at(-1).type, 'input_audio_buffer.commit');
assert.equal(app.ui.state().pttHeld, false);
await app.ui.handleEvent({ type: 'response.created', response: { id: 'aura-r1' } });
app.ui.setPendingFrame('LOCKED1');
app.ui.flushCapture();
app.ui.setPendingFrame('LOCKED2');
app.ui.flushCapture();
assert.equal(visionAppends(app).length, 1, 'open text turn blocks the next vision commit');
await app.ui.handleEvent({
type: 'response.output_audio_transcript.done', response_id: 'aura-r1', transcript: '看到了。',
});
assert.equal(app.ui.state().assistantActive, true, 'text end must not wait for playback');
app.ui.flushCapture();
assert.deepEqual(visionAppends(app).at(-1).video_frames, ['LOCKED1', 'LOCKED2']);
assert.equal(app.sockets[0].sent.at(-1).type, 'input_audio_buffer.commit');
await app.ui.handleEvent({ type: 'response.created', response: { id: 'aura-r2' } });
app.ui.setPendingFrame('SILENT1');
app.ui.flushCapture();
app.ui.setPendingFrame('SILENT2');
app.ui.flushCapture();
assert.equal(visionAppends(app).length, 2);
await app.ui.handleEvent({ type: 'response.listen', response_id: 'aura-r2' });
app.ui.flushCapture();
assert.deepEqual(visionAppends(app).at(-1).video_frames, ['SILENT1', 'SILENT2']);
await app.ui.stopSession({ terminal: false });
});
test('AURA hold-to-talk stops local playback and does not cancel the response', async () => {
const app = shell('aura-ptt', 'stt', { realPlayback: true });
await app.ui.startSession();
receive(app, { type: 'response.created', response: { id: 'old' } });
receive(app, audioChunk('old'));
await app.ui.awaitQueue();
const player = app.nodes.find(node => node.name === 'fullduplex-pcm-playback').player;
for (let index = 0; index < 75; index++) renderPlayback(player);
assert.ok(renderPlayback(player).some(sample => sample !== 0));
const sentBefore = app.sockets[0].sent.length;
app.ui.setPttHeld(true);
assertSilent(player);
const extra = app.sockets[0].sent.slice(sentBefore);
assert.ok(extra.every(event => event.type === 'playback.ack'), 'hold must not abort the server response');
receive(app, audioChunk('old'));
await app.ui.awaitQueue();
assertSilent(player);
await app.ui.stopSession({ terminal: false });
});
test('AURA sticky camera reuses the last frame on speech without a fresh pending', async () => {
const app = shell('aura-ptt');
await app.ui.startSession();
app.ui.setPendingFrame('STICKY');
app.ui.setPttHeld(true);
app.ui.capture();
app.ui.flushCapture();
assert.equal(app.sockets[0].sent.at(-1).video_frames[0], 'STICKY');
// Pending cleared; sticky last frame still available for the next speech flush.
app.ui.capture();
app.ui.flushCapture();
assert.equal(app.sockets[0].sent.at(-1).video_frames[0], 'STICKY');
await app.ui.stopSession({ terminal: false });
});
// Exercise the real WebSocket dispatch queue together with the actual playback
// processor: sending a clear message alone does not prove the speaker is silent.
function receive(app, event) {