1663 lines
55 KiB
Python
1663 lines
55 KiB
Python
|
import asyncio
|
||
|
|
import base64
|
||
|
|
import importlib.util
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import wave
|
||
|
|
from pathlib import Path
|
||
|
|
from types import SimpleNamespace
|
||
|
|
from urllib.parse import parse_qs, urlsplit
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||
|
|
|
||
|
|
DEMO_PATH = Path(__file__).resolve().parents[1] / "e2e/online_serving/minicpmo_realtime_duplex_scenarios.py"
|
||
|
|
MULTI_DEMO_PATH = (
|
||
|
|
Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py"
|
||
|
|
)
|
||
|
|
PAIR_DEMO_PATH = Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_demo_pair.py"
|
||
|
|
SOFT_INTERRUPT_DEMO_PATH = (
|
||
|
|
Path(__file__).resolve().parents[1] / "e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _load_demo_module():
|
||
|
|
spec = importlib.util.spec_from_file_location("minicpmo_realtime_duplex_demo_test", DEMO_PATH)
|
||
|
|
assert spec is not None and spec.loader is not None
|
||
|
|
module = importlib.util.module_from_spec(spec)
|
||
|
|
sys.modules[spec.name] = module
|
||
|
|
spec.loader.exec_module(module)
|
||
|
|
return module
|
||
|
|
|
||
|
|
|
||
|
|
def _load_multi_demo_module():
|
||
|
|
spec = importlib.util.spec_from_file_location("minicpmo_realtime_duplex_multi_session_test", MULTI_DEMO_PATH)
|
||
|
|
assert spec is not None and spec.loader is not None
|
||
|
|
module = importlib.util.module_from_spec(spec)
|
||
|
|
sys.modules[spec.name] = module
|
||
|
|
spec.loader.exec_module(module)
|
||
|
|
return module
|
||
|
|
|
||
|
|
|
||
|
|
def _load_pair_demo_module():
|
||
|
|
spec = importlib.util.spec_from_file_location("minicpmo_realtime_duplex_demo_pair_test", PAIR_DEMO_PATH)
|
||
|
|
assert spec is not None and spec.loader is not None
|
||
|
|
module = importlib.util.module_from_spec(spec)
|
||
|
|
sys.modules[spec.name] = module
|
||
|
|
spec.loader.exec_module(module)
|
||
|
|
return module
|
||
|
|
|
||
|
|
|
||
|
|
def _load_soft_interrupt_demo_module():
|
||
|
|
spec = importlib.util.spec_from_file_location(
|
||
|
|
"minicpmo_realtime_duplex_soft_interrupt_test",
|
||
|
|
SOFT_INTERRUPT_DEMO_PATH,
|
||
|
|
)
|
||
|
|
assert spec is not None and spec.loader is not None
|
||
|
|
module = importlib.util.module_from_spec(spec)
|
||
|
|
sys.modules[spec.name] = module
|
||
|
|
spec.loader.exec_module(module)
|
||
|
|
return module
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_multi_session_script_is_directly_executable():
|
||
|
|
result = subprocess.run(
|
||
|
|
[sys.executable, str(MULTI_DEMO_PATH), "--help"],
|
||
|
|
cwd=MULTI_DEMO_PATH.parents[3],
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
timeout=20,
|
||
|
|
check=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.returncode == 0, result.stderr
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_pair_script_is_directly_executable():
|
||
|
|
result = subprocess.run(
|
||
|
|
[sys.executable, str(PAIR_DEMO_PATH), "--help"],
|
||
|
|
cwd=PAIR_DEMO_PATH.parents[3],
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
timeout=20,
|
||
|
|
check=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.returncode == 0, result.stderr
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_script_is_directly_executable():
|
||
|
|
result = subprocess.run(
|
||
|
|
[sys.executable, str(SOFT_INTERRUPT_DEMO_PATH), "--help"],
|
||
|
|
cwd=SOFT_INTERRUPT_DEMO_PATH.parents[3],
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
timeout=20,
|
||
|
|
check=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result.returncode == 0, result.stderr
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_pair_requires_distinct_inputs_and_outputs(tmp_path):
|
||
|
|
demo = _load_pair_demo_module()
|
||
|
|
wav_a = tmp_path / "request_a.wav"
|
||
|
|
wav_b = tmp_path / "request_b.wav"
|
||
|
|
wav_a.write_bytes(b"same")
|
||
|
|
wav_b.write_bytes(b"same")
|
||
|
|
|
||
|
|
args = SimpleNamespace(
|
||
|
|
input_wav_a=str(wav_a),
|
||
|
|
input_wav_b=str(wav_b),
|
||
|
|
output_dir_a=str(tmp_path / "session_a"),
|
||
|
|
output_dir_b=str(tmp_path / "session_a"),
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(ValueError, match="output directories must be different"):
|
||
|
|
demo._validate_pair_args(args)
|
||
|
|
|
||
|
|
args.output_dir_b = str(tmp_path / "session_b")
|
||
|
|
with pytest.raises(ValueError, match="input WAV files must have different content"):
|
||
|
|
demo._validate_pair_args(args)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_pair_launches_demo_processes_concurrently(tmp_path, monkeypatch):
|
||
|
|
demo = _load_pair_demo_module()
|
||
|
|
fake_demo = tmp_path / "fake_realtime_duplex_demo.py"
|
||
|
|
start_log = tmp_path / "starts.log"
|
||
|
|
fake_demo.write_text(
|
||
|
|
"\n".join(
|
||
|
|
[
|
||
|
|
"import argparse, json, os, time",
|
||
|
|
"from pathlib import Path",
|
||
|
|
"parser = argparse.ArgumentParser()",
|
||
|
|
"parser.add_argument('--url')",
|
||
|
|
"parser.add_argument('--model')",
|
||
|
|
"parser.add_argument('--input-wav')",
|
||
|
|
"parser.add_argument('--output-dir')",
|
||
|
|
"parser.add_argument('--chunk-ms')",
|
||
|
|
"parser.add_argument('--timeout-s')",
|
||
|
|
"parser.add_argument('--session-id')",
|
||
|
|
"parser.add_argument('--ref-audio')",
|
||
|
|
"parser.add_argument('--require-audio', action='store_true')",
|
||
|
|
"parser.add_argument('--no-realtime-pacing', action='store_true')",
|
||
|
|
"args = parser.parse_args()",
|
||
|
|
"if args.ref_audio != os.environ.get('EXPECTED_REF_AUDIO'):",
|
||
|
|
" raise SystemExit(f'unexpected ref audio: {args.ref_audio}')",
|
||
|
|
"output = Path(args.output_dir)",
|
||
|
|
"output.mkdir(parents=True, exist_ok=True)",
|
||
|
|
"response_id = 'resp-' + output.name",
|
||
|
|
"with open(os.environ['PAIR_START_LOG'], 'a', encoding='utf-8') as f:",
|
||
|
|
" f.write(f'{output.name} {time.time()}\\n')",
|
||
|
|
"time.sleep(0.25)",
|
||
|
|
"events = [",
|
||
|
|
" {'type': 'session.created'},",
|
||
|
|
" {'type': 'response.created', 'response': {'id': response_id}},",
|
||
|
|
" {'type': 'response.audio.delta', 'response_id': response_id, "
|
||
|
|
"'delta': 'AAAA', 'sample_rate_hz': 24000, '_client_received_at_s': 10.1},",
|
||
|
|
" {'type': 'response.audio.delta', 'response_id': response_id, "
|
||
|
|
"'delta': 'AAAA', 'sample_rate_hz': 24000, '_client_received_at_s': 10.6},",
|
||
|
|
" {'type': 'response.done', 'response_id': response_id},",
|
||
|
|
" {'type': 'session.closed'},",
|
||
|
|
"]",
|
||
|
|
"Path(output / 'events.jsonl').write_text(",
|
||
|
|
" ''.join(json.dumps(event) + '\\n' for event in events),",
|
||
|
|
" encoding='utf-8',",
|
||
|
|
")",
|
||
|
|
"result = {'ok': True, 'response_ids': [response_id], 'audio_bytes': 1, 'output_dir': str(output)}",
|
||
|
|
"Path(output / 'result.json').write_text(json.dumps(result), encoding='utf-8')",
|
||
|
|
"print(json.dumps(result))",
|
||
|
|
]
|
||
|
|
),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(demo, "DEMO_PATH", fake_demo)
|
||
|
|
monkeypatch.setenv("PAIR_START_LOG", str(start_log))
|
||
|
|
monkeypatch.setenv("EXPECTED_REF_AUDIO", str(tmp_path / "ref.wav"))
|
||
|
|
wav_a = tmp_path / "request_a.wav"
|
||
|
|
wav_b = tmp_path / "request_b.wav"
|
||
|
|
wav_a.write_bytes(b"request a")
|
||
|
|
wav_b.write_bytes(b"request b")
|
||
|
|
|
||
|
|
result = asyncio.run(
|
||
|
|
demo.run_pair(
|
||
|
|
SimpleNamespace(
|
||
|
|
url="ws://localhost:28889/v1/realtime?duplex=1",
|
||
|
|
model="/data/why/MiniCPM-o-4_5",
|
||
|
|
input_wav_a=str(wav_a),
|
||
|
|
input_wav_b=str(wav_b),
|
||
|
|
output_dir_a=str(tmp_path / "duplex_session_a"),
|
||
|
|
output_dir_b=str(tmp_path / "duplex_session_b"),
|
||
|
|
ref_audio=str(tmp_path / "ref.wav"),
|
||
|
|
summary_output=str(tmp_path / "summary.json"),
|
||
|
|
chunk_ms=200,
|
||
|
|
timeout_s=5.0,
|
||
|
|
no_realtime_pacing=False,
|
||
|
|
require_audio=True,
|
||
|
|
min_audio_deltas_per_session=2,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
starts = [float(line.split()[1]) for line in start_log.read_text(encoding="utf-8").splitlines()]
|
||
|
|
assert len(starts) == 2
|
||
|
|
assert max(starts) - min(starts) < 0.2
|
||
|
|
assert result["ok"] is True
|
||
|
|
assert result["input_wavs_distinct"] is True
|
||
|
|
assert result["output_dirs_distinct"] is True
|
||
|
|
assert result["identity_isolation_ok"] is True
|
||
|
|
assert result["response_audio_contract_ok"] is True
|
||
|
|
assert {session["audio_delta_count"] for session in result["sessions"]} == {2}
|
||
|
|
for session in result["sessions"]:
|
||
|
|
assert session["all_audio_deltas_same_response"] is True
|
||
|
|
assert session["response_id_nonempty"] is True
|
||
|
|
assert session["first_audio_delta_before_done"] is True
|
||
|
|
assert len(session["audio_delta_timings"]) == 2
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_pair_rejects_false_green_audio_contract(tmp_path):
|
||
|
|
demo = _load_pair_demo_module()
|
||
|
|
output = tmp_path / "session"
|
||
|
|
output.mkdir()
|
||
|
|
response_id = "resp-a"
|
||
|
|
events = [
|
||
|
|
{"type": "response.created", "response": {"id": response_id}},
|
||
|
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "sample_rate_hz": 24000},
|
||
|
|
{"type": "response.audio.delta", "response_id": "resp-other", "delta": "AAAA", "sample_rate_hz": 24000},
|
||
|
|
{"type": "response.done", "response_id": response_id},
|
||
|
|
]
|
||
|
|
(output / "events.jsonl").write_text(
|
||
|
|
"".join(demo.json.dumps(event) + "\n" for event in events),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
(output / "result.json").write_text(
|
||
|
|
demo.json.dumps({"ok": True, "response_ids": [response_id], "audio_bytes": 4}),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
|
||
|
|
session = demo._summarize_session(
|
||
|
|
label="a",
|
||
|
|
input_wav="request.wav",
|
||
|
|
output_dir=str(output),
|
||
|
|
returncode=0,
|
||
|
|
stdout="",
|
||
|
|
stderr="",
|
||
|
|
)
|
||
|
|
|
||
|
|
assert session["ok"] is False
|
||
|
|
assert session["all_audio_deltas_same_response"] is False
|
||
|
|
assert session["response_audio_contract_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_pair_default_requires_multiple_audio_deltas(monkeypatch):
|
||
|
|
demo = _load_pair_demo_module()
|
||
|
|
monkeypatch.setattr(
|
||
|
|
demo.sys,
|
||
|
|
"argv",
|
||
|
|
[
|
||
|
|
"pair.py",
|
||
|
|
"--input-wav-a",
|
||
|
|
"a.wav",
|
||
|
|
"--input-wav-b",
|
||
|
|
"b.wav",
|
||
|
|
"--output-dir-a",
|
||
|
|
"a",
|
||
|
|
"--output-dir-b",
|
||
|
|
"b",
|
||
|
|
"--ref-audio",
|
||
|
|
"ref.wav",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
args = demo.parse_args()
|
||
|
|
|
||
|
|
assert args.ref_audio == "ref.wav"
|
||
|
|
assert args.min_audio_deltas_per_session == 2
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_pair_requires_ref_audio(monkeypatch):
|
||
|
|
demo = _load_pair_demo_module()
|
||
|
|
monkeypatch.setattr(
|
||
|
|
demo.sys,
|
||
|
|
"argv",
|
||
|
|
[
|
||
|
|
"pair.py",
|
||
|
|
"--input-wav-a",
|
||
|
|
"a.wav",
|
||
|
|
"--input-wav-b",
|
||
|
|
"b.wav",
|
||
|
|
"--output-dir-a",
|
||
|
|
"a",
|
||
|
|
"--output-dir-b",
|
||
|
|
"b",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(SystemExit):
|
||
|
|
demo.parse_args()
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_requires_ref_audio(monkeypatch):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
monkeypatch.setattr(
|
||
|
|
demo.sys,
|
||
|
|
"argv",
|
||
|
|
[
|
||
|
|
"soft.py",
|
||
|
|
"--input-wav",
|
||
|
|
"input.wav",
|
||
|
|
"--output-dir",
|
||
|
|
"out",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(SystemExit):
|
||
|
|
demo.parse_args()
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_accepts_explicit_ref_audio(monkeypatch):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
monkeypatch.setattr(
|
||
|
|
demo.sys,
|
||
|
|
"argv",
|
||
|
|
[
|
||
|
|
"soft.py",
|
||
|
|
"--input-wav",
|
||
|
|
"input.wav",
|
||
|
|
"--output-dir",
|
||
|
|
"out",
|
||
|
|
"--ref-audio",
|
||
|
|
"ref.wav",
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
args = demo.parse_args()
|
||
|
|
|
||
|
|
assert args.ref_audio == "ref.wav"
|
||
|
|
assert args.validation_mode == "model-policy"
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"extra_args",
|
||
|
|
[
|
||
|
|
["--expect-second-response-substring", "一加一等于二"],
|
||
|
|
["--input-sha256", "a" * 64],
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_realtime_duplex_soft_interrupt_response_required_needs_bound_fixture(monkeypatch, extra_args):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
monkeypatch.setattr(
|
||
|
|
demo.sys,
|
||
|
|
"argv",
|
||
|
|
[
|
||
|
|
"soft.py",
|
||
|
|
"--input-wav",
|
||
|
|
"input.wav",
|
||
|
|
"--output-dir",
|
||
|
|
"out",
|
||
|
|
"--ref-audio",
|
||
|
|
"ref.wav",
|
||
|
|
"--validation-mode",
|
||
|
|
"response-required",
|
||
|
|
*extra_args,
|
||
|
|
],
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(SystemExit):
|
||
|
|
demo.parse_args()
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_rejects_input_checksum_mismatch(tmp_path):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
input_wav = tmp_path / "input.wav"
|
||
|
|
input_wav.write_bytes(b"two-turn fixture")
|
||
|
|
|
||
|
|
with pytest.raises(ValueError, match="input WAV SHA256 mismatch"):
|
||
|
|
demo._validate_input_sha256(input_wav, "0" * 64)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_accepts_multi_delta_handoff_sequence(tmp_path):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
output = tmp_path / "soft_interrupt"
|
||
|
|
output.mkdir()
|
||
|
|
first_response_id = "resp-first"
|
||
|
|
second_response_id = "resp-second"
|
||
|
|
events = [
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 1.0},
|
||
|
|
{"type": "response.created", "response": {"id": first_response_id}, "_client_received_at_s": 2.0},
|
||
|
|
{"type": "response.speak", "response_id": first_response_id, "_client_received_at_s": 2.0},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 2.1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "中国古代四大发明",
|
||
|
|
"_client_received_at_s": 2.1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 2.6,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "是造纸术。",
|
||
|
|
"_client_received_at_s": 2.6,
|
||
|
|
},
|
||
|
|
{"type": "response.done", "response_id": first_response_id, "_client_received_at_s": 3.0},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 3.2},
|
||
|
|
{"type": "response.created", "response": {"id": second_response_id}, "_client_received_at_s": 4.0},
|
||
|
|
{"type": "response.speak", "response_id": second_response_id, "_client_received_at_s": 4.0},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 4.1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "一加一等于",
|
||
|
|
"_client_received_at_s": 4.1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 4.6,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "二。",
|
||
|
|
"_client_received_at_s": 4.6,
|
||
|
|
},
|
||
|
|
{"type": "response.done", "response_id": second_response_id, "_client_received_at_s": 5.0},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 5.2},
|
||
|
|
{"type": "input_audio_buffer.committed", "_client_received_at_s": 6.0},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 6.1},
|
||
|
|
]
|
||
|
|
(output / "events.jsonl").write_text(
|
||
|
|
"".join(demo.json.dumps(event) + "\n" for event in events),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
(output / "result.json").write_text(
|
||
|
|
demo.json.dumps({"ok": True, "response_ids": [first_response_id, second_response_id]}),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
|
||
|
|
summary = demo.summarize_artifacts(
|
||
|
|
output_dir=output,
|
||
|
|
validation_mode="response-required",
|
||
|
|
min_responses=2,
|
||
|
|
min_audio_deltas_per_response=2,
|
||
|
|
expect_second_response_substring="一加一等于二",
|
||
|
|
)
|
||
|
|
|
||
|
|
assert summary["ok"] is True
|
||
|
|
assert summary["second_response_before_final_commit"] is True
|
||
|
|
assert summary["final_listen_after_commit"] is True
|
||
|
|
assert summary["response_audio_contract_ok"] is True
|
||
|
|
assert summary["response_summaries"][0]["audio_delta_count"] == 2
|
||
|
|
assert summary["response_summaries"][1]["transcript"] == "一加一等于二。"
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_model_policy_accepts_single_response(tmp_path):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
output = tmp_path / "model_policy"
|
||
|
|
output.mkdir()
|
||
|
|
response_id = "resp-only"
|
||
|
|
events = [
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 1.0},
|
||
|
|
{"type": "response.created", "response": {"id": response_id}, "_client_received_at_s": 2.0},
|
||
|
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.1},
|
||
|
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.15},
|
||
|
|
{"type": "response.done", "response_id": response_id, "_client_received_at_s": 2.2},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 2.3},
|
||
|
|
{"type": "input_audio_buffer.committed", "_client_received_at_s": 3.0},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 3.1},
|
||
|
|
]
|
||
|
|
(output / "events.jsonl").write_text(
|
||
|
|
"".join(demo.json.dumps(event) + "\n" for event in events),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
(output / "result.json").write_text(
|
||
|
|
demo.json.dumps({"ok": True, "response_ids": [response_id]}),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
|
||
|
|
summary = demo.summarize_artifacts(
|
||
|
|
output_dir=output,
|
||
|
|
validation_mode="model-policy",
|
||
|
|
min_responses=2,
|
||
|
|
min_audio_deltas_per_response=2,
|
||
|
|
expect_second_response_substring=None,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert summary["ok"] is True
|
||
|
|
assert summary["validation_mode"] == "model-policy"
|
||
|
|
assert summary["enough_responses"] is True
|
||
|
|
assert summary["response_before_final_commit"] is True
|
||
|
|
assert summary["listen_after_response_before_commit"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_response_required_rejects_single_response(tmp_path):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
output = tmp_path / "response_required"
|
||
|
|
output.mkdir()
|
||
|
|
response_id = "resp-only"
|
||
|
|
events = [
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 1.0},
|
||
|
|
{"type": "response.created", "response": {"id": response_id}, "_client_received_at_s": 2.0},
|
||
|
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.1},
|
||
|
|
{"type": "response.audio.delta", "response_id": response_id, "delta": "AAAA", "_client_received_at_s": 2.15},
|
||
|
|
{"type": "response.done", "response_id": response_id, "_client_received_at_s": 2.2},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 2.3},
|
||
|
|
{"type": "input_audio_buffer.committed", "_client_received_at_s": 3.0},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 3.1},
|
||
|
|
]
|
||
|
|
(output / "events.jsonl").write_text(
|
||
|
|
"".join(demo.json.dumps(event) + "\n" for event in events),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
(output / "result.json").write_text(
|
||
|
|
demo.json.dumps({"ok": True, "response_ids": [response_id]}),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
|
||
|
|
summary = demo.summarize_artifacts(
|
||
|
|
output_dir=output,
|
||
|
|
validation_mode="response-required",
|
||
|
|
min_responses=2,
|
||
|
|
min_audio_deltas_per_response=2,
|
||
|
|
expect_second_response_substring="一加一等于二",
|
||
|
|
)
|
||
|
|
|
||
|
|
assert summary["ok"] is False
|
||
|
|
assert summary["enough_responses"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_soft_interrupt_checks_expected_text_in_second_response(tmp_path):
|
||
|
|
demo = _load_soft_interrupt_demo_module()
|
||
|
|
output = tmp_path / "second_response_text"
|
||
|
|
output.mkdir()
|
||
|
|
first_response_id = "resp-first"
|
||
|
|
second_response_id = "resp-second"
|
||
|
|
events = [
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 1.0},
|
||
|
|
{"type": "response.created", "response": {"id": first_response_id}, "_client_received_at_s": 2.0},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 2.1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 2.2,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": first_response_id,
|
||
|
|
"delta": "一加一等于二",
|
||
|
|
"_client_received_at_s": 2.2,
|
||
|
|
},
|
||
|
|
{"type": "response.done", "response_id": first_response_id, "_client_received_at_s": 2.3},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 2.4},
|
||
|
|
{"type": "response.created", "response": {"id": second_response_id}, "_client_received_at_s": 3.0},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 3.1,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "AAAA",
|
||
|
|
"_client_received_at_s": 3.2,
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": second_response_id,
|
||
|
|
"delta": "不知道",
|
||
|
|
"_client_received_at_s": 3.2,
|
||
|
|
},
|
||
|
|
{"type": "response.done", "response_id": second_response_id, "_client_received_at_s": 3.3},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 3.4},
|
||
|
|
{"type": "input_audio_buffer.committed", "_client_received_at_s": 4.0},
|
||
|
|
{"type": "response.listen", "_client_received_at_s": 4.1},
|
||
|
|
]
|
||
|
|
(output / "events.jsonl").write_text(
|
||
|
|
"".join(demo.json.dumps(event) + "\n" for event in events),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
(output / "result.json").write_text(
|
||
|
|
demo.json.dumps({"ok": True, "response_ids": [first_response_id, second_response_id]}),
|
||
|
|
encoding="utf-8",
|
||
|
|
)
|
||
|
|
|
||
|
|
summary = demo.summarize_artifacts(
|
||
|
|
output_dir=output,
|
||
|
|
validation_mode="response-required",
|
||
|
|
min_responses=2,
|
||
|
|
min_audio_deltas_per_response=2,
|
||
|
|
expect_second_response_substring="一加一等于二",
|
||
|
|
)
|
||
|
|
|
||
|
|
assert summary["ok"] is False
|
||
|
|
assert summary["second_response_transcript_expectation_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_multi_session_resume_url_disables_autostart():
|
||
|
|
demo = _load_multi_demo_module()
|
||
|
|
|
||
|
|
url = demo._with_resume_mode("ws://localhost:8113/v1/realtime?duplex=1&model=openbmb%2FMiniCPM-o-4_5")
|
||
|
|
|
||
|
|
query = parse_qs(urlsplit(url).query)
|
||
|
|
assert query["model"] == ["openbmb/MiniCPM-o-4_5"]
|
||
|
|
assert query["resume"] == ["1"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_multi_session_rejects_cross_session_response_identity():
|
||
|
|
demo = _load_multi_demo_module()
|
||
|
|
|
||
|
|
assert demo._validate_identity_isolation(
|
||
|
|
[
|
||
|
|
{"completed_response_ids": ["resp-a"]},
|
||
|
|
{"completed_response_ids": ["resp-b"]},
|
||
|
|
]
|
||
|
|
)
|
||
|
|
assert not demo._validate_identity_isolation(
|
||
|
|
[
|
||
|
|
{"completed_response_ids": ["resp-shared"]},
|
||
|
|
{"completed_response_ids": ["resp-shared"]},
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_multi_session_reads_nested_terminal_identity():
|
||
|
|
demo = _load_multi_demo_module()
|
||
|
|
|
||
|
|
assert demo._event_session_id({"type": "session.heartbeat_ack", "session_id": "sid"}) == "sid"
|
||
|
|
assert (
|
||
|
|
demo._event_session_id({"type": "session.closed", "event": {"type": "session.closed", "session_id": "sid"}})
|
||
|
|
== "sid"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_resolves_distinct_turn_inputs():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
primary = Path("first.wav")
|
||
|
|
|
||
|
|
assert demo._turn_input_paths(primary, [], turns=3) == [primary, primary, primary]
|
||
|
|
assert demo._turn_input_paths(primary, ["second.wav", "third.wav"], turns=3) == [
|
||
|
|
primary,
|
||
|
|
Path("second.wav"),
|
||
|
|
Path("third.wav"),
|
||
|
|
]
|
||
|
|
with pytest.raises(ValueError, match="one --turn-input-wav"):
|
||
|
|
demo._turn_input_paths(primary, ["second.wav"], turns=3)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_explicitly_enables_native_runtime_before_connect():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
url = demo._url_with_model(
|
||
|
|
"ws://localhost:8099/v1/realtime?duplex=1",
|
||
|
|
"openbmb/MiniCPM-o-4_5",
|
||
|
|
)
|
||
|
|
|
||
|
|
query = parse_qs(urlsplit(url).query)
|
||
|
|
assert query["minicpmo45_native_duplex"] == ["1"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_explicit_session_id_reaches_autostart_query():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
url = demo._url_with_model(
|
||
|
|
"ws://localhost:8099/v1/realtime?duplex=1",
|
||
|
|
"openbmb/MiniCPM-o-4_5",
|
||
|
|
session_id="reopen-e2e",
|
||
|
|
)
|
||
|
|
|
||
|
|
query = parse_qs(urlsplit(url).query)
|
||
|
|
assert query["session_id"] == ["reopen-e2e"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_session_update_uses_explicit_session_id():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
event = demo._session_update_event(
|
||
|
|
SimpleNamespace(
|
||
|
|
model="openbmb/MiniCPM-o-4_5",
|
||
|
|
output_audio_format="pcm16",
|
||
|
|
short_ack_ms=1200,
|
||
|
|
session_id="reopen-e2e",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert event["type"] == "session.update"
|
||
|
|
assert "session_id" not in event
|
||
|
|
assert event["session"]["session_id"] == "reopen-e2e"
|
||
|
|
assert event["session"]["extra_body"]["minicpmo45_native_duplex"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_response_required_uses_deterministic_sampling():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
event = demo._session_update_event(
|
||
|
|
SimpleNamespace(
|
||
|
|
model="openbmb/MiniCPM-o-4_5",
|
||
|
|
output_audio_format="pcm16",
|
||
|
|
short_ack_ms=1200,
|
||
|
|
session_id=None,
|
||
|
|
validation_mode="response-required",
|
||
|
|
temperature=None,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert event["session"]["temperature"] == 0.0
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_model_policy_preserves_default_sampling():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
event = demo._session_update_event(
|
||
|
|
SimpleNamespace(
|
||
|
|
model="openbmb/MiniCPM-o-4_5",
|
||
|
|
output_audio_format="pcm16",
|
||
|
|
short_ack_ms=1200,
|
||
|
|
session_id=None,
|
||
|
|
validation_mode="model-policy",
|
||
|
|
temperature=None,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert "temperature" not in event["session"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_resolves_explicit_turn_durations():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
assert demo._turn_durations([], turns=3, first_turn_ms=3000) == [3000, 1200, 1200]
|
||
|
|
assert demo._turn_durations([0, 900, 1500], turns=3, first_turn_ms=3000) == [None, 900, 1500]
|
||
|
|
with pytest.raises(ValueError, match="one --turn-duration-ms"):
|
||
|
|
demo._turn_durations([0, 900], turns=3, first_turn_ms=3000)
|
||
|
|
with pytest.raises(ValueError, match="non-negative"):
|
||
|
|
demo._turn_durations([0, -1, 900], turns=3, first_turn_ms=3000)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_resolves_transcript_labels_for_requested_turns():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
assert demo._turn_transcripts("first", turns=1) == ["first"]
|
||
|
|
assert demo._turn_transcripts("first", turns=4) == ["first", "继续", "再说一次", "turn-4"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_reads_response_playback_cursor():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.done",
|
||
|
|
"response_id": "resp-1",
|
||
|
|
"response": {
|
||
|
|
"id": "resp-1",
|
||
|
|
"metadata": {
|
||
|
|
"playback": {
|
||
|
|
"sent_ms": 27920,
|
||
|
|
"played_ms": 0,
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert state.response_playback_sent_ms("resp-1") == 27920
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_partitions_timing_by_response_identity():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
for response_id, created_at_s, audio_at_s, token_count in (
|
||
|
|
("resp-1", 10.0, 10.1, 3),
|
||
|
|
("resp-2", 20.0, 20.2, 5),
|
||
|
|
):
|
||
|
|
state.add(
|
||
|
|
{"type": "response.created", "response": {"id": response_id}},
|
||
|
|
received_at_s=created_at_s,
|
||
|
|
)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": response_id,
|
||
|
|
"delta": base64.b64encode(b"audio").decode("ascii"),
|
||
|
|
"metadata": {
|
||
|
|
"audio_duration_ms": 80,
|
||
|
|
"vllm_omni": {
|
||
|
|
"stage_metrics": {
|
||
|
|
"0": {
|
||
|
|
"num_tokens_out": token_count,
|
||
|
|
"vllm_itls_ms": [10.0],
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
},
|
||
|
|
},
|
||
|
|
received_at_s=audio_at_s,
|
||
|
|
)
|
||
|
|
|
||
|
|
timings = state.response_timing_summaries()
|
||
|
|
|
||
|
|
assert timings["resp-1"]["stage0_tokens"]["output_token_count"] == 3
|
||
|
|
assert timings["resp-1"]["audio_output"]["response_created_to_first_audio_ms"] == 100.0
|
||
|
|
assert timings["resp-1"]["audio_output"]["commit_to_first_audio_ms"] is None
|
||
|
|
assert timings["resp-2"]["stage0_tokens"]["output_token_count"] == 5
|
||
|
|
assert timings["resp-2"]["audio_output"]["response_created_to_first_audio_ms"] == 200.0
|
||
|
|
assert timings["resp-2"]["audio_output"]["commit_to_first_audio_ms"] is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_model_policy_accepts_one_listen_per_streamed_turn():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "session.created"})
|
||
|
|
for turn in range(3):
|
||
|
|
state.add({"type": "input_audio_buffer.speech_started", "turn": turn})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.listen",
|
||
|
|
"response": {
|
||
|
|
"status": "listening",
|
||
|
|
"metadata": {"model_listen": True},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
state.add({"type": "input_audio_buffer.committed", "turn": turn})
|
||
|
|
|
||
|
|
assert state.model_policy_event_order_ok(expected_turns=3)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_model_policy_accepts_continuous_input_without_commits():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "session.created"})
|
||
|
|
state.add({"type": "input_audio_buffer.speech_started"})
|
||
|
|
for _ in range(3):
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.listen",
|
||
|
|
"response": {"metadata": {"model_listen": True}},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert not state.model_policy_event_order_ok(expected_turns=3)
|
||
|
|
assert state.model_policy_event_order_ok(expected_turns=3, require_input_commit=False)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_accepts_overlap_unit_terminating_continuous_response():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "response.created", "response_id": "resp-1"})
|
||
|
|
state.add({"type": "response.done", "response_id": "resp-1"})
|
||
|
|
|
||
|
|
assert demo._continuous_overlap_terminal_is_outcome(
|
||
|
|
state,
|
||
|
|
validation_mode="model-policy",
|
||
|
|
model_unit_ready_while_active=True,
|
||
|
|
before_created=1,
|
||
|
|
before_model_listen=0,
|
||
|
|
)
|
||
|
|
assert not demo._continuous_overlap_terminal_is_outcome(
|
||
|
|
state,
|
||
|
|
validation_mode="response-required",
|
||
|
|
model_unit_ready_while_active=True,
|
||
|
|
before_created=1,
|
||
|
|
before_model_listen=0,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_full_turn_duration_does_not_slice_audio():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
pcm16 = b"\x01\x00" * (demo.PCM16_SAMPLE_RATE * 2)
|
||
|
|
|
||
|
|
assert demo._select_turn_audio(pcm16, None) == pcm16
|
||
|
|
assert len(demo._select_turn_audio(pcm16, 1000)) == demo.PCM16_SAMPLE_RATE * demo.PCM16_BYTES_PER_SAMPLE
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_distinct_inputs_compare_audio_content():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
paths = [Path("first.wav"), Path("second.wav"), Path("third.wav")]
|
||
|
|
|
||
|
|
assert demo._turn_inputs_are_distinct(paths, [b"first", b"second", b"third"]) is True
|
||
|
|
assert demo._turn_inputs_are_distinct(paths, [b"same", b"same", b"third"]) is False
|
||
|
|
assert demo._turn_inputs_are_distinct([paths[0], paths[0]], [b"first", b"second"]) is False
|
||
|
|
|
||
|
|
|
||
|
|
def _add_response_transcript(state, response_id, *, transcript, audio=True):
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": response_id,
|
||
|
|
"delta": "YQ==" if audio else "",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
if transcript:
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.delta",
|
||
|
|
"response_id": response_id,
|
||
|
|
"delta": transcript,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.audio_transcript.done",
|
||
|
|
"response_id": response_id,
|
||
|
|
"transcript": transcript,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.done",
|
||
|
|
"response_id": response_id,
|
||
|
|
"response": {"id": response_id, "status": "completed"},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_checks_delta_done_and_turn_independence():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(
|
||
|
|
state,
|
||
|
|
"resp-1",
|
||
|
|
transcript="第一轮回答",
|
||
|
|
)
|
||
|
|
_add_response_transcript(
|
||
|
|
state,
|
||
|
|
"resp-2",
|
||
|
|
transcript="第二轮回答",
|
||
|
|
)
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-1", "resp-2"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["transcript_delta_done_ok"] is True
|
||
|
|
assert result["cross_turn_independent_ok"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_speak_gate_accepts_one_decision_event_per_response():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-1"}})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.speak",
|
||
|
|
"response_id": "resp-1",
|
||
|
|
"metadata": {"model_speak": True},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert demo._evaluate_response_speak_contract(state)["response_speak_contract_ok"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_speak_gate_rejects_duplicate_event():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-1"}})
|
||
|
|
state.add({"type": "response.speak", "response_id": "resp-1"})
|
||
|
|
state.add({"type": "response.speak", "response_id": "resp-1"})
|
||
|
|
|
||
|
|
result = demo._evaluate_response_speak_contract(state)
|
||
|
|
|
||
|
|
assert result["response_speak_contract_ok"] is False
|
||
|
|
assert result["duplicate_response_speak_ids"] == ["resp-1"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_speak_gate_rejects_text_channel():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-1"}})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.speak",
|
||
|
|
"response_id": "resp-1",
|
||
|
|
"text": "must be emitted through response.audio_transcript.delta",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
result = demo._evaluate_response_speak_contract(state)
|
||
|
|
|
||
|
|
assert result["response_speak_contract_ok"] is False
|
||
|
|
assert result["text_bearing_response_speak_count"] == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_cross_turn_tail_reuse():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(state, "resp-1", transcript="第一轮回答")
|
||
|
|
_add_response_transcript(state, "resp-2", transcript="第二轮仍然带着第一轮回答")
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-1", "resp-2"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["cross_turn_independent_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_terminal_only_previous_tail():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(state, "resp-1", transcript="上一轮回答结尾是的吗？")
|
||
|
|
_add_response_transcript(state, "resp-2", transcript="的吗？")
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-1", "resp-2"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["cross_turn_independent_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_delta_done_mismatch():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(state, "resp-1", transcript="delta文本")
|
||
|
|
done = next(event for event in state.events if event.get("type") == "response.audio_transcript.done")
|
||
|
|
done["transcript"] = "另一个done文本"
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-1"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["transcript_delta_done_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_terminal_only_stale_tail():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(
|
||
|
|
state,
|
||
|
|
"resp-empty",
|
||
|
|
transcript="的吗？",
|
||
|
|
audio=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-empty"],
|
||
|
|
expected_empty_response_ids={"resp-empty"},
|
||
|
|
require_cross_turn_independence=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["empty_turns_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_audio_without_transcript():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(
|
||
|
|
state,
|
||
|
|
"resp-audio-no-text",
|
||
|
|
transcript="",
|
||
|
|
audio=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-audio-no-text"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["nonempty_audio_has_transcript_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_response_required_rejects_unselected_audio_only_response():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(state, "resp-audio-only", transcript="", audio=True)
|
||
|
|
_add_response_transcript(state, "resp-with-text", transcript="你好", audio=True)
|
||
|
|
|
||
|
|
assert (
|
||
|
|
demo._all_audio_responses_have_transcript(
|
||
|
|
state,
|
||
|
|
["resp-audio-only", "resp-with-text"],
|
||
|
|
)
|
||
|
|
is False
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_incomplete_model_turn_sentence():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(state, "resp-1", transcript="哎，不是说好不")
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-1"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=False,
|
||
|
|
require_terminal_punctuation=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["terminal_punctuation_ok"] is False
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_accepts_short_interjection_without_punctuation():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
_add_response_transcript(state, "resp-1", transcript="哈哈")
|
||
|
|
|
||
|
|
result = demo._evaluate_transcript_integrity(
|
||
|
|
state,
|
||
|
|
["resp-1"],
|
||
|
|
expected_empty_response_ids=set(),
|
||
|
|
require_cross_turn_independence=False,
|
||
|
|
require_terminal_punctuation=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["terminal_punctuation_ok"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_response_required_skips_audio_only_model_turn(monkeypatch):
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
|
||
|
|
async def fake_send_pcm16(*args, **kwargs):
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-audio-only"}})
|
||
|
|
_add_response_transcript(state, "resp-audio-only", transcript="", audio=True)
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-with-text"}})
|
||
|
|
_add_response_transcript(state, "resp-with-text", transcript="你好", audio=True)
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
async def send(self, payload):
|
||
|
|
return None
|
||
|
|
|
||
|
|
monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16)
|
||
|
|
|
||
|
|
response_id, outcome = asyncio.run(
|
||
|
|
demo._send_clean_turn(
|
||
|
|
FakeWebSocket(),
|
||
|
|
state,
|
||
|
|
b"\x00\x00",
|
||
|
|
transcript="fixture",
|
||
|
|
duration_ms=None,
|
||
|
|
chunk_ms=200,
|
||
|
|
timeout_s=0.1,
|
||
|
|
require_audio=True,
|
||
|
|
validation_mode="response-required",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response_id == "resp-with-text"
|
||
|
|
assert outcome == "speak"
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_no_hint_realtime_turn_omits_transcript_hint(monkeypatch):
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
captured = {}
|
||
|
|
|
||
|
|
async def fake_send_pcm16(*args, **kwargs):
|
||
|
|
del args
|
||
|
|
captured.update(kwargs)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.listen",
|
||
|
|
"response": {"metadata": {"model_listen": True}},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
async def send(self, payload):
|
||
|
|
del payload
|
||
|
|
|
||
|
|
monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16)
|
||
|
|
|
||
|
|
response_id, outcome = asyncio.run(
|
||
|
|
demo._send_clean_turn(
|
||
|
|
FakeWebSocket(),
|
||
|
|
state,
|
||
|
|
b"\x00\x00",
|
||
|
|
transcript="turn label only",
|
||
|
|
duration_ms=None,
|
||
|
|
chunk_ms=200,
|
||
|
|
timeout_s=0.1,
|
||
|
|
require_audio=False,
|
||
|
|
validation_mode="model-policy",
|
||
|
|
send_transcript_hint=False,
|
||
|
|
realtime_input=True,
|
||
|
|
model_policy_settle_s=0.01,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response_id is None
|
||
|
|
assert outcome == "listen"
|
||
|
|
assert captured["hints"] == {}
|
||
|
|
assert captured["realtime_delay"] is True
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_continuous_input_omits_browser_commit(monkeypatch):
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
|
||
|
|
async def fake_send_pcm16(*args, **kwargs):
|
||
|
|
del args, kwargs
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-continuous"}})
|
||
|
|
_add_response_transcript(state, "resp-continuous", transcript="连续输入", audio=True)
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
def __init__(self):
|
||
|
|
self.messages = []
|
||
|
|
|
||
|
|
async def send(self, payload):
|
||
|
|
self.messages.append(demo.json.loads(payload))
|
||
|
|
|
||
|
|
monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16)
|
||
|
|
ws = FakeWebSocket()
|
||
|
|
|
||
|
|
response_id, outcome = asyncio.run(
|
||
|
|
demo._send_clean_turn(
|
||
|
|
ws,
|
||
|
|
state,
|
||
|
|
b"\x00\x00",
|
||
|
|
transcript="fixture",
|
||
|
|
duration_ms=None,
|
||
|
|
chunk_ms=200,
|
||
|
|
timeout_s=0.1,
|
||
|
|
require_audio=True,
|
||
|
|
validation_mode="response-required",
|
||
|
|
commit_input=False,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response_id == "resp-continuous"
|
||
|
|
assert outcome == "speak"
|
||
|
|
assert not any(message.get("type") == "input_audio_buffer.commit" for message in ws.messages)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_no_hint_gate_does_not_require_transcription_event():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
assert demo._input_transcription_ok(0, transcript_hints_enabled=False)
|
||
|
|
assert not demo._input_transcription_ok(0, transcript_hints_enabled=True)
|
||
|
|
assert demo._input_transcription_ok(3, transcript_hints_enabled=True)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_gate_rejects_server_errors():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "error", "code": "runtime_signal_failed", "error": "signal failed"})
|
||
|
|
|
||
|
|
errors = demo._unexpected_error_events(state)
|
||
|
|
|
||
|
|
assert errors == [{"type": "error", "code": "runtime_signal_failed", "error": "signal failed"}]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_model_policy_waits_for_speak_after_intermediate_listen(monkeypatch):
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
|
||
|
|
async def fake_send_pcm16(*args, **kwargs):
|
||
|
|
del args, kwargs
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.listen",
|
||
|
|
"response": {"metadata": {"model_listen": True}},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
async def delayed_speak():
|
||
|
|
await asyncio.sleep(0.01)
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-delayed"}})
|
||
|
|
_add_response_transcript(state, "resp-delayed", transcript="延迟回答", audio=False)
|
||
|
|
|
||
|
|
asyncio.create_task(delayed_speak())
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
async def send(self, payload):
|
||
|
|
del payload
|
||
|
|
|
||
|
|
monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16)
|
||
|
|
|
||
|
|
response_id, outcome = asyncio.run(
|
||
|
|
demo._send_clean_turn(
|
||
|
|
FakeWebSocket(),
|
||
|
|
state,
|
||
|
|
b"\x00\x00",
|
||
|
|
transcript="fixture",
|
||
|
|
duration_ms=None,
|
||
|
|
chunk_ms=200,
|
||
|
|
timeout_s=0.2,
|
||
|
|
require_audio=False,
|
||
|
|
validation_mode="model-policy",
|
||
|
|
model_policy_settle_s=0.05,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response_id == "resp-delayed"
|
||
|
|
assert outcome == "speak"
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_playback_gate_covers_unassigned_completed_response():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
for response_id in ("resp-assigned", "resp-unassigned"):
|
||
|
|
state.add({"type": "response.created", "response": {"id": response_id}})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.done",
|
||
|
|
"response": {
|
||
|
|
"id": response_id,
|
||
|
|
"metadata": {"playback": {"sent_ms": 1200}},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "playback.acknowledged",
|
||
|
|
"event": {
|
||
|
|
"item_id": "item_resp-assigned",
|
||
|
|
"history_committed": True,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert not demo._all_response_playback_history_committed(
|
||
|
|
state,
|
||
|
|
state.completed_response_ids(),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_acks_unassigned_completed_response():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
for response_id in ("resp-assigned", "resp-unassigned"):
|
||
|
|
state.add({"type": "response.created", "response": {"id": response_id}})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.done",
|
||
|
|
"response": {
|
||
|
|
"id": response_id,
|
||
|
|
"metadata": {"playback": {"sent_ms": 1200}},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "playback.acknowledged",
|
||
|
|
"event": {
|
||
|
|
"item_id": "item_resp-assigned",
|
||
|
|
"history_committed": True,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
def __init__(self):
|
||
|
|
self.messages = []
|
||
|
|
|
||
|
|
async def send(self, payload):
|
||
|
|
message = demo.json.loads(payload)
|
||
|
|
self.messages.append(message)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "playback.acknowledged",
|
||
|
|
"event": {
|
||
|
|
"item_id": message["item_id"],
|
||
|
|
"history_committed": True,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
ws = FakeWebSocket()
|
||
|
|
asyncio.run(demo._ack_all_completed_response_playback(ws, state, timeout_s=0.1))
|
||
|
|
|
||
|
|
assert [message["response_id"] for message in ws.messages] == ["resp-unassigned"]
|
||
|
|
assert demo._all_response_playback_history_committed(state, state.completed_response_ids())
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_model_policy_does_not_assume_one_response_per_physical_input():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
assert demo._response_cardinality_ok(
|
||
|
|
["resp-1", "resp-2", "resp-3", "resp-4"],
|
||
|
|
expected_turns=3,
|
||
|
|
validation_mode="model-policy",
|
||
|
|
)
|
||
|
|
assert not demo._response_cardinality_ok(
|
||
|
|
["resp-1", "resp-2", "resp-3", "resp-4"],
|
||
|
|
expected_turns=3,
|
||
|
|
validation_mode="response-required",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_only_requires_cross_turn_independence_for_response_required():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
assert not demo._requires_cross_turn_independence(
|
||
|
|
validation_mode="model-policy",
|
||
|
|
distinct_inputs_required=True,
|
||
|
|
)
|
||
|
|
assert demo._requires_cross_turn_independence(
|
||
|
|
validation_mode="response-required",
|
||
|
|
distinct_inputs_required=True,
|
||
|
|
)
|
||
|
|
assert not demo._requires_cross_turn_independence(
|
||
|
|
validation_mode="response-required",
|
||
|
|
distinct_inputs_required=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_drains_late_model_response_before_close():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-first"}})
|
||
|
|
state.add({"type": "response.done", "response": {"id": "resp-first"}})
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
def __init__(self):
|
||
|
|
self.messages = []
|
||
|
|
|
||
|
|
async def send(self, payload):
|
||
|
|
self.messages.append(demo.json.loads(payload))
|
||
|
|
|
||
|
|
async def run_fixture():
|
||
|
|
async def finish_late_response():
|
||
|
|
await asyncio.sleep(0.01)
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-late"}})
|
||
|
|
await asyncio.sleep(0.01)
|
||
|
|
state.add({"type": "response.done", "response": {"id": "resp-late"}})
|
||
|
|
|
||
|
|
late_response = asyncio.create_task(finish_late_response())
|
||
|
|
ws = FakeWebSocket()
|
||
|
|
await demo._drain_model_policy_responses(
|
||
|
|
ws,
|
||
|
|
state,
|
||
|
|
timeout_s=0.2,
|
||
|
|
settle_s=0.03,
|
||
|
|
)
|
||
|
|
await late_response
|
||
|
|
return ws
|
||
|
|
|
||
|
|
ws = asyncio.run(run_fixture())
|
||
|
|
|
||
|
|
assert state.completed_response_ids() == ["resp-first", "resp-late"]
|
||
|
|
assert not [message for message in ws.messages if message.get("type") == "session.close"]
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_model_response_drain_times_out_with_active_ids():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-active"}})
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
async def send(self, payload):
|
||
|
|
del payload
|
||
|
|
|
||
|
|
with pytest.raises(TimeoutError, match="resp-active"):
|
||
|
|
asyncio.run(
|
||
|
|
demo._drain_model_policy_responses(
|
||
|
|
FakeWebSocket(),
|
||
|
|
state,
|
||
|
|
timeout_s=0.03,
|
||
|
|
settle_s=0.01,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_waits_at_each_model_unit_and_stops_after_speak():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
def __init__(self):
|
||
|
|
self.messages = []
|
||
|
|
|
||
|
|
async def send(self, payload):
|
||
|
|
self.messages.append(demo.json.loads(payload))
|
||
|
|
|
||
|
|
async def run_fixture():
|
||
|
|
ws = FakeWebSocket()
|
||
|
|
model_unit_message_counts = []
|
||
|
|
|
||
|
|
async def on_model_unit_ready():
|
||
|
|
model_unit_message_counts.append(len(ws.messages))
|
||
|
|
await asyncio.sleep(0)
|
||
|
|
return len(model_unit_message_counts) < 2
|
||
|
|
|
||
|
|
await demo._send_pcm16(
|
||
|
|
ws,
|
||
|
|
b"\x01\x00" * (demo.PCM16_SAMPLE_RATE * 3),
|
||
|
|
chunk_ms=200,
|
||
|
|
realtime_delay=False,
|
||
|
|
on_model_unit_ready=on_model_unit_ready,
|
||
|
|
)
|
||
|
|
return ws, model_unit_message_counts
|
||
|
|
|
||
|
|
ws, model_unit_message_counts = asyncio.run(run_fixture())
|
||
|
|
|
||
|
|
assert model_unit_message_counts == [5, 10]
|
||
|
|
assert len(ws.messages) == 10
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_listen_only_overlap_sends_next_turn_before_first_done(monkeypatch):
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
send_calls = 0
|
||
|
|
|
||
|
|
async def fake_send_pcm16(*args, **kwargs):
|
||
|
|
nonlocal send_calls
|
||
|
|
send_calls += 1
|
||
|
|
if send_calls == 1:
|
||
|
|
assert len(args[1]) == 2 + demo.PCM16_SAMPLE_RATE * demo.PCM16_BYTES_PER_SAMPLE
|
||
|
|
state.add({"type": "input_audio_buffer.speech_started", "turn": 1})
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-first"}})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": "resp-first",
|
||
|
|
"delta": "YQ==",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
assert await kwargs["on_model_unit_ready"]() is False
|
||
|
|
return
|
||
|
|
|
||
|
|
del args
|
||
|
|
assert not state.response_done("resp-first")
|
||
|
|
kwargs["on_model_unit_ready"]()
|
||
|
|
state.add({"type": "input_audio_buffer.speech_started", "turn": 2})
|
||
|
|
_add_response_transcript(state, "resp-first", transcript="第一轮完成", audio=False)
|
||
|
|
state.add({"type": "response.created", "response": {"id": "resp-second"}})
|
||
|
|
_add_response_transcript(state, "resp-second", transcript="第二轮完成", audio=True)
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
def __init__(self):
|
||
|
|
self.messages = []
|
||
|
|
|
||
|
|
async def send(self, payload):
|
||
|
|
self.messages.append(payload)
|
||
|
|
|
||
|
|
monkeypatch.setattr(demo, "_send_pcm16", fake_send_pcm16)
|
||
|
|
ws = FakeWebSocket()
|
||
|
|
|
||
|
|
response_ids, outcomes, overlap_ok = asyncio.run(
|
||
|
|
demo._send_listen_only_overlap_pair(
|
||
|
|
ws,
|
||
|
|
state,
|
||
|
|
b"\x00\x00",
|
||
|
|
b"\x01\x00",
|
||
|
|
transcripts=("first", "second"),
|
||
|
|
durations_ms=(None, None),
|
||
|
|
chunk_ms=200,
|
||
|
|
timeout_s=0.2,
|
||
|
|
send_transcript_hint=False,
|
||
|
|
realtime_input=True,
|
||
|
|
model_policy_settle_s=0.01,
|
||
|
|
validation_mode="response-required",
|
||
|
|
silence_ms=1000,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert response_ids == ["resp-first", "resp-second"]
|
||
|
|
assert outcomes == ["speak", "speak"]
|
||
|
|
assert overlap_ok is True
|
||
|
|
assert sum(demo.json.loads(message)["type"] == "input_audio_buffer.commit" for message in ws.messages) == 2
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_playback_ack_identifies_and_commits_response():
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
response_id = "resp-overlap-second"
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.done",
|
||
|
|
"response": {
|
||
|
|
"id": response_id,
|
||
|
|
"metadata": {"playback": {"sent_ms": 1200}},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
class FakeWebSocket:
|
||
|
|
def __init__(self):
|
||
|
|
self.messages = []
|
||
|
|
|
||
|
|
async def send(self, payload):
|
||
|
|
message = demo.json.loads(payload)
|
||
|
|
self.messages.append(message)
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "playback.acknowledged",
|
||
|
|
"event": {
|
||
|
|
"item_id": message.get("item_id"),
|
||
|
|
"history_committed": message.get("item_id") == f"item_{response_id}",
|
||
|
|
},
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
ws = FakeWebSocket()
|
||
|
|
asyncio.run(
|
||
|
|
demo._ack_response_playback(
|
||
|
|
ws,
|
||
|
|
state,
|
||
|
|
response_id,
|
||
|
|
timeout_s=0.1,
|
||
|
|
label="overlap second",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert ws.messages == [
|
||
|
|
{
|
||
|
|
"type": "playback.ack",
|
||
|
|
"response_id": response_id,
|
||
|
|
"item_id": f"item_{response_id}",
|
||
|
|
"played_ms": 1200,
|
||
|
|
"committed_ms": 1200,
|
||
|
|
}
|
||
|
|
]
|
||
|
|
assert state.playback_history_committed_count == 1
|
||
|
|
|
||
|
|
|
||
|
|
def test_realtime_duplex_demo_writes_audio_per_response(tmp_path):
|
||
|
|
demo = _load_demo_module()
|
||
|
|
state = demo.DemoState()
|
||
|
|
for response_id, payload in (("resp-1", b"\x01\x00"), ("resp-2", b"\x02\x00")):
|
||
|
|
state.add({"type": "response.created", "response": {"id": response_id}})
|
||
|
|
state.add(
|
||
|
|
{
|
||
|
|
"type": "response.audio.delta",
|
||
|
|
"response_id": response_id,
|
||
|
|
"delta": demo.base64.b64encode(payload).decode(),
|
||
|
|
"sample_rate_hz": 24000,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
demo._write_demo_artifacts(state, tmp_path, output_audio_format="pcm16")
|
||
|
|
|
||
|
|
for index, expected in enumerate((b"\x01\x00", b"\x02\x00"), start=1):
|
||
|
|
with wave.open(str(tmp_path / f"response_{index:02d}.wav"), "rb") as wf:
|
||
|
|
assert wf.getframerate() == 24000
|
||
|
|
assert wf.readframes(wf.getnframes()) == expected
|