[WIP][Bugfix][MiniCPM-o] Fix Full duplex bugs for minicpm-challenge (#5496)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
60 files changed
+3696
-634
No files matched your search
@@ -83,7 +83,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing text, generating audio output through the talker token2wav path."""
|
||||
"""Test processing text and generating audio through Talker and Code2Wav."""
|
||||
request_config = {"prompts": get_question("text"), "modalities": ["audio"]}
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@@ -1174,10 +1174,13 @@ async def _send_listen_only_overlap_pair(
|
||||
before_second_listen = state.model_listen_count
|
||||
overlap_started_while_active = not state.response_done(first_response_id)
|
||||
model_unit_ready_while_active = False
|
||||
second_unit_event_index: int | None = None
|
||||
|
||||
def record_model_unit_ready() -> None:
|
||||
nonlocal model_unit_ready_while_active
|
||||
nonlocal model_unit_ready_while_active, second_unit_event_index
|
||||
model_unit_ready_while_active = not state.response_done(first_response_id)
|
||||
if model_unit_ready_while_active:
|
||||
second_unit_event_index = len(state.events)
|
||||
|
||||
await _send_pcm16(
|
||||
ws,
|
||||
@@ -1239,14 +1242,13 @@ async def _send_listen_only_overlap_pair(
|
||||
label=transcripts[1],
|
||||
)
|
||||
|
||||
second_speech_index = _nth_event_index(state, "input_audio_buffer.speech_started", 2)
|
||||
first_done_index = _event_index_for_response(state, "response.done", first_response_id)
|
||||
overlap_ok = (
|
||||
overlap_started_while_active
|
||||
and model_unit_ready_while_active
|
||||
and second_speech_index is not None
|
||||
and second_unit_event_index is not None
|
||||
and first_done_index is not None
|
||||
and second_speech_index < first_done_index
|
||||
and second_unit_event_index < first_done_index
|
||||
)
|
||||
return [first_response_id, second_response_id], ["speak", second_outcome], overlap_ok
|
||||
|
||||
@@ -1380,6 +1382,17 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
await _wait_for(state, lambda: state.count("session.closed") > 0, timeout_s=20, label="session.closed")
|
||||
finally:
|
||||
if state.count("session.created") > 0 and state.count("session.closed") == 0:
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: state.count("session.closed") > 0,
|
||||
timeout_s=min(args.timeout_s, 5),
|
||||
label="session.closed during cleanup",
|
||||
)
|
||||
except (ConnectionClosed, TimeoutError):
|
||||
pass
|
||||
stop.set()
|
||||
reader.cancel()
|
||||
try:
|
||||
|
||||
@@ -19,7 +19,11 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from minicpmo_realtime_duplex_scenarios import _url_with_model, run_demo # noqa: E402
|
||||
from minicpmo_realtime_duplex_scenarios import ( # noqa: E402
|
||||
_ref_audio_data_url,
|
||||
_url_with_model,
|
||||
run_demo,
|
||||
)
|
||||
|
||||
|
||||
def _with_resume_mode(url: str) -> str:
|
||||
@@ -114,7 +118,12 @@ async def _open_admission_session(
|
||||
args: argparse.Namespace,
|
||||
session_id: str,
|
||||
) -> tuple[object, dict[str, object]]:
|
||||
url = _url_with_model(args.url, args.model, session_id=session_id)
|
||||
url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
ws = await websockets.connect(url, max_size=64 * 1024 * 1024)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
@@ -125,6 +134,7 @@ async def _open_admission_session(
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**({"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) else {}),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -151,7 +161,12 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
|
||||
accepted.append(await _open_admission_session(args, f"{prefix}-accepted-{index}"))
|
||||
|
||||
overflow_id = f"{prefix}-overflow"
|
||||
overflow_url = _url_with_model(args.url, args.model, session_id=overflow_id)
|
||||
overflow_url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=overflow_id,
|
||||
)
|
||||
async with websockets.connect(overflow_url, max_size=64 * 1024 * 1024) as overflow:
|
||||
await overflow.send(
|
||||
json.dumps(
|
||||
@@ -162,6 +177,11 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -207,16 +227,27 @@ async def _resume_probe(
|
||||
session_id: str,
|
||||
expect_expired: bool = False,
|
||||
) -> dict[str, object]:
|
||||
url = _url_with_model(args.url, args.model, session_id=session_id)
|
||||
url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024) as first:
|
||||
await first.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -306,7 +337,12 @@ async def _takeover_probe(
|
||||
*,
|
||||
session_id: str,
|
||||
) -> dict[str, object]:
|
||||
url = _url_with_model(args.url, args.model, session_id=session_id)
|
||||
url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
first = await websockets.connect(url, max_size=64 * 1024 * 1024)
|
||||
second = None
|
||||
try:
|
||||
@@ -315,9 +351,15 @@ async def _takeover_probe(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -451,20 +493,7 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]:
|
||||
args,
|
||||
session_id=f"takeover-{args.takeover_session_index}-{uuid.uuid4().hex}",
|
||||
)
|
||||
expiry_result = None
|
||||
if args.expire_session_index is not None:
|
||||
if not 0 <= args.expire_session_index < args.sessions:
|
||||
raise ValueError("--expire-session-index is outside the session range")
|
||||
expiry_result = await _resume_probe(
|
||||
args,
|
||||
session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}",
|
||||
expect_expired=True,
|
||||
)
|
||||
admission_result = (
|
||||
await _admission_probe(args, limit=args.verify_admission_limit)
|
||||
if args.verify_admission_limit is not None
|
||||
else None
|
||||
)
|
||||
lifecycle_result = await run_lifecycle_probes(args)
|
||||
|
||||
session_results = await asyncio.gather(
|
||||
*(run_demo(_demo_args(args, index)) for index in range(args.sessions)),
|
||||
@@ -487,16 +516,15 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]:
|
||||
and semantic_isolation_ok
|
||||
and (resume_result is None or resume_result.get("ok") is True)
|
||||
and (takeover_result is None or takeover_result.get("ok") is True)
|
||||
and (expiry_result is None or expiry_result.get("ok") is True)
|
||||
and (admission_result is None or admission_result.get("ok") is True)
|
||||
and lifecycle_result["ok"] is True
|
||||
),
|
||||
"session_count": args.sessions,
|
||||
"identity_isolation_ok": identity_isolation_ok,
|
||||
"semantic_isolation_ok": semantic_isolation_ok,
|
||||
"resume": resume_result,
|
||||
"takeover": takeover_result,
|
||||
"expiry": expiry_result,
|
||||
"admission": admission_result,
|
||||
"expiry": lifecycle_result["expiry"],
|
||||
"admission": lifecycle_result["admission"],
|
||||
"failures": failures,
|
||||
"sessions": completed,
|
||||
}
|
||||
@@ -507,6 +535,34 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]:
|
||||
return result
|
||||
|
||||
|
||||
async def run_lifecycle_probes(args: argparse.Namespace) -> dict[str, object]:
|
||||
"""Run expiry and admission probes without requiring model output."""
|
||||
if args.sessions < 1:
|
||||
raise ValueError("--sessions must be positive")
|
||||
expiry_result = None
|
||||
if args.expire_session_index is not None:
|
||||
if not 0 <= args.expire_session_index < args.sessions:
|
||||
raise ValueError("--expire-session-index is outside the session range")
|
||||
expiry_result = await _resume_probe(
|
||||
args,
|
||||
session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}",
|
||||
expect_expired=True,
|
||||
)
|
||||
admission_result = (
|
||||
await _admission_probe(args, limit=args.verify_admission_limit)
|
||||
if args.verify_admission_limit is not None
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"ok": (
|
||||
(expiry_result is None or expiry_result.get("ok") is True)
|
||||
and (admission_result is None or admission_result.get("ok") is True)
|
||||
),
|
||||
"expiry": expiry_result,
|
||||
"admission": admission_result,
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:8113/v1/realtime?duplex=1")
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
This E2E driver runs the public ``realtime_duplex_demo.py`` against a live
|
||||
duplex backend. Arbitrary audio defaults to the model-policy lifecycle contract.
|
||||
The stronger response-required mode binds the two-response contract to a known
|
||||
input checksum and expected second response.
|
||||
The stronger response-required mode binds the multi-response contract to a
|
||||
known input checksum and expected follow-up response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -39,6 +39,17 @@ def _validate_input_sha256(path: Path, expected: str) -> str:
|
||||
return actual
|
||||
|
||||
|
||||
def _input_duration_s(path: Path) -> float:
|
||||
with wave.open(str(path), "rb") as wav_file:
|
||||
return wav_file.getnframes() / wav_file.getframerate()
|
||||
|
||||
|
||||
def _client_process_timeout_s(input_wav: Path, protocol_timeout_s: float) -> float:
|
||||
# The child first streams the WAV in real time, then can independently
|
||||
# exhaust its post-commit and session-close protocol waits.
|
||||
return _input_duration_s(input_wav) + 2 * protocol_timeout_s + 30.0
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, object]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
@@ -211,7 +222,7 @@ def summarize_artifacts(
|
||||
validation_mode: str,
|
||||
min_responses: int,
|
||||
min_audio_deltas_per_response: int,
|
||||
expect_second_response_substring: str | None,
|
||||
expect_followup_response_substring: str | None,
|
||||
) -> dict[str, object]:
|
||||
if validation_mode not in {"model-policy", "response-required"}:
|
||||
raise ValueError(f"unsupported validation mode: {validation_mode}")
|
||||
@@ -274,12 +285,20 @@ def summarize_artifacts(
|
||||
)
|
||||
final_listen_after_commit = commit_index is not None and any(index > commit_index for index in listen_indices)
|
||||
transcript = "".join(str(summary.get("transcript") or "") for summary in response_summaries)
|
||||
second_response_transcript = (
|
||||
str(response_summaries[1].get("transcript") or "") if len(response_summaries) >= 2 else ""
|
||||
followup_response_transcripts = [
|
||||
str(summary.get("transcript") or "")
|
||||
for summary in response_summaries[1:]
|
||||
if commit_index is not None
|
||||
and isinstance(summary.get("created_index"), int)
|
||||
and summary["created_index"] < commit_index
|
||||
]
|
||||
followup_response_transcript_ok = any(
|
||||
bool(_normalize_text(transcript)) for transcript in followup_response_transcripts
|
||||
)
|
||||
followup_response_transcript_expectation_ok = not expect_followup_response_substring or any(
|
||||
_normalize_text(expect_followup_response_substring) in _normalize_text(transcript)
|
||||
for transcript in followup_response_transcripts
|
||||
)
|
||||
second_response_transcript_expectation_ok = not expect_second_response_substring or _normalize_text(
|
||||
expect_second_response_substring
|
||||
) in _normalize_text(second_response_transcript)
|
||||
error_events = [
|
||||
event
|
||||
for event in events
|
||||
@@ -310,9 +329,8 @@ def summarize_artifacts(
|
||||
mode_contract_ok = validation_mode == "model-policy" or (
|
||||
second_response_before_final_commit
|
||||
and listen_before_first_response
|
||||
and listen_between_responses
|
||||
and listen_after_last_done
|
||||
and second_response_transcript_expectation_ok
|
||||
and followup_response_transcript_ok
|
||||
)
|
||||
ok = common_contract_ok and mode_contract_ok
|
||||
return {
|
||||
@@ -337,9 +355,10 @@ def summarize_artifacts(
|
||||
"listen_after_response_before_commit": listen_after_response_before_commit,
|
||||
"final_listen_after_commit": final_listen_after_commit,
|
||||
"transcript": transcript,
|
||||
"second_response_transcript": second_response_transcript,
|
||||
"expect_second_response_substring": expect_second_response_substring,
|
||||
"second_response_transcript_expectation_ok": second_response_transcript_expectation_ok,
|
||||
"followup_response_transcripts": followup_response_transcripts,
|
||||
"followup_response_transcript_ok": followup_response_transcript_ok,
|
||||
"expect_followup_response_substring": expect_followup_response_substring,
|
||||
"followup_response_transcript_expectation_ok": followup_response_transcript_expectation_ok,
|
||||
"error_count": len(error_events),
|
||||
"cancelled_count": cancelled_count,
|
||||
"compact_sequence": _compact_sequence(events),
|
||||
@@ -388,7 +407,7 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]:
|
||||
try:
|
||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=args.timeout_s + 30.0,
|
||||
timeout=_client_process_timeout_s(input_wav, args.timeout_s),
|
||||
)
|
||||
except TimeoutError:
|
||||
process.kill()
|
||||
@@ -406,7 +425,7 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]:
|
||||
validation_mode=args.validation_mode,
|
||||
min_responses=args.min_responses,
|
||||
min_audio_deltas_per_response=args.min_audio_deltas_per_response,
|
||||
expect_second_response_substring=args.expect_second_response_substring,
|
||||
expect_followup_response_substring=args.expect_followup_response_substring,
|
||||
)
|
||||
summary.update(
|
||||
{
|
||||
@@ -444,14 +463,12 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--min-responses", type=int, default=2)
|
||||
parser.add_argument("--min-audio-deltas-per-response", type=int, default=2)
|
||||
parser.add_argument("--input-sha256")
|
||||
parser.add_argument("--expect-second-response-substring")
|
||||
parser.add_argument("--expect-followup-response-substring")
|
||||
args = parser.parse_args()
|
||||
if args.validation_mode == "response-required" and args.min_responses < 2:
|
||||
parser.error("--min-responses must be at least 2")
|
||||
if args.validation_mode == "response-required" and not args.input_sha256:
|
||||
parser.error("--input-sha256 is required in response-required mode")
|
||||
if args.validation_mode == "response-required" and not args.expect_second_response_substring:
|
||||
parser.error("--expect-second-response-substring is required in response-required mode")
|
||||
if args.min_audio_deltas_per_response < 1:
|
||||
parser.error("--min-audio-deltas-per-response must be positive")
|
||||
return args
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
"""
|
||||
E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output.
|
||||
"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output.
|
||||
|
||||
MiniCPM-o 4.5 has ``async_chunk: false``, ``max_num_seqs: 1`` on both stages,
|
||||
and the vocoder runs in-process inside the talker stage rather than as a separate
|
||||
Code2Wav stage.
|
||||
The batching deployment uses async chunk transfer across separate Thinker,
|
||||
Talker, and Code2Wav stages.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -26,10 +24,7 @@ test_params = [
|
||||
model=_MODEL,
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
use_stage_cli=True,
|
||||
server_args=[
|
||||
"--trust-remote-code",
|
||||
"--no-async-chunk",
|
||||
],
|
||||
server_args=["--trust-remote-code"],
|
||||
),
|
||||
id="default",
|
||||
)
|
||||
@@ -99,7 +94,7 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text + audio output via OpenAI API.
|
||||
This exercises the talker TTS region detection and in-process token2wav vocoder.
|
||||
This exercises Talker TTS region detection and the Code2Wav stage.
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text
|
||||
Output Modal: text + audio
|
||||
@@ -133,13 +128,14 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
audio_data_url=audio_data_url,
|
||||
content_text=get_prompt("mix"),
|
||||
content_text=get_prompt(),
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {"text": ["Beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""CI coverage for the MiniCPM-o 4.5 native-duplex Realtime API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.e2e.online_serving.minicpmo_realtime_duplex_scenarios import (
|
||||
_ref_audio_data_url,
|
||||
run_demo,
|
||||
)
|
||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
||||
run_multi_session,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.minicpmo_4_5_duplex import (
|
||||
CORE_SERVER_PARAMS,
|
||||
SERVER_PARAMS,
|
||||
demo_args,
|
||||
multi_session_args,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||
|
||||
pytestmark = pytest.mark.omni
|
||||
|
||||
|
||||
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
|
||||
async def receive() -> list[dict[str, object]]:
|
||||
events: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
while not required_types.issubset(seen):
|
||||
raw = await ws.recv()
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
event = json.loads(raw)
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
events.append(event)
|
||||
event_type = event.get("type")
|
||||
if event_type == "error":
|
||||
raise AssertionError(f"WebSocket protocol smoke received an error: {event}")
|
||||
if isinstance(event_type, str):
|
||||
seen.add(event_type)
|
||||
return events
|
||||
|
||||
return await asyncio.wait_for(receive(), timeout=timeout_s)
|
||||
|
||||
|
||||
async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]:
|
||||
session_id = f"duplex-ci-protocol-{uuid.uuid4().hex}"
|
||||
websocket_url = build_realtime_url(url, model, autostart=False, session_id=session_id)
|
||||
async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": model,
|
||||
"modalities": ["audio", "text"],
|
||||
"ref_audio": _ref_audio_data_url(str(ref_audio)),
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
events = await _receive_protocol_events(
|
||||
ws,
|
||||
{"session.created", "session.updated"},
|
||||
timeout_s=60,
|
||||
)
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
events.extend(await _receive_protocol_events(ws, {"session.closed"}, timeout_s=60))
|
||||
return events
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None:
|
||||
ref_audio = resolve_ref_audio(model_prefix)
|
||||
events = asyncio.run(
|
||||
_run_protocol_smoke(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=ref_audio,
|
||||
)
|
||||
)
|
||||
event_types = [event.get("type") for event in events]
|
||||
assert "session.created" in event_types
|
||||
assert "session.updated" in event_types
|
||||
assert event_types[-1] == "session.closed"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
result = asyncio.run(
|
||||
run_demo(
|
||||
demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
)
|
||||
)
|
||||
assert result["ok"] is True
|
||||
assert result["audio_delta_count"] > 0
|
||||
assert result["done_count"] == 1
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
assert result["transcript_delta_done_ok"] is True
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
result = asyncio.run(
|
||||
run_multi_session(
|
||||
multi_session_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "multi_session",
|
||||
response_required=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
assert result["ok"] is True
|
||||
assert result["session_count"] == 2
|
||||
assert result["resume"]["ok"] is True
|
||||
assert result["takeover"]["ok"] is True
|
||||
assert not result["failures"]
|
||||
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
|
||||
assert all(session["done_count"] == 1 for session in result["sessions"])
|
||||
assert all(session["error_count"] == 0 for session in result["sessions"])
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Nightly lifecycle coverage for the MiniCPM-o 4.5 native-duplex API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
||||
run_lifecycle_probes,
|
||||
)
|
||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_soft_interrupt import (
|
||||
run_soft_interrupt,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
SOFT_INTERRUPT_SHA256,
|
||||
multi_session_args,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
validated_soft_interrupt_wav,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
args = multi_session_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "admission_expiry",
|
||||
response_required=False,
|
||||
)
|
||||
args.sessions = 1
|
||||
args.disconnect_session_index = None
|
||||
args.takeover_session_index = None
|
||||
args.expire_session_index = 0
|
||||
args.verify_admission_limit = 2
|
||||
result = asyncio.run(run_lifecycle_probes(args))
|
||||
|
||||
assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2)
|
||||
assert result["expiry"]["ok"] is True
|
||||
assert result["expiry"]["error_code"] == "session_resume_expired"
|
||||
assert result["admission"]["ok"] is True
|
||||
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
input_wav = validated_soft_interrupt_wav()
|
||||
result = asyncio.run(
|
||||
run_soft_interrupt(
|
||||
SimpleNamespace(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=str(input_wav),
|
||||
ref_audio=str(resolve_ref_audio(model_prefix)),
|
||||
output_dir=str(tmp_path / "soft_interrupt"),
|
||||
summary_output=None,
|
||||
chunk_ms=200,
|
||||
timeout_s=180.0,
|
||||
require_audio=True,
|
||||
no_realtime_pacing=False,
|
||||
validation_mode="response-required",
|
||||
min_responses=2,
|
||||
min_audio_deltas_per_response=2,
|
||||
input_sha256=SOFT_INTERRUPT_SHA256,
|
||||
expect_followup_response_substring=None,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2)
|
||||
assert result["error_count"] == 0
|
||||
assert result["response_lifecycle_ok"] is True
|
||||
assert result["response_audio_contract_ok"] is True
|
||||
assert result["followup_response_transcript_ok"] is True
|
||||
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
E2E Online expansion tests for MiniCPM-o 4.5 covering modality combinations,
|
||||
in-process token2wav vocoder behavior, max_num_seqs=1 serial execution,
|
||||
and edge-case validation.
|
||||
"""E2E online expansion tests for MiniCPM-o 4.5.
|
||||
|
||||
These cover modality combinations, separate Code2Wav behavior, sequential
|
||||
request isolation, longer audio output, and Chinese speech.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -94,10 +94,8 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Verify that sequential requests produce independent results and state
|
||||
does not leak between requests. MiniCPM-o 4.5 has max_num_seqs=1 so
|
||||
requests are processed one-at-a-time; the second request must not
|
||||
receive the first request's audio or token state.
|
||||
Verify that sequential requests produce independent results and that the
|
||||
second request does not receive the first request's audio or token state.
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text (two different prompts)
|
||||
Output Modal: text + audio (both)
|
||||
@@ -138,7 +136,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text input generating a longer audio output to exercise the
|
||||
token2wav decoder across multiple frames.
|
||||
Code2Wav stage across multiple frames.
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text (longer prompt)
|
||||
Output Modal: text + audio
|
||||
@@ -146,14 +144,14 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
content_text="Tell me a short story about a cat in about 50 words.",
|
||||
content_text="What is the capital of China? Answer in 40 words.",
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {"audio": ["cat"]},
|
||||
"key_words": {"audio": ["Beijing"]},
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user