391 lines
15 KiB
Python
391 lines
15 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""Minimal single-input MiniCPM-o 4.5 Realtime duplex demo.
|
|
|
|
Run this after starting the duplex server. Strict lifecycle, overlap, and
|
|
multi-session validation lives under ``tests/e2e/online_serving``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import base64
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
if str(REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
|
PCM16_BYTES_PER_SAMPLE, # noqa: F401 - compatibility export
|
|
PCM16_SAMPLE_RATE, # noqa: F401 - compatibility export
|
|
RealtimeDuplexClient,
|
|
RealtimeEventCollector,
|
|
build_realtime_url,
|
|
read_pcm16_wav,
|
|
wait_for,
|
|
write_pcm16_wav,
|
|
)
|
|
from vllm_omni.experimental.fullduplex.client import chunk_period_ms as _chunk_period_ms # noqa: E402
|
|
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
|
has_residual_model_unit as _has_residual_model_unit,
|
|
)
|
|
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
|
reference_audio_data_url as _ref_audio_data_url,
|
|
)
|
|
|
|
|
|
class _StreamingOutputWriter:
|
|
"""Persist and report output deltas as the client receives them."""
|
|
|
|
def __init__(self, output_dir: Path) -> None:
|
|
self.output_dir = output_dir
|
|
self.audio_chunk_dir = output_dir / "audio_chunks"
|
|
self.audio_chunk_paths: list[Path] = []
|
|
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
self.audio_chunk_dir.mkdir(parents=True, exist_ok=True)
|
|
(self.output_dir / "output.pcm").write_bytes(b"")
|
|
|
|
def handle(self, event: dict[str, object]) -> None:
|
|
event_type = event.get("type")
|
|
if event_type in {
|
|
"response.audio_transcript.delta",
|
|
"response.output_text.delta",
|
|
}:
|
|
delta = event.get("delta")
|
|
if isinstance(delta, str) and delta:
|
|
print(delta, end="", file=sys.stderr, flush=True)
|
|
return
|
|
if event_type != "response.audio.delta":
|
|
return
|
|
|
|
delta = event.get("delta") or event.get("audio")
|
|
if not isinstance(delta, str) or not delta:
|
|
return
|
|
try:
|
|
pcm16 = base64.b64decode(delta)
|
|
except ValueError:
|
|
return
|
|
if not pcm16:
|
|
return
|
|
|
|
chunk_index = len(self.audio_chunk_paths) + 1
|
|
chunk_path = self.audio_chunk_dir / f"chunk_{chunk_index:04d}.wav"
|
|
sample_rate_hz = event.get("sample_rate_hz")
|
|
if not isinstance(sample_rate_hz, int) or sample_rate_hz <= 0:
|
|
sample_rate_hz = 24_000
|
|
with (self.output_dir / "output.pcm").open("ab") as output_pcm:
|
|
output_pcm.write(pcm16)
|
|
write_pcm16_wav(chunk_path, pcm16, sample_rate_hz=sample_rate_hz)
|
|
self.audio_chunk_paths.append(chunk_path)
|
|
print(
|
|
f"\n[audio chunk {chunk_index}: {len(pcm16)} bytes -> {chunk_path}]",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
|
|
|
|
class _StreamingEventCollector(RealtimeEventCollector):
|
|
def __init__(self, writer: _StreamingOutputWriter) -> None:
|
|
super().__init__()
|
|
self._writer = writer
|
|
|
|
def add(self, event: dict[str, object], *, received_at_s: float | None = None) -> None:
|
|
super().add(event, received_at_s=received_at_s)
|
|
self._writer.handle(self.events[-1])
|
|
|
|
|
|
def _input_committed_index(
|
|
events: list[dict[str, object]],
|
|
after_index: int,
|
|
) -> int | None:
|
|
for index, event in enumerate(events[max(after_index, 0) :], start=max(after_index, 0)):
|
|
if event.get("type") == "input_audio_buffer.committed":
|
|
return index
|
|
return None
|
|
|
|
|
|
def _post_commit_model_decision(
|
|
events: list[dict[str, object]],
|
|
committed_index: int | None,
|
|
) -> str | None:
|
|
if committed_index is None:
|
|
return None
|
|
for event in events[committed_index + 1 :]:
|
|
event_type = event.get("type")
|
|
if event_type == "response.listen":
|
|
return "listen"
|
|
if event_type == "response.done":
|
|
response = event.get("response")
|
|
if not isinstance(response, dict) or response.get("status") != "cancelled":
|
|
return "speak"
|
|
return None
|
|
|
|
|
|
def _latest_model_decision(
|
|
events: list[dict[str, object]],
|
|
after_index: int,
|
|
) -> str | None:
|
|
decision: str | None = None
|
|
for event in events[max(after_index, 0) :]:
|
|
event_type = event.get("type")
|
|
if event_type == "response.listen":
|
|
decision = "listen"
|
|
elif event_type == "response.done":
|
|
response = event.get("response")
|
|
if not isinstance(response, dict) or response.get("status") != "cancelled":
|
|
decision = "speak"
|
|
return decision
|
|
|
|
|
|
def _response_in_progress(events: list[dict[str, object]]) -> bool:
|
|
return sum(event.get("type") == "response.created" for event in events) > sum(
|
|
event.get("type") == "response.done" for event in events
|
|
)
|
|
|
|
|
|
def _event_count_after(
|
|
events: list[dict[str, object]],
|
|
event_type: str,
|
|
index: int | None,
|
|
) -> int:
|
|
if index is None:
|
|
return 0
|
|
return sum(event.get("type") == event_type for event in events[index + 1 :])
|
|
|
|
|
|
async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
|
input_pcm16 = read_pcm16_wav(Path(args.input_wav))
|
|
if not input_pcm16:
|
|
raise ValueError("input WAV has no audio")
|
|
|
|
output_dir = Path(args.output_dir)
|
|
stream_writer = _StreamingOutputWriter(output_dir)
|
|
|
|
url = build_realtime_url(
|
|
args.url,
|
|
args.model,
|
|
autostart=False if args.ref_audio else None,
|
|
session_id=args.session_id,
|
|
)
|
|
client = RealtimeDuplexClient(url)
|
|
client.events = _StreamingEventCollector(stream_writer)
|
|
async with client:
|
|
await client.configure(
|
|
args.model,
|
|
ref_audio=_ref_audio_data_url(args.ref_audio),
|
|
session_id=args.session_id,
|
|
temperature=args.temperature,
|
|
timeout_s=args.timeout_s,
|
|
)
|
|
stream_event_cursor = len(client.events.events)
|
|
await client.stream_pcm16(
|
|
input_pcm16,
|
|
chunk_ms=args.chunk_ms,
|
|
realtime=not args.no_realtime_pacing,
|
|
)
|
|
commit_event_cursor = len(client.events.events)
|
|
stream_decision = _latest_model_decision(client.events.events, stream_event_cursor)
|
|
input_has_residual_model_unit = _has_residual_model_unit(
|
|
input_pcm16,
|
|
chunk_period_ms=_chunk_period_ms(client.events.events),
|
|
)
|
|
wait_for_post_commit_decision = False
|
|
commit_sent_at_s = time.monotonic()
|
|
await client.commit()
|
|
wait_error: str | None = None
|
|
committed_index: int | None = None
|
|
post_commit_decision: str | None = None
|
|
try:
|
|
await wait_for(
|
|
lambda: _input_committed_index(client.events.events, commit_event_cursor) is not None,
|
|
timeout_s=args.timeout_s,
|
|
label="input_audio_buffer.committed",
|
|
)
|
|
committed_index = _input_committed_index(client.events.events, commit_event_cursor)
|
|
stream_decision = _latest_model_decision(client.events.events[: committed_index + 1], stream_event_cursor)
|
|
wait_for_post_commit_decision = input_has_residual_model_unit or _response_in_progress(
|
|
client.events.events[: committed_index + 1]
|
|
)
|
|
if wait_for_post_commit_decision:
|
|
await wait_for(
|
|
lambda: _post_commit_model_decision(client.events.events, committed_index) is not None,
|
|
timeout_s=args.timeout_s,
|
|
label="post-commit model decision or response drain",
|
|
)
|
|
post_commit_decision = _post_commit_model_decision(client.events.events, committed_index)
|
|
except TimeoutError as exc:
|
|
wait_error = str(exc)
|
|
await client.acknowledge_playback()
|
|
close_error: str | None = None
|
|
try:
|
|
await client.close_session(timeout_s=args.timeout_s)
|
|
except TimeoutError as exc:
|
|
close_error = str(exc)
|
|
|
|
audio = client.events.audio_bytes()
|
|
first_text_at_s = client.events.first_received_at(
|
|
"response.audio_transcript.delta",
|
|
"response.output_text.delta",
|
|
after_s=commit_sent_at_s,
|
|
)
|
|
first_audio_at_s = client.events.first_received_at(
|
|
"response.audio.delta",
|
|
after_s=commit_sent_at_s,
|
|
)
|
|
response_created_at_s = client.events.first_received_at(
|
|
"response.created",
|
|
after_s=commit_sent_at_s,
|
|
)
|
|
response_done_at_s = client.events.first_received_at(
|
|
"response.done",
|
|
after_s=commit_sent_at_s,
|
|
)
|
|
audio_duration_s = len(audio) / (client.events.output_sample_rate_hz * 2)
|
|
response_generation_s = (
|
|
response_done_at_s - response_created_at_s
|
|
if response_done_at_s is not None and response_created_at_s is not None
|
|
else None
|
|
)
|
|
transcript_deltas = [
|
|
str(event.get("delta", ""))
|
|
for event in client.events.events
|
|
if event.get("type")
|
|
in {
|
|
"response.audio_transcript.delta",
|
|
"response.output_text.delta",
|
|
}
|
|
]
|
|
response_id = client.events.response_ids[0] if client.events.response_ids else None
|
|
timing = client.events.timing_summary(
|
|
after_s=commit_sent_at_s,
|
|
input_committed_at_s=commit_sent_at_s,
|
|
response_id=response_id,
|
|
)
|
|
errors = client.events.errors()
|
|
if wait_error:
|
|
errors.append({"type": "client.timeout", "message": wait_error})
|
|
if close_error:
|
|
errors.append({"type": "client.timeout", "message": close_error})
|
|
model_decision = post_commit_decision or stream_decision
|
|
(output_dir / "events.jsonl").write_text(
|
|
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events),
|
|
encoding="utf-8",
|
|
)
|
|
(output_dir / "output.pcm").write_bytes(audio)
|
|
if audio:
|
|
write_pcm16_wav(
|
|
output_dir / "output.wav",
|
|
audio,
|
|
sample_rate_hz=client.events.output_sample_rate_hz,
|
|
)
|
|
|
|
result = {
|
|
"ok": (
|
|
client.events.count("session.created") > 0
|
|
and client.events.count("session.closed") > 0
|
|
and not errors
|
|
and model_decision is not None
|
|
and (bool(audio) or not args.require_audio)
|
|
),
|
|
"model_decision": model_decision,
|
|
"post_commit": {
|
|
"input_committed_event_index": committed_index,
|
|
"decision": post_commit_decision,
|
|
"decision_required": wait_for_post_commit_decision,
|
|
"input_had_residual_model_unit": input_has_residual_model_unit,
|
|
"response_listen_count": _event_count_after(
|
|
client.events.events,
|
|
"response.listen",
|
|
committed_index,
|
|
),
|
|
"response_done_count": _event_count_after(
|
|
client.events.events,
|
|
"response.done",
|
|
committed_index,
|
|
),
|
|
},
|
|
"audio_bytes": len(audio),
|
|
"audio_chunk_count": len(stream_writer.audio_chunk_paths),
|
|
"audio_chunk_files": [str(path) for path in stream_writer.audio_chunk_paths],
|
|
"output_sample_rate_hz": client.events.output_sample_rate_hz,
|
|
"latency": {
|
|
"ttft_ms": (
|
|
round((first_text_at_s - commit_sent_at_s) * 1000, 2) if first_text_at_s is not None else None
|
|
),
|
|
"ttfp_ms": (
|
|
round((first_audio_at_s - commit_sent_at_s) * 1000, 2) if first_audio_at_s is not None else None
|
|
),
|
|
"rtf": (
|
|
round(response_generation_s / audio_duration_s, 4)
|
|
if response_generation_s is not None and audio_duration_s > 0
|
|
else None
|
|
),
|
|
"response_generation_ms": (
|
|
round(response_generation_s * 1000, 2) if response_generation_s is not None else None
|
|
),
|
|
"text_stream_ms": (
|
|
round((response_done_at_s - first_text_at_s) * 1000, 2)
|
|
if response_done_at_s is not None and first_text_at_s is not None
|
|
else None
|
|
),
|
|
"transcript_delta_count": len(transcript_deltas),
|
|
"audio_duration_s": round(audio_duration_s, 3),
|
|
"measurement_origin": "input_audio_buffer.commit send",
|
|
},
|
|
"timing": timing,
|
|
"response_ids": client.events.response_ids,
|
|
"transcript": "".join(transcript_deltas),
|
|
"errors": errors,
|
|
"output_dir": str(output_dir),
|
|
}
|
|
(output_dir / "result.json").write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
return result
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
|
|
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
|
parser.add_argument("--session-id")
|
|
parser.add_argument("--input-wav", required=True)
|
|
parser.add_argument(
|
|
"--ref-audio",
|
|
required=True,
|
|
help=(
|
|
"Reference WAV for the MiniCPM-o duplex assistant voice. "
|
|
"This demo matches the official flow by always providing a reference audio clip."
|
|
),
|
|
)
|
|
parser.add_argument("--output-dir", default="/tmp/minicpmo_realtime_duplex_demo")
|
|
parser.add_argument("--chunk-ms", type=int, default=200)
|
|
parser.add_argument("--timeout-s", type=float, default=60.0)
|
|
parser.add_argument(
|
|
"--temperature",
|
|
type=float,
|
|
default=None,
|
|
help="Stage0 sampling temperature; response-required soft-interrupt uses 0.0.",
|
|
)
|
|
parser.add_argument("--no-realtime-pacing", action="store_true")
|
|
parser.add_argument("--require-audio", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> None:
|
|
result = asyncio.run(run_demo(parse_args()))
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
if not result["ok"]:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|