anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Core][Frontend] Unified Full-duplex Framework (#7413)

Browse Source 
Signed-off-by: chickeyton <ngton2014@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Opus 5 authored and GitHub committed 2026-09-16 15:35:36 +08:00
1 parent 9005d78903
commit 99ff4f3070
182 files changed
+26951 -34743
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

examples/online_serving/barge_in_client.py
+143 -85
View File
@@ -14,7 +14,10 @@ Scenario (see ``barge_in_client_flow.md`` for the diagram)
The demo is model-neutral: it drives any duplex model through the public
:class:`vllm_omni.clients.duplex.DuplexClient` over
``/v1/realtime?duplex=1``. Model-specific session shape comes from the
``/v1/realtime?duplex=1``, or — with ``--inline`` — through
:class:`vllm_omni.clients.inline_duplex.InlineDuplexClient` on an in-process
:class:`vllm_omni.entrypoints.duplex_omni.DuplexOmni` (no server needed; the
model loads in this process). Model-specific session shape comes from the
per-model presets under ``vllm_omni.clients.<model>`` selected with
``--preset`` (default ``minicpmo_4_5``, whose model decides listen/speak on
its own — no VAD).
@@ -31,6 +34,15 @@ then:
--interrupt-wav follow_up_16k.wav \
--output-dir ./duplex_out
or, without a server, in-process (``--deploy-config`` is optional and defaults
to the model's bundled duplex profile):
python barge_in_client.py --inline \
--model /path/to/MiniCPM-o-4_5 \
--ref-audio /path/to/reference_voice.wav \
--question-wav question_16k.wav \
--interrupt-wav follow_up_16k.wav
Input WAVs must be mono 16 kHz PCM16; when a preset's session takes a
different capture format (the PersonaPlex preset streams 24 kHz float32),
the audio is converted before streaming. ``--ref-audio`` is required by the
@@ -55,6 +67,7 @@ if str(REPO_ROOT) not in sys.path:
from vllm_omni.clients.duplex import ( # noqa: E402
AudioFormat,
DuplexClient,
DuplexClientBase,
EventCollector,
SessionConfig,
acknowledge_collected_playback,
@@ -139,6 +152,39 @@ def _fold_responses(collector: EventCollector) -> dict[str, dict[str, object]]:
return out
def _make_client(args: argparse.Namespace, config: SessionConfig) -> DuplexClientBase:
"""Websocket client by default; ``--inline`` drives an in-process DuplexOmni."""
if not args.inline:
return DuplexClient(
args.url,
model=args.model,
config=config,
reconnect=None,
heartbeat_interval_s=None,
handshake_timeout_s=args.timeout_s,
)
# Imported lazily: the in-process path loads the model (and its engine
# dependencies) in this process, which the websocket path never needs.
from vllm_omni.clients.inline_duplex import InlineDuplexClient
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
# Models shipped with custom HF code (MiniCPM-o) need trust_remote_code to
# resolve their pipeline; harmless for the others.
omni_kwargs: dict[str, object] = {"model": args.model, "trust_remote_code": True}
if args.deploy_config:
omni_kwargs["deploy_config"] = args.deploy_config
omni = DuplexOmni(**omni_kwargs)
client = InlineDuplexClient(
omni,
model=args.model,
config=config,
handshake_timeout_s=args.timeout_s,
)
# The client owns no engine; stop the in-process one when the run ends.
client.owned_omni = omni # type: ignore[attr-defined]
return client
async def run(args: argparse.Namespace) -> int:
config = _session_config(args.preset, ref_audio_path=args.ref_audio)
question = _convert_to_session_format(read_pcm16_wav(Path(args.question_wav)), config.input_audio)
@@ -148,105 +194,117 @@ async def run(args: argparse.Namespace) -> int:
output_dir.mkdir(parents=True, exist_ok=True)
collector = EventCollector()
client = DuplexClient(
args.url,
model=args.model,
config=config,
reconnect=None,
heartbeat_interval_s=None,
handshake_timeout_s=args.timeout_s,
)
client = _make_client(args, config)
# GREET: open the voice session (handshake happens on enter).
async with client:
consume_task = asyncio.create_task(collector.consume(client))
try:
async with client:
consume_task = asyncio.create_task(collector.consume(client))
# ASK: stream the question at realtime pace, then commit the turn.
# The model decides on its own to answer (no VAD).
print("[1] streaming question ...", file=sys.stderr)
await client.stream_pcm(question, chunk_ms=chunk_ms)
await client.commit(create_response=False)
await wait_for_condition(
lambda: collector.count("response.created") > 0,
timeout_s=args.timeout_s,
label="first response.created",
)
print("[1] assistant is answering", file=sys.stderr)
# ASK: stream the question at realtime pace, then commit the turn.
# The model decides on its own to answer (no VAD).
print("[1] streaming question ...", file=sys.stderr)
await client.stream_pcm(question, chunk_ms=chunk_ms)
await client.commit(create_response=False)
await wait_for_condition(
lambda: collector.count("response.created") > 0,
timeout_s=args.timeout_s,
label="first response.created",
)
print("[1] assistant is answering", file=sys.stderr)
# Let the answer play for a moment, as a listening user would.
await asyncio.sleep(args.listen_s)
# Let the answer play for a moment, as a listening user would.
await asyncio.sleep(args.listen_s)
# INTERRUPT: talk over the assistant while its audio still streams.
# The serving overlap policy decides: enough overlapped speech barges
# in (the first response is cancelled) or a short remark is deferred
# until the model yields the turn.
print("[2] interrupting mid-answer ...", file=sys.stderr)
ids_before_interrupt = set(collector.response_ids)
await client.stream_pcm(interrupt, chunk_ms=chunk_ms)
await client.commit(create_response=False)
# INTERRUPT: talk over the assistant while its audio still streams.
# The serving overlap policy decides: enough overlapped speech barges
# in (the first response is cancelled) or a short remark is deferred
# until the model yields the turn.
print("[2] interrupting mid-answer ...", file=sys.stderr)
ids_before_interrupt = set(collector.response_ids)
await client.stream_pcm(interrupt, chunk_ms=chunk_ms)
await client.commit(create_response=False)
# ANSWER AGAIN: wait for a response created *after* the interruption
# to finish. The first answer may still be streaming here; matching on
# any response.done would let its completion satisfy the wait and cut
# off the actual follow-up answer at close.
def follow_up_answer_done() -> bool:
return any(
event.get("type") == "response.done"
and collector.response_id(event) is not None
and collector.response_id(event) not in ids_before_interrupt
for event in collector.events
# ANSWER AGAIN: wait for a response created *after* the interruption
# to finish. The first answer may still be streaming here; matching on
# any response.done would let its completion satisfy the wait and cut
# off the actual follow-up answer at close.
def follow_up_answer_done() -> bool:
return any(
event.get("type") == "response.done"
and collector.response_id(event) is not None
and collector.response_id(event) not in ids_before_interrupt
for event in collector.events
)
await wait_for_condition(
follow_up_answer_done,
timeout_s=args.timeout_s,
label="response.done for the response answering the interruption",
)
await wait_for_condition(
follow_up_answer_done,
timeout_s=args.timeout_s,
label="response.done for the response answering the interruption",
)
# HANG UP: report playback, close the session cleanly.
await acknowledge_collected_playback(client, collector)
await client.close(timeout_s=args.timeout_s)
try:
await asyncio.wait_for(consume_task, timeout=5.0)
except (asyncio.TimeoutError, asyncio.CancelledError):
consume_task.cancel()
# HANG UP: report playback, close the session cleanly.
await acknowledge_collected_playback(client, collector)
await client.close(timeout_s=args.timeout_s)
try:
await asyncio.wait_for(consume_task, timeout=5.0)
except (asyncio.TimeoutError, asyncio.CancelledError):
consume_task.cancel()
# Report: one WAV per response; statuses show what the overlap did.
responses = _fold_responses(collector)
summary: list[dict[str, object]] = []
for index, (response_id, item) in enumerate(responses.items(), start=1):
audio = item["audio"]
assert isinstance(audio, bytes)
sample_rate_hz = int(item["sample_rate_hz"]) # type: ignore[arg-type]
wav_path = output_dir / f"response_{index}_{item['status']}.wav"
if audio:
write_pcm16_wav(wav_path, audio, sample_rate_hz=sample_rate_hz)
summary.append(
{
"response_id": response_id,
"status": item["status"],
"audio_s": round(len(audio) / (sample_rate_hz * 2), 2),
"text": item["text"],
"wav": wav_path.name if audio else None,
}
# Report: one WAV per response; statuses show what the overlap did.
responses = _fold_responses(collector)
summary: list[dict[str, object]] = []
for index, (response_id, item) in enumerate(responses.items(), start=1):
audio = item["audio"]
assert isinstance(audio, bytes)
sample_rate_hz = int(item["sample_rate_hz"]) # type: ignore[arg-type]
wav_path = output_dir / f"response_{index}_{item['status']}.wav"
if audio:
write_pcm16_wav(wav_path, audio, sample_rate_hz=sample_rate_hz)
summary.append(
{
"response_id": response_id,
"status": item["status"],
"audio_s": round(len(audio) / (sample_rate_hz * 2), 2),
"text": item["text"],
"wav": wav_path.name if audio else None,
}
)
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False))
(output_dir / "events.jsonl").write_text(
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in collector.events),
encoding="utf-8",
)
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False))
(output_dir / "events.jsonl").write_text(
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in collector.events),
encoding="utf-8",
)
print(json.dumps(summary, indent=2, ensure_ascii=False))
cancelled = sum(1 for item in summary if item["status"] == "cancelled")
print(
f"\n{len(summary)} responses ({cancelled} interrupted); outputs in {output_dir}",
file=sys.stderr,
)
print(json.dumps(summary, indent=2, ensure_ascii=False))
cancelled = sum(1 for item in summary if item["status"] == "cancelled")
print(
f"\n{len(summary)} responses ({cancelled} interrupted); outputs in {output_dir}",
file=sys.stderr,
)
finally:
owned_omni = getattr(client, "owned_omni", None)
if owned_omni is not None:
owned_omni.shutdown()
return 0
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime")
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
parser.add_argument(
"--model",
default="openbmb/MiniCPM-o-4_5",
help="served model name, or the model path with --inline",
)
parser.add_argument(
"--inline",
action="store_true",
help="run the model in this process through DuplexOmni + InlineDuplexClient instead of connecting to --url",
)
parser.add_argument(
"--deploy-config",
default=None,
help="deploy YAML for --inline (default: the model's bundled profile)",
)
parser.add_argument(
"--preset",
choices=["minicpmo_4_5", "personaplex", "none"],
examples/online_serving/barge_in_client_flow.md
+9 -6
View File
@@ -1,8 +1,10 @@
# `barge_in_client.py` — what happens, at a glance
The demo plays one everyday conversation pattern against the full-duplex
server: ask something, get talked-over mid-answer, and see the assistant
cope. High-level flow below — for the exact wire events, read the
The demo plays one everyday conversation pattern against a full-duplex
model: ask something, get talked-over mid-answer, and see the assistant
cope. It runs either against a server over `/v1/realtime?duplex=1` or, with
`--inline`, against an in-process `DuplexOmni` — the conversation below is the
same in both modes. High-level flow — for the exact wire events, read the
docstring in `barge_in_client.py`.
```mermaid
@@ -48,7 +50,8 @@ answer is saved as `response_1_cancelled.wav`, a completed one as
duration per answer.
Companion: the runtime architecture lives in
[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client
API is `vllm_omni.clients.duplex.DuplexClient`, documented with the wire
protocol in
[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client API is
`vllm_omni.clients.duplex.DuplexClient` over the websocket and
`vllm_omni.clients.inline_duplex.InlineDuplexClient` in-process — both are
documented with the wire protocol in
[`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md).
examples/online_serving/minicpmo/barge_in_serve.sh
+5 -5
View File
@@ -5,10 +5,10 @@
# Server side for barge_in_client.py: start MiniCPM-o 4.5 in native
# full-duplex mode.
#
# The duplex deploy profile registers the WS /v1/duplex endpoint (with
# /v1/realtime?duplex=1 projecting onto the same handler), enables the
# engine duplex control plane, and bounds the Thinker/Talker/Code2Wav
# stages to four live duplex sessions on one GPU. See
# MiniCPM-o 4.5 declares a duplex plugin, so vllm-omni serve runs it through
# DuplexOmni: the server is duplex-only (WS /v1/realtime?duplex=1, alias
# /v1/duplex, plus /v1/models and /health) and bounds the
# Thinker/Talker/Code2Wav stages to four live duplex sessions on one GPU. See
# vllm_omni/deploy/minicpmo_4_5.yaml (session_mode: duplex) for the session
# limits; every other duplex_session knob keeps its runtime default
# (idle TTL 300s, disconnect grace 30s, 16MiB pending input per session).
@@ -22,4 +22,4 @@ exec vllm-omni serve "$MODEL" \
--deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \
--trust-remote-code \
--host 0.0.0.0 --port "$PORT"
# duplex endpoints: ws://<host>:$PORT/v1/duplex and /v1/realtime?duplex=1
# duplex endpoint: ws://<host>:$PORT/v1/realtime?duplex=1 (alias /v1/duplex)
examples/online_serving/minicpmo/realtime_duplex_demo.py
-2
View File
@@ -445,7 +445,6 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
ref_audio=_ref_audio_data_url(args.ref_audio),
temperature=args.temperature,
),
session_id=args.session_id,
reconnect=None,
heartbeat_interval_s=None,
handshake_timeout_s=args.timeout_s,
@@ -638,7 +637,6 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
parser.add_argument("--session-id")
parser.add_argument(
"--input-wav",
help="Mono 16 kHz PCM16 clip to stream. Optional when --input-video carries the audio track.",
examples/online_serving/minicpmo/realtime_web/app/static/app.js
-2
View File
@@ -86,7 +86,6 @@
url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
url.searchParams.set('duplex', '1');
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
url.searchParams.set('native_duplex', '1');
url.searchParams.set('autostart', '0');
return url.toString();
}
@@ -458,7 +457,6 @@
settled = true;
const extraBody = {
auto_response: true,
native_duplex: true,
};
const session = {
modalities: ['audio', 'text'],