[Model] Add native full-duplex Nemotron VoiceChat serving (#6089)

Signed-off-by: Sysy <1370724210@qq.com>
Signed-off-by: sy03 <137072421@qq.com>
Co-authored-by: sy03 <137072421@qq.com>
This commit is contained in:
Sy03andsy03 authored and GitHub committed 2026-08-26 13:19:20 -07:00
1 parent c2723979c0
commit 432b2c6ca3
40 files changed
+3560 -154

No files matched your search

@@ -0,0 +1,94 @@
from __future__ import annotations
from types import SimpleNamespace
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import (
NemotronVoiceChatDataPlaneContext,
NemotronVoiceChatDataPlaneSession,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_RUNTIME = {
"nvc_text_bos_id": 0,
"nvc_text_eos_id": 1,
"nvc_text_pad_id": 12,
"nvc_function_sotc_id": 20,
"nvc_function_eotc_id": 21,
"nvc_tokenizer_ref": "test-tokenizer",
}
def _projector(encode_audio=lambda *_: None) -> NemotronVoiceChatDataPlaneSession:
projector = NemotronVoiceChatDataPlaneSession(encode_audio)
projector.configure_runtime(_RUNTIME, tokenizer=SimpleNamespace(decode=lambda *_args, **_kwargs: "text"))
return projector
def _stage0_output(request_id: str = "req-0", *, text_token: int = 12, function_token: int | None = None) -> object:
metadata: dict[str, object] = {"nvc_text_token_ids": [text_token]}
if function_token is not None:
metadata["nvc_function_token"] = [function_token]
completion = SimpleNamespace(multimodal_output=metadata)
return SimpleNamespace(
stage_id=0,
request_output=SimpleNamespace(request_id=request_id, outputs=[completion]),
)
def test_function_channel_projects_completed_call_without_ending_speech() -> None:
projector = _projector()
projector._decode = lambda token_ids: (
'[{"name":"weather","arguments":{"city":"Shanghai"}}]' if token_ids == [99] else ""
)
context = NemotronVoiceChatDataPlaneContext(epoch=0)
events = []
for function_token in (20, 99, 21):
events.extend(projector.project_output(_stage0_output(function_token=function_token), context=context))
listen = [event for event in events if event.get("is_listen") is True]
function = [event for event in events if event.get("function_call") is True]
assert len(listen) == 3
assert len(function) == 1
assert function[0]["name"] == "weather"
assert function[0]["arguments"] == '{"city":"Shanghai"}'
projector._decode = lambda _token_ids: "not-json"
bad_events = []
for function_token in (20, 98, 21):
bad_events.extend(projector.project_output(_stage0_output(function_token=function_token)))
assert any(event.get("error_code") == "nemotron_function_call_parse_error" for event in bad_events)
def test_new_epoch_request_does_not_inherit_partial_function_or_frame_state() -> None:
projector = _projector(lambda *_: "audio")
old_request = duplex_resource_request_id(DuplexFence("sid", epoch=1), "stage0")
new_request = duplex_resource_request_id(DuplexFence("sid", epoch=2), "stage0")
projector.begin_request(old_request)
list(projector.project_output(_stage0_output(old_request, text_token=42, function_token=20)))
projector.begin_request(new_request)
# EOTC in the new epoch must not close the old epoch's partial function.
new_events = list(projector.project_output(_stage0_output(new_request, text_token=1, function_token=21)))
assert not [event for event in new_events if event.get("function_call") is True]
assert not [event for event in new_events if event.get("end_of_turn") is True]
audio = SimpleNamespace(
stage_id=2,
request_output=SimpleNamespace(
request_id=new_request,
outputs=[
SimpleNamespace(multimodal_output={"model_outputs": [np.ones(1764, dtype=np.float32)], "sr": [22050]})
],
),
)
audio_events = list(projector.project_output(audio))
assert sum(event.get("end_of_turn") is True for event in audio_events) == 1
@@ -0,0 +1,73 @@
from __future__ import annotations
import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import (
NemotronVoiceChatPcmAppendBuffer,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _payload(samples: np.ndarray, *, sample_rate_hz: int = 16000, fmt: str = "pcm_f32le") -> dict[str, object]:
return {
"type": "audio",
"audio": base64.b64encode(np.asarray(samples, dtype=np.float32).tobytes()).decode("ascii"),
"format": fmt,
"sample_rate_hz": sample_rate_hz,
}
def _append(buffer, samples, operation_id):
return buffer.prepare_append(
_payload(np.asarray(samples, dtype=np.float32)),
operation_id=operation_id,
chunk_period_ms=80,
allow_emit=True,
)
def test_irregular_browser_packets_emit_exact_ordered_80ms_frame() -> None:
buffer = NemotronVoiceChatPcmAppendBuffer()
assert _append(buffer, np.arange(1000), "packet-1") is None
reservation = _append(buffer, np.arange(1000, 1280), "packet-2")
assert reservation is not None
assert reservation.operation_id == "packet-2"
decoded = np.frombuffer(base64.b64decode(reservation.payload["audio"]), dtype=np.float32)
np.testing.assert_array_equal(decoded, np.arange(1280, dtype=np.float32))
assert buffer.pending_byte_count == 0
def test_append_rejects_invalid_frame_contract() -> None:
cases = (
(1281, 16000, "pcm_f32le", 0.0, "at most 1280"),
(1, 8000, "pcm_f32le", 0.0, "sample_rate_hz"),
(1, 16000, "audio/pcm", 0.0, "format"),
(1, 16000, "pcm_f32le", np.nan, "finite"),
)
for samples, sample_rate_hz, fmt, value, message in cases:
with pytest.raises(ValueError, match=message):
NemotronVoiceChatPcmAppendBuffer().prepare_append(
_payload(np.full(samples, value, dtype=np.float32), sample_rate_hz=sample_rate_hz, fmt=fmt),
operation_id="invalid",
chunk_period_ms=80,
allow_emit=True,
)
def test_terminal_commit_rejects_more_than_one_frame_tail() -> None:
buffer = NemotronVoiceChatPcmAppendBuffer()
for index, samples in enumerate((1000, 281)):
buffer.prepare_append(
_payload(np.zeros(samples, dtype=np.float32)),
operation_id=f"tail-{index}",
chunk_period_ms=80,
allow_emit=False,
)
with pytest.raises(ValueError, match="more than one 80 ms frame"):
buffer.prepare_commit(operation_id="commit", chunk_period_ms=80)
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import base64
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.sampling_params import SamplingParams
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import (
NemotronVoiceChatDuplexRuntimeExtension,
)
from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import (
NemotronVoiceChatServingRuntimeAdapter,
_render_tool_response,
)
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
NemotronVoiceChatThinkerForConditionalGeneration,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _frame() -> dict[str, object]:
raw = np.zeros(1280, dtype=np.float32).tobytes()
return {
"type": "audio",
"audio": base64.b64encode(raw).decode("ascii"),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
def _plan(extension, *, input_seq: int):
return extension.plan_append(
request_id="req",
fence=DuplexFence("sid", incarnation=2, epoch=3),
session_config={},
runtime_config={
"nvc_prompt_token_ids": [0, 42, 1],
"nvc_text_pad_id": 12,
"nvc_max_model_len": 8192,
},
seq=input_seq,
turn_seq=input_seq,
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
payload=_frame(),
final=False,
sampling_params=SamplingParams(),
)
def test_first_append_prefills_prompt_then_each_append_consumes_one_frame() -> None:
extension = NemotronVoiceChatDuplexRuntimeExtension()
params = extension.configure_sampling_params(
runtime_config={}, defaults=(SamplingParams(temperature=0.7, max_tokens=4),)
)[0]
assert params.temperature == 0.0 and params.max_tokens == 1 and params.top_p == 1.0 and params.top_k == 0
first = _plan(extension, input_seq=1)
later = _plan(extension, input_seq=2)
assert first.prompt["prompt_token_ids"] == [0, 42, 1, 12]
assert later.prompt["prompt_token_ids"] == [12]
assert first.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 1
assert later.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 2
def test_append_rejects_stage0_context_overflow() -> None:
with pytest.raises(ValueError, match="max_model_len"):
_plan(NemotronVoiceChatDuplexRuntimeExtension(), input_seq=8190)
def test_function_output_becomes_versioned_nvidia_channel_tokens() -> None:
encoded: list[str] = []
adapter = NemotronVoiceChatServingRuntimeAdapter(lambda *_: None)
adapter._tokenizer = SimpleNamespace(
encode=lambda text, **_kwargs: encoded.append(text) or [31, 32, 33],
)
first = adapter.runtime_config_for_function_output(
{"type": "function_call_output", "call_id": "call-1", "output": '{"result":20}'},
{},
)
second = adapter.runtime_config_for_function_output(
{"type": "function_call_output", "call_id": "call-2", "output": "plain text"},
first,
)
assert _render_tool_response('{"result":20}') == '<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>'
assert encoded == [
'<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>',
'<TOOL_RESPONSE>["plain text"]</TOOL_RESPONSE>',
]
assert first["nvc_function_response_generation"] == 1
assert second["nvc_function_response_generation"] == 2
assert second["nvc_function_response_token_ids"] == [31, 32, 33]
assert second["nvc_function_response_call_id"] == "call-2"
assert second["nvc_function_response_batches"] == [
{"generation": 1, "call_id": "call-1", "token_ids": [31, 32, 33]},
{"generation": 2, "call_id": "call-2", "token_ids": [31, 32, 33]},
]
session = {"function_response_generation": 0, "forced_function_tokens": []}
NemotronVoiceChatThinkerForConditionalGeneration._sync_forced_function_response(session, second)
assert session["function_response_generation"] == 2
assert session["forced_function_token"] == 31
assert session["forced_function_tokens"] == [31, 32, 33, 31, 32, 33]
@@ -0,0 +1,364 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import argparse
import asyncio
import base64
import json
import math
import uuid
import wave
from collections.abc import Sequence
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import numpy as np
from scipy.signal import resample_poly
from vllm_omni.experimental.fullduplex.client import RealtimeDuplexClient, wait_for, write_pcm16_wav
INPUT_SAMPLE_RATE_HZ = 16_000
OUTPUT_SAMPLE_RATE_HZ = 22_050
FRAME_SAMPLES = 1_280
FRAME_PERIOD_S = FRAME_SAMPLES / INPUT_SAMPLE_RATE_HZ
DEFAULT_FUNCTION_TOOLS = [
{
"type": "function",
"name": "generate_random_number",
"description": "Generate a random integer between min and max (inclusive).",
"parameters": {
"type": "object",
"properties": {
"min": {"type": "integer", "description": "Minimum value (inclusive)"},
"max": {"type": "integer", "description": "Maximum value (inclusive)"},
},
"required": ["min", "max"],
},
}
]
DEFAULT_INSTRUCTIONS = "You are NVIDIA Voice Chat. Answer briefly. Start by greeting the user."
DEFAULT_FUNCTION_INSTRUCTIONS = (
"You are NVIDIA Voice Chat. If the user's request matches an available tool, "
"you MUST call that tool instead of answering from your own knowledge. "
"Use only argument values spoken by the user and never invent missing values."
)
def _url(base_url: str, model: str, session_id: str) -> str:
parts = urlsplit(base_url)
query = dict(parse_qsl(parts.query, keep_blank_values=True))
query.update(duplex="1", model=model, autostart="0", session_id=session_id)
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
def _read_wav(path: Path, *, input_channel: int = 0) -> np.ndarray:
with wave.open(str(path), "rb") as wav_file:
channels = wav_file.getnchannels()
width = wav_file.getsampwidth()
source_rate = wav_file.getframerate()
raw = wav_file.readframes(wav_file.getnframes())
dtypes = {1: np.uint8, 2: np.dtype("<i2"), 4: np.dtype("<i4")}
if width not in dtypes:
raise ValueError(f"unsupported input sample width: {width}")
pcm = np.frombuffer(raw, dtype=dtypes[width]).astype(np.float32)
pcm = (pcm - 128.0) / 128.0 if width == 1 else pcm / float(1 << (width * 8 - 1))
if not 0 <= input_channel < channels:
raise ValueError(f"input channel {input_channel} is outside WAV channel count {channels}")
if channels > 1:
pcm = pcm.reshape(-1, channels)[:, input_channel]
if source_rate != INPUT_SAMPLE_RATE_HZ:
divisor = math.gcd(source_rate, INPUT_SAMPLE_RATE_HZ)
pcm = resample_poly(pcm, up=INPUT_SAMPLE_RATE_HZ // divisor, down=source_rate // divisor)
return np.ascontiguousarray(pcm, dtype="<f4")
async def _stream(client: RealtimeDuplexClient, pcm: np.ndarray, *, max_frames: int | None, realtime: bool) -> int:
count = math.ceil(pcm.size / FRAME_SAMPLES)
if max_frames is not None:
count = min(count, max_frames)
for seq in range(count):
frame = pcm[seq * FRAME_SAMPLES : (seq + 1) * FRAME_SAMPLES]
frame = np.pad(frame, (0, FRAME_SAMPLES - frame.size)).astype("<f4")
await client.send(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(frame).decode("ascii"),
"format": "pcm_f32le",
"sample_rate_hz": INPUT_SAMPLE_RATE_HZ,
"duration_ms": 80,
"audio_end_ms": (seq + 1) * 80,
}
)
if realtime:
await asyncio.sleep(FRAME_PERIOD_S)
return count
def _events(client: RealtimeDuplexClient, event_type: str) -> list[dict[str, object]]:
return [event for event in client.events.events if event.get("type") == event_type]
async def _return_function_output_when_ready(
client: RealtimeDuplexClient,
*,
output: str,
timeout_s: float,
) -> tuple[str, int]:
await wait_for(
lambda: bool(client.events.errors()) or client.events.count("response.function_call_arguments.done") > 0,
timeout_s=timeout_s,
label="function call to execute",
)
if client.events.errors():
raise AssertionError(f"function call failed before tool execution: {client.events.errors()}")
function_done = _events(client, "response.function_call_arguments.done")[-1]
call_id = function_done.get("call_id")
if not isinstance(call_id, str) or not call_id:
raise AssertionError(f"completed function call has no call_id: {function_done}")
event_count_before_output = len(client.events.events)
await client.send(
{
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": output,
},
}
)
return call_id, event_count_before_output
def _write_events(path: Path, client: RealtimeDuplexClient) -> None:
path.write_text(
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events),
encoding="utf-8",
)
@asynccontextmanager
async def _managed_client(client: RealtimeDuplexClient, *, timeout_s: float):
async with client:
try:
yield
finally:
if client.events.count("session.created") and not client.events.count("session.closed"):
with suppress(Exception):
await client.close_session(timeout_s=min(timeout_s, 30.0))
async def run(args: argparse.Namespace) -> dict[str, object]:
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
if args.expect_function_call and args.instructions == DEFAULT_INSTRUCTIONS:
instructions = DEFAULT_FUNCTION_INSTRUCTIONS
else:
instructions = args.instructions
tools = DEFAULT_FUNCTION_TOOLS if args.expect_function_call else None
session_id = f"nemotron-voicechat-{uuid.uuid4().hex}"
client = RealtimeDuplexClient(_url(args.url, args.model, session_id))
async with _managed_client(client, timeout_s=args.timeout_s):
session_payload: dict[str, object] = {
"session_id": session_id,
"model": args.model,
"modalities": ["audio", "text"],
"input_audio_format": "pcm_f32le",
"output_audio_format": "pcm16",
"instructions": instructions,
"idle_timeout_s": args.timeout_s,
"turn_detection": None,
"extra_body": {"auto_response": True},
}
if tools is not None:
session_payload["tools"] = tools
await client.send({"type": "session.update", "session": session_payload})
await wait_for(
lambda: client.events.count("session.created") > 0 or bool(client.events.errors()),
timeout_s=args.timeout_s,
label="session.created",
)
if client.events.errors():
raise AssertionError(f"session setup failed: {client.events.errors()}")
created = _events(client, "session.created")[-1]
session = created.get("session")
capabilities = session.get("capabilities") if isinstance(session, dict) else None
expected = dict(
implementation_level="model_native_duplex",
chunk_period_ms=80,
supports_core_resumable_request=True,
supports_core_kv_lease=False,
supports_multi_session=False,
)
if not isinstance(capabilities, dict) or any(capabilities.get(key) != value for key, value in expected.items()):
raise AssertionError(f"unexpected capabilities: {capabilities}")
function_output_task = (
asyncio.create_task(
_return_function_output_when_ready(
client,
output=args.function_output,
timeout_s=args.timeout_s,
)
)
if args.function_output is not None
else None
)
pcm = _read_wav(Path(args.input_wav), input_channel=args.input_channel)
frame_count = await _stream(client, pcm, max_frames=args.max_frames, realtime=not args.no_realtime)
completed_responses_at_commit = client.events.count("response.done")
await client.send({"type": "input_audio_buffer.commit", "final": True})
await wait_for(
lambda: (
bool(client.events.errors())
or (
client.events.count("response.function_call_arguments.done") > 0
if args.expect_function_call
else (
client.events.count("response.audio.delta") >= args.minimum_audio_chunks
and (client.events.count("response.done") > completed_responses_at_commit)
)
)
),
timeout_s=args.timeout_s,
label="model output",
)
await asyncio.sleep(args.drain_s)
if client.events.errors():
raise AssertionError(f"Realtime session emitted errors: {client.events.errors()}")
done_events = _events(client, "response.done")
if not args.expect_function_call:
response = done_events[-1].get("response") if done_events else None
status = response.get("status") if isinstance(response, dict) else None
if status != "completed":
raise AssertionError(f"response did not complete successfully: {done_events[-1:]}")
function_events = [
event for event in client.events.events if str(event.get("type", "")).startswith("response.function_call")
]
function_items = [
event
for event in _events(client, "response.output_item.done")
if isinstance(event.get("item"), dict) and event["item"].get("type") == "function_call"
]
if args.expect_function_call and not any(
event.get("type") == "response.function_call_arguments.done" for event in function_events
):
raise AssertionError(f"no completed function call: {function_events}")
if args.expect_function_call:
matching_items = [
event for event in function_items if event["item"].get("name") == args.expected_function_name
]
if not matching_items:
raise AssertionError(f"expected {args.expected_function_name!r}, got {function_items}")
function_item = matching_items[-1]["item"]
try:
function_arguments = json.loads(str(function_item.get("arguments", "")))
except json.JSONDecodeError as exc:
raise AssertionError(f"function arguments are not JSON: {function_item}") from exc
if args.expected_function_arguments is not None:
expected_arguments = json.loads(args.expected_function_arguments)
if function_arguments != expected_arguments:
raise AssertionError(
f"function arguments differ: expected={expected_arguments!r}, actual={function_arguments!r}"
)
if args.function_output is not None:
call_id = function_item.get("call_id")
if not isinstance(call_id, str) or not call_id:
raise AssertionError(f"function item has no call_id: {function_item}")
assert function_output_task is not None
returned_call_id, event_count_before_output = await function_output_task
assert returned_call_id == call_id
def tool_result_completed() -> bool:
later = client.events.events[event_count_before_output:]
transcript = "".join(
str(event.get("delta", ""))
for event in later
if event.get("type") == "response.audio_transcript.delta"
).lower()
return (
any(event.get("type") == "response.audio.delta" for event in later)
and any(event.get("type") == "response.done" for event in later)
and (args.expected_post_tool_text is None or args.expected_post_tool_text.lower() in transcript)
)
await wait_for(
lambda: bool(client.events.errors()) or tool_result_completed(),
timeout_s=args.timeout_s,
label="completed response after function output",
)
if client.events.errors():
raise AssertionError(f"function output failed: {client.events.errors()}")
await asyncio.sleep(args.drain_s)
audio = client.events.audio_bytes()
audio_events = _events(client, "response.audio.delta")
rates = {event.get("sample_rate_hz") for event in audio_events}
if not args.expect_function_call and audio and rates != {OUTPUT_SAMPLE_RATE_HZ}:
raise AssertionError(f"unexpected output sample rates: {rates}")
if not args.expect_function_call and args.minimum_audio_chunks and not audio:
raise AssertionError("model produced no audio")
expected_bytes = 2 * OUTPUT_SAMPLE_RATE_HZ * expected["chunk_period_ms"] // 1000
packet_sizes = [len(base64.b64decode(str(event.get("delta", "")), validate=True)) for event in audio_events]
if not args.expect_function_call and any(size != expected_bytes for size in packet_sizes):
raise AssertionError(f"audio deltas are not fixed 80 ms PCM16 packets: {packet_sizes}")
audio_pcm = np.frombuffer(audio, dtype="<i2").astype(np.float32) / 32768.0
audio_rms = float(np.sqrt(np.mean(np.square(audio_pcm)))) if audio_pcm.size else 0.0
if not args.expect_function_call and audio and audio_rms < args.minimum_audio_rms:
raise AssertionError(
f"model output RMS {audio_rms:.6f} is below {args.minimum_audio_rms:.6f}; "
"received packets contain only silence"
)
_write_events(output_dir / "events.jsonl", client)
if audio:
write_pcm16_wav(output_dir / "output.wav", audio, sample_rate_hz=OUTPUT_SAMPLE_RATE_HZ)
result = {
"ok": True,
"session_id": session_id,
"input_frames": frame_count,
"capabilities": capabilities,
"event_counts": {
event_type: client.events.count(event_type)
for event_type in sorted({str(event.get("type")) for event in client.events.events})
},
"audio_bytes": len(audio),
"audio_rms": audio_rms,
"function_events": function_events,
"function_items": function_items,
"output_dir": str(output_dir),
}
(output_dir / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
return result
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--url", default="ws://127.0.0.1:8125/v1/realtime")
parser.add_argument("--model", required=True)
parser.add_argument("--input-wav", required=True)
parser.add_argument("--input-channel", type=int, default=0)
parser.add_argument("--output-dir", default="/tmp/nemotron-voicechat-duplex")
parser.add_argument("--instructions", default=DEFAULT_INSTRUCTIONS)
parser.add_argument("--max-frames", type=int)
parser.add_argument("--minimum-audio-chunks", type=int, default=1)
parser.add_argument("--minimum-audio-rms", type=float, default=1e-4)
parser.add_argument("--expect-function-call", action="store_true")
parser.add_argument("--expected-function-name", default="generate_random_number")
parser.add_argument("--expected-function-arguments")
parser.add_argument("--function-output")
parser.add_argument("--expected-post-tool-text")
parser.add_argument("--no-realtime", action="store_true")
parser.add_argument("--drain-s", type=float, default=2.0)
parser.add_argument("--timeout-s", type=float, default=600.0)
return parser.parse_args(argv)
def main() -> None:
print(json.dumps(asyncio.run(run(parse_args())), ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
@@ -52,9 +52,8 @@ def get_prompt(prompt_type="zh"):
def _set_rocm_request_timeout(request_config: dict) -> None:
if current_omni_platform.is_rocm():
# A cold AITER MHA kernel build took 301 seconds on MI300X.
# Allow 600 seconds for the first request; later requests reuse the cache.
request_config["timeout"] = 600.0
# Cold AITER MHA builds on loaded MI300 agents can exceed 20 minutes.
request_config["timeout"] = 1800.0
tts_server_params = [
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
import os
from pathlib import Path
import pytest
from huggingface_hub import snapshot_download
from tests.e2e.online_serving.nemotron_voicechat_realtime_duplex import parse_args, run
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams, get_model_prefix
from tests.helpers.stage_config import get_deploy_config_path
MODEL = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_MODEL", "nvidia/NVIDIA-NemotronLabs-VoiceChat-11B")
DEPLOY_CONFIG = get_deploy_config_path("nemotron_labs_voicechat_duplex.yaml")
TOKENIZER = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_LLM_PATH")
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize(
"omni_server",
[
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=DEPLOY_CONFIG,
env_dict={"NEMOTRON_VOICECHAT_LLM_PATH": TOKENIZER} if TOKENIZER else None,
),
id="native-duplex",
)
],
indirect=True,
)
def test_native_duplex_turn_taking_streams_model_audio(omni_server, tmp_path: Path) -> None:
model_prefix = get_model_prefix()
root = Path(model_prefix) / MODEL if model_prefix else Path(MODEL)
if not root.is_dir():
root = Path(snapshot_download(MODEL, local_files_only=True))
args = parse_args(
(
f"--url ws://{omni_server.host}:{omni_server.port}/v1/realtime --model {omni_server.model} "
f"--input-wav {root / 'turn_taking.wav'} --input-channel 0 --max-frames 190 "
f"--minimum-audio-chunks 48 --minimum-audio-rms 0.001 --no-realtime --timeout-s 300 "
f"--output-dir {tmp_path / 'native_duplex'}"
).split()
)
result = asyncio.run(run(args))
assert result["ok"] is True
assert result["input_frames"] == 190
assert result["event_counts"]["response.speak"] > 0
assert result["audio_bytes"] >= result["input_frames"] * 1764 * 2 // 4