[Model] Add native full-duplex Nemotron VoiceChat serving (#6089)

Signed-off-by: Sysy <1370724210@qq.com>
Signed-off-by: sy03 <137072421@qq.com>
Co-authored-by: sy03 <137072421@qq.com>
This commit is contained in:
Sy03andsy03 authored and GitHub committed 2026-08-26 13:19:20 -07:00
1 parent c2723979c0
commit 432b2c6ca3
40 files changed
+3560 -154

No files matched your search

@@ -198,6 +198,7 @@ def test_resumable_segment_boundary_keeps_pre_transition_send_watermark() -> Non
inter_stage_output,
session,
True,
new_token_ids=[42],
confirmed_num_computed_tokens=26,
)
@@ -427,3 +428,33 @@ def test_ready_async_chunk_prompt_replacement_releases_stale_kv_once() -> None:
assert sched.chunk_transfer_adapter.replaced_streaming_prompt_ids == set()
assert sched.chunk_transfer_adapter.requests_with_ready_chunks == {session.request_id}
assert sched.chunk_transfer_adapter.requests_num_chunks_sent == {}
def test_chunk_segment_cleanup_keeps_requeued_resumable_receiver() -> None:
"""A WAITING_FOR_CHUNK stop must not delete its newly parked session."""
session = _make_request()
session.resumable = True
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
def queue(*requests):
result = MagicMock()
result.requests = set(requests)
result.add_request.side_effect = result.requests.add
result.remove_requests.side_effect = result.requests.difference_update
return result
sched = OmniARScheduler.__new__(OmniARScheduler)
sched.running = []
sched.waiting = queue()
sched.skipped_waiting = queue(session)
sched.num_waiting_for_streaming_input = 1
sched.chunk_transfer_adapter = SimpleNamespace(segment_finished_requests={session.request_id})
sched._resume_downstream_chunk_receiver(session)
sched._remove_stopped_requests_from_queues(set(), {session})
assert session.status == RequestStatus.WAITING
assert session in sched.waiting.requests
assert session not in sched.skipped_waiting.requests
assert sched.num_waiting_for_streaming_input == 0
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
@@ -1209,6 +1209,67 @@ def test_cleanup_preserves_pending_save(build_adapter):
assert len(adapter._pending_save_reqs) == 1
def test_abort_invalidates_queued_sender_task_before_external_id_reuse(build_adapter):
adapter, connector = build_adapter(stage_id=1)
request = _req("req-old", RequestStatus.WAITING, external_req_id="ext-reused")
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct(
codes=CodesStruct(audio=torch.tensor([1], dtype=torch.long))
)
adapter.save_async(multimodal_output=None, request=request)
stale_task = adapter._pending_save_reqs.popleft()
adapter.cleanup(request.request_id, request.external_req_id)
adapter._send_single_request(stale_task)
connector.put.assert_not_called()
assert request.external_req_id not in adapter.put_req_chunk
replacement = _req("req-new", RequestStatus.WAITING, external_req_id=request.external_req_id)
adapter.save_async(multimodal_output=None, request=replacement)
current_task = adapter._pending_save_reqs.popleft()
adapter._send_single_request(current_task)
connector.put.assert_called_once()
assert connector.put.call_args.kwargs["put_key"] == "ext-reused_1_0"
def test_finish_requests_does_not_wait_for_inflight_send(build_adapter):
adapter, connector = build_adapter(stage_id=1)
first = _req("req-first", RequestStatus.WAITING, external_req_id="ext-first")
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct()
put_started = threading.Event()
release_put = threading.Event()
def blocking_put(**kwargs):
put_started.set()
release_put.wait(timeout=2)
return True, 1, {}
connector.put.side_effect = blocking_put
adapter.save_async(multimodal_output=None, request=first)
task = adapter._pending_save_reqs.popleft()
sender = threading.Thread(target=adapter._send_single_request, args=(task,))
sender.start()
assert put_started.wait(timeout=1)
cleaner = threading.Thread(
target=adapter.finish_requests,
args=([first.request_id], RequestStatus.FINISHED_ABORTED, {first.request_id: first}),
)
cleaner.start()
try:
cleaner.join(timeout=0.5)
assert not cleaner.is_alive()
assert task["sender_token"].cancelled
assert first.external_req_id in adapter._sender_tokens
finally:
release_put.set()
sender.join(timeout=1)
cleaner.join(timeout=1)
assert not sender.is_alive()
assert first.external_req_id not in adapter._sender_tokens
assert first.external_req_id not in adapter.put_req_chunk
def test_cleanup_only_affects_target_request(build_adapter):
"""Cleanup for one request must not affect another request's state."""
adapter, _ = build_adapter(stage_id=1)
@@ -2216,3 +2277,43 @@ def test_expiry_is_per_request(build_adapter):
assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == {"stalled"}
assert "healthy" in adapter._waiting_since
def test_abort_clears_native_codec_state_before_external_id_reuse(build_adapter):
from vllm_omni.model_executor.stage_input_processors.nemotron_voicechat import (
talker2code2wav_async_chunk,
)
adapter, _ = build_adapter(stage_id=1, connector_extra={"codec_chunk_frames": 1})
external_req_id = "ext-native-abort"
first = _req("req-native-abort", RequestStatus.WAITING, external_req_id=external_req_id)
first.resumable = True
first.model_intermediate_buffer = None
first.additional_information = {"meta": {"codec_streaming": True}, "nvc_logical_prompt_len": 1}
first_frame = torch.full((1, 31), 101, dtype=torch.long)
talker2code2wav_async_chunk(
adapter,
{"codes": {"audio": first_frame}, "meta": {"codec_streaming": True}},
first,
is_finished=False,
)
adapter.requests_num_chunks_sent[external_req_id] = 1
adapter.finish_requests([first.request_id], RequestStatus.FINISHED_ABORTED, {first.request_id: first})
assert external_req_id not in adapter.request_payload
assert external_req_id not in adapter.requests_num_chunks_sent
replacement = _req("req-native-abort", RequestStatus.WAITING, external_req_id=external_req_id)
replacement.resumable = True
replacement.model_intermediate_buffer = None
replacement.additional_information = first.additional_information
replacement_payload = talker2code2wav_async_chunk(
adapter,
{"codes": {"audio": torch.full((1, 31), 202, dtype=torch.long)}, "meta": {"codec_streaming": True}},
replacement,
is_finished=False,
)
assert torch.equal(replacement_payload.codes.audio, torch.full((1, 31), 202, dtype=torch.long))
@@ -0,0 +1,94 @@
from __future__ import annotations
from types import SimpleNamespace
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import (
NemotronVoiceChatDataPlaneContext,
NemotronVoiceChatDataPlaneSession,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_RUNTIME = {
"nvc_text_bos_id": 0,
"nvc_text_eos_id": 1,
"nvc_text_pad_id": 12,
"nvc_function_sotc_id": 20,
"nvc_function_eotc_id": 21,
"nvc_tokenizer_ref": "test-tokenizer",
}
def _projector(encode_audio=lambda *_: None) -> NemotronVoiceChatDataPlaneSession:
projector = NemotronVoiceChatDataPlaneSession(encode_audio)
projector.configure_runtime(_RUNTIME, tokenizer=SimpleNamespace(decode=lambda *_args, **_kwargs: "text"))
return projector
def _stage0_output(request_id: str = "req-0", *, text_token: int = 12, function_token: int | None = None) -> object:
metadata: dict[str, object] = {"nvc_text_token_ids": [text_token]}
if function_token is not None:
metadata["nvc_function_token"] = [function_token]
completion = SimpleNamespace(multimodal_output=metadata)
return SimpleNamespace(
stage_id=0,
request_output=SimpleNamespace(request_id=request_id, outputs=[completion]),
)
def test_function_channel_projects_completed_call_without_ending_speech() -> None:
projector = _projector()
projector._decode = lambda token_ids: (
'[{"name":"weather","arguments":{"city":"Shanghai"}}]' if token_ids == [99] else ""
)
context = NemotronVoiceChatDataPlaneContext(epoch=0)
events = []
for function_token in (20, 99, 21):
events.extend(projector.project_output(_stage0_output(function_token=function_token), context=context))
listen = [event for event in events if event.get("is_listen") is True]
function = [event for event in events if event.get("function_call") is True]
assert len(listen) == 3
assert len(function) == 1
assert function[0]["name"] == "weather"
assert function[0]["arguments"] == '{"city":"Shanghai"}'
projector._decode = lambda _token_ids: "not-json"
bad_events = []
for function_token in (20, 98, 21):
bad_events.extend(projector.project_output(_stage0_output(function_token=function_token)))
assert any(event.get("error_code") == "nemotron_function_call_parse_error" for event in bad_events)
def test_new_epoch_request_does_not_inherit_partial_function_or_frame_state() -> None:
projector = _projector(lambda *_: "audio")
old_request = duplex_resource_request_id(DuplexFence("sid", epoch=1), "stage0")
new_request = duplex_resource_request_id(DuplexFence("sid", epoch=2), "stage0")
projector.begin_request(old_request)
list(projector.project_output(_stage0_output(old_request, text_token=42, function_token=20)))
projector.begin_request(new_request)
# EOTC in the new epoch must not close the old epoch's partial function.
new_events = list(projector.project_output(_stage0_output(new_request, text_token=1, function_token=21)))
assert not [event for event in new_events if event.get("function_call") is True]
assert not [event for event in new_events if event.get("end_of_turn") is True]
audio = SimpleNamespace(
stage_id=2,
request_output=SimpleNamespace(
request_id=new_request,
outputs=[
SimpleNamespace(multimodal_output={"model_outputs": [np.ones(1764, dtype=np.float32)], "sr": [22050]})
],
),
)
audio_events = list(projector.project_output(audio))
assert sum(event.get("end_of_turn") is True for event in audio_events) == 1
@@ -0,0 +1,73 @@
from __future__ import annotations
import base64
import numpy as np
import pytest
from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import (
NemotronVoiceChatPcmAppendBuffer,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _payload(samples: np.ndarray, *, sample_rate_hz: int = 16000, fmt: str = "pcm_f32le") -> dict[str, object]:
return {
"type": "audio",
"audio": base64.b64encode(np.asarray(samples, dtype=np.float32).tobytes()).decode("ascii"),
"format": fmt,
"sample_rate_hz": sample_rate_hz,
}
def _append(buffer, samples, operation_id):
return buffer.prepare_append(
_payload(np.asarray(samples, dtype=np.float32)),
operation_id=operation_id,
chunk_period_ms=80,
allow_emit=True,
)
def test_irregular_browser_packets_emit_exact_ordered_80ms_frame() -> None:
buffer = NemotronVoiceChatPcmAppendBuffer()
assert _append(buffer, np.arange(1000), "packet-1") is None
reservation = _append(buffer, np.arange(1000, 1280), "packet-2")
assert reservation is not None
assert reservation.operation_id == "packet-2"
decoded = np.frombuffer(base64.b64decode(reservation.payload["audio"]), dtype=np.float32)
np.testing.assert_array_equal(decoded, np.arange(1280, dtype=np.float32))
assert buffer.pending_byte_count == 0
def test_append_rejects_invalid_frame_contract() -> None:
cases = (
(1281, 16000, "pcm_f32le", 0.0, "at most 1280"),
(1, 8000, "pcm_f32le", 0.0, "sample_rate_hz"),
(1, 16000, "audio/pcm", 0.0, "format"),
(1, 16000, "pcm_f32le", np.nan, "finite"),
)
for samples, sample_rate_hz, fmt, value, message in cases:
with pytest.raises(ValueError, match=message):
NemotronVoiceChatPcmAppendBuffer().prepare_append(
_payload(np.full(samples, value, dtype=np.float32), sample_rate_hz=sample_rate_hz, fmt=fmt),
operation_id="invalid",
chunk_period_ms=80,
allow_emit=True,
)
def test_terminal_commit_rejects_more_than_one_frame_tail() -> None:
buffer = NemotronVoiceChatPcmAppendBuffer()
for index, samples in enumerate((1000, 281)):
buffer.prepare_append(
_payload(np.zeros(samples, dtype=np.float32)),
operation_id=f"tail-{index}",
chunk_period_ms=80,
allow_emit=False,
)
with pytest.raises(ValueError, match="more than one 80 ms frame"):
buffer.prepare_commit(operation_id="commit", chunk_period_ms=80)
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import base64
from types import SimpleNamespace
import numpy as np
import pytest
from vllm.sampling_params import SamplingParams
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import (
NemotronVoiceChatDuplexRuntimeExtension,
)
from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import (
NemotronVoiceChatServingRuntimeAdapter,
_render_tool_response,
)
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
NemotronVoiceChatThinkerForConditionalGeneration,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _frame() -> dict[str, object]:
raw = np.zeros(1280, dtype=np.float32).tobytes()
return {
"type": "audio",
"audio": base64.b64encode(raw).decode("ascii"),
"format": "pcm_f32le",
"sample_rate_hz": 16000,
}
def _plan(extension, *, input_seq: int):
return extension.plan_append(
request_id="req",
fence=DuplexFence("sid", incarnation=2, epoch=3),
session_config={},
runtime_config={
"nvc_prompt_token_ids": [0, 42, 1],
"nvc_text_pad_id": 12,
"nvc_max_model_len": 8192,
},
seq=input_seq,
turn_seq=input_seq,
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
payload=_frame(),
final=False,
sampling_params=SamplingParams(),
)
def test_first_append_prefills_prompt_then_each_append_consumes_one_frame() -> None:
extension = NemotronVoiceChatDuplexRuntimeExtension()
params = extension.configure_sampling_params(
runtime_config={}, defaults=(SamplingParams(temperature=0.7, max_tokens=4),)
)[0]
assert params.temperature == 0.0 and params.max_tokens == 1 and params.top_p == 1.0 and params.top_k == 0
first = _plan(extension, input_seq=1)
later = _plan(extension, input_seq=2)
assert first.prompt["prompt_token_ids"] == [0, 42, 1, 12]
assert later.prompt["prompt_token_ids"] == [12]
assert first.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 1
assert later.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 2
def test_append_rejects_stage0_context_overflow() -> None:
with pytest.raises(ValueError, match="max_model_len"):
_plan(NemotronVoiceChatDuplexRuntimeExtension(), input_seq=8190)
def test_function_output_becomes_versioned_nvidia_channel_tokens() -> None:
encoded: list[str] = []
adapter = NemotronVoiceChatServingRuntimeAdapter(lambda *_: None)
adapter._tokenizer = SimpleNamespace(
encode=lambda text, **_kwargs: encoded.append(text) or [31, 32, 33],
)
first = adapter.runtime_config_for_function_output(
{"type": "function_call_output", "call_id": "call-1", "output": '{"result":20}'},
{},
)
second = adapter.runtime_config_for_function_output(
{"type": "function_call_output", "call_id": "call-2", "output": "plain text"},
first,
)
assert _render_tool_response('{"result":20}') == '<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>'
assert encoded == [
'<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>',
'<TOOL_RESPONSE>["plain text"]</TOOL_RESPONSE>',
]
assert first["nvc_function_response_generation"] == 1
assert second["nvc_function_response_generation"] == 2
assert second["nvc_function_response_token_ids"] == [31, 32, 33]
assert second["nvc_function_response_call_id"] == "call-2"
assert second["nvc_function_response_batches"] == [
{"generation": 1, "call_id": "call-1", "token_ids": [31, 32, 33]},
{"generation": 2, "call_id": "call-2", "token_ids": [31, 32, 33]},
]
session = {"function_response_generation": 0, "forced_function_tokens": []}
NemotronVoiceChatThinkerForConditionalGeneration._sync_forced_function_response(session, second)
assert session["function_response_generation"] == 2
assert session["forced_function_token"] == 31
assert session["forced_function_tokens"] == [31, 32, 33, 31, 32, 33]
@@ -0,0 +1,364 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import argparse
import asyncio
import base64
import json
import math
import uuid
import wave
from collections.abc import Sequence
from contextlib import asynccontextmanager, suppress
from pathlib import Path
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import numpy as np
from scipy.signal import resample_poly
from vllm_omni.experimental.fullduplex.client import RealtimeDuplexClient, wait_for, write_pcm16_wav
INPUT_SAMPLE_RATE_HZ = 16_000
OUTPUT_SAMPLE_RATE_HZ = 22_050
FRAME_SAMPLES = 1_280
FRAME_PERIOD_S = FRAME_SAMPLES / INPUT_SAMPLE_RATE_HZ
DEFAULT_FUNCTION_TOOLS = [
{
"type": "function",
"name": "generate_random_number",
"description": "Generate a random integer between min and max (inclusive).",
"parameters": {
"type": "object",
"properties": {
"min": {"type": "integer", "description": "Minimum value (inclusive)"},
"max": {"type": "integer", "description": "Maximum value (inclusive)"},
},
"required": ["min", "max"],
},
}
]
DEFAULT_INSTRUCTIONS = "You are NVIDIA Voice Chat. Answer briefly. Start by greeting the user."
DEFAULT_FUNCTION_INSTRUCTIONS = (
"You are NVIDIA Voice Chat. If the user's request matches an available tool, "
"you MUST call that tool instead of answering from your own knowledge. "
"Use only argument values spoken by the user and never invent missing values."
)
def _url(base_url: str, model: str, session_id: str) -> str:
parts = urlsplit(base_url)
query = dict(parse_qsl(parts.query, keep_blank_values=True))
query.update(duplex="1", model=model, autostart="0", session_id=session_id)
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
def _read_wav(path: Path, *, input_channel: int = 0) -> np.ndarray:
with wave.open(str(path), "rb") as wav_file:
channels = wav_file.getnchannels()
width = wav_file.getsampwidth()
source_rate = wav_file.getframerate()
raw = wav_file.readframes(wav_file.getnframes())
dtypes = {1: np.uint8, 2: np.dtype("<i2"), 4: np.dtype("<i4")}
if width not in dtypes:
raise ValueError(f"unsupported input sample width: {width}")
pcm = np.frombuffer(raw, dtype=dtypes[width]).astype(np.float32)
pcm = (pcm - 128.0) / 128.0 if width == 1 else pcm / float(1 << (width * 8 - 1))
if not 0 <= input_channel < channels:
raise ValueError(f"input channel {input_channel} is outside WAV channel count {channels}")
if channels > 1:
pcm = pcm.reshape(-1, channels)[:, input_channel]
if source_rate != INPUT_SAMPLE_RATE_HZ:
divisor = math.gcd(source_rate, INPUT_SAMPLE_RATE_HZ)
pcm = resample_poly(pcm, up=INPUT_SAMPLE_RATE_HZ // divisor, down=source_rate // divisor)
return np.ascontiguousarray(pcm, dtype="<f4")
async def _stream(client: RealtimeDuplexClient, pcm: np.ndarray, *, max_frames: int | None, realtime: bool) -> int:
count = math.ceil(pcm.size / FRAME_SAMPLES)
if max_frames is not None:
count = min(count, max_frames)
for seq in range(count):
frame = pcm[seq * FRAME_SAMPLES : (seq + 1) * FRAME_SAMPLES]
frame = np.pad(frame, (0, FRAME_SAMPLES - frame.size)).astype("<f4")
await client.send(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(frame).decode("ascii"),
"format": "pcm_f32le",
"sample_rate_hz": INPUT_SAMPLE_RATE_HZ,
"duration_ms": 80,
"audio_end_ms": (seq + 1) * 80,
}
)
if realtime:
await asyncio.sleep(FRAME_PERIOD_S)
return count
def _events(client: RealtimeDuplexClient, event_type: str) -> list[dict[str, object]]:
return [event for event in client.events.events if event.get("type") == event_type]
async def _return_function_output_when_ready(
client: RealtimeDuplexClient,
*,
output: str,
timeout_s: float,
) -> tuple[str, int]:
await wait_for(
lambda: bool(client.events.errors()) or client.events.count("response.function_call_arguments.done") > 0,
timeout_s=timeout_s,
label="function call to execute",
)
if client.events.errors():
raise AssertionError(f"function call failed before tool execution: {client.events.errors()}")
function_done = _events(client, "response.function_call_arguments.done")[-1]
call_id = function_done.get("call_id")
if not isinstance(call_id, str) or not call_id:
raise AssertionError(f"completed function call has no call_id: {function_done}")
event_count_before_output = len(client.events.events)
await client.send(
{
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": output,
},
}
)
return call_id, event_count_before_output
def _write_events(path: Path, client: RealtimeDuplexClient) -> None:
path.write_text(
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events),
encoding="utf-8",
)
@asynccontextmanager
async def _managed_client(client: RealtimeDuplexClient, *, timeout_s: float):
async with client:
try:
yield
finally:
if client.events.count("session.created") and not client.events.count("session.closed"):
with suppress(Exception):
await client.close_session(timeout_s=min(timeout_s, 30.0))
async def run(args: argparse.Namespace) -> dict[str, object]:
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
if args.expect_function_call and args.instructions == DEFAULT_INSTRUCTIONS:
instructions = DEFAULT_FUNCTION_INSTRUCTIONS
else:
instructions = args.instructions
tools = DEFAULT_FUNCTION_TOOLS if args.expect_function_call else None
session_id = f"nemotron-voicechat-{uuid.uuid4().hex}"
client = RealtimeDuplexClient(_url(args.url, args.model, session_id))
async with _managed_client(client, timeout_s=args.timeout_s):
session_payload: dict[str, object] = {
"session_id": session_id,
"model": args.model,
"modalities": ["audio", "text"],
"input_audio_format": "pcm_f32le",
"output_audio_format": "pcm16",
"instructions": instructions,
"idle_timeout_s": args.timeout_s,
"turn_detection": None,
"extra_body": {"auto_response": True},
}
if tools is not None:
session_payload["tools"] = tools
await client.send({"type": "session.update", "session": session_payload})
await wait_for(
lambda: client.events.count("session.created") > 0 or bool(client.events.errors()),
timeout_s=args.timeout_s,
label="session.created",
)
if client.events.errors():
raise AssertionError(f"session setup failed: {client.events.errors()}")
created = _events(client, "session.created")[-1]
session = created.get("session")
capabilities = session.get("capabilities") if isinstance(session, dict) else None
expected = dict(
implementation_level="model_native_duplex",
chunk_period_ms=80,
supports_core_resumable_request=True,
supports_core_kv_lease=False,
supports_multi_session=False,
)
if not isinstance(capabilities, dict) or any(capabilities.get(key) != value for key, value in expected.items()):
raise AssertionError(f"unexpected capabilities: {capabilities}")
function_output_task = (
asyncio.create_task(
_return_function_output_when_ready(
client,
output=args.function_output,
timeout_s=args.timeout_s,
)
)
if args.function_output is not None
else None
)
pcm = _read_wav(Path(args.input_wav), input_channel=args.input_channel)
frame_count = await _stream(client, pcm, max_frames=args.max_frames, realtime=not args.no_realtime)
completed_responses_at_commit = client.events.count("response.done")
await client.send({"type": "input_audio_buffer.commit", "final": True})
await wait_for(
lambda: (
bool(client.events.errors())
or (
client.events.count("response.function_call_arguments.done") > 0
if args.expect_function_call
else (
client.events.count("response.audio.delta") >= args.minimum_audio_chunks
and (client.events.count("response.done") > completed_responses_at_commit)
)
)
),
timeout_s=args.timeout_s,
label="model output",
)
await asyncio.sleep(args.drain_s)
if client.events.errors():
raise AssertionError(f"Realtime session emitted errors: {client.events.errors()}")
done_events = _events(client, "response.done")
if not args.expect_function_call:
response = done_events[-1].get("response") if done_events else None
status = response.get("status") if isinstance(response, dict) else None
if status != "completed":
raise AssertionError(f"response did not complete successfully: {done_events[-1:]}")
function_events = [
event for event in client.events.events if str(event.get("type", "")).startswith("response.function_call")
]
function_items = [
event
for event in _events(client, "response.output_item.done")
if isinstance(event.get("item"), dict) and event["item"].get("type") == "function_call"
]
if args.expect_function_call and not any(
event.get("type") == "response.function_call_arguments.done" for event in function_events
):
raise AssertionError(f"no completed function call: {function_events}")
if args.expect_function_call:
matching_items = [
event for event in function_items if event["item"].get("name") == args.expected_function_name
]
if not matching_items:
raise AssertionError(f"expected {args.expected_function_name!r}, got {function_items}")
function_item = matching_items[-1]["item"]
try:
function_arguments = json.loads(str(function_item.get("arguments", "")))
except json.JSONDecodeError as exc:
raise AssertionError(f"function arguments are not JSON: {function_item}") from exc
if args.expected_function_arguments is not None:
expected_arguments = json.loads(args.expected_function_arguments)
if function_arguments != expected_arguments:
raise AssertionError(
f"function arguments differ: expected={expected_arguments!r}, actual={function_arguments!r}"
)
if args.function_output is not None:
call_id = function_item.get("call_id")
if not isinstance(call_id, str) or not call_id:
raise AssertionError(f"function item has no call_id: {function_item}")
assert function_output_task is not None
returned_call_id, event_count_before_output = await function_output_task
assert returned_call_id == call_id
def tool_result_completed() -> bool:
later = client.events.events[event_count_before_output:]
transcript = "".join(
str(event.get("delta", ""))
for event in later
if event.get("type") == "response.audio_transcript.delta"
).lower()
return (
any(event.get("type") == "response.audio.delta" for event in later)
and any(event.get("type") == "response.done" for event in later)
and (args.expected_post_tool_text is None or args.expected_post_tool_text.lower() in transcript)
)
await wait_for(
lambda: bool(client.events.errors()) or tool_result_completed(),
timeout_s=args.timeout_s,
label="completed response after function output",
)
if client.events.errors():
raise AssertionError(f"function output failed: {client.events.errors()}")
await asyncio.sleep(args.drain_s)
audio = client.events.audio_bytes()
audio_events = _events(client, "response.audio.delta")
rates = {event.get("sample_rate_hz") for event in audio_events}
if not args.expect_function_call and audio and rates != {OUTPUT_SAMPLE_RATE_HZ}:
raise AssertionError(f"unexpected output sample rates: {rates}")
if not args.expect_function_call and args.minimum_audio_chunks and not audio:
raise AssertionError("model produced no audio")
expected_bytes = 2 * OUTPUT_SAMPLE_RATE_HZ * expected["chunk_period_ms"] // 1000
packet_sizes = [len(base64.b64decode(str(event.get("delta", "")), validate=True)) for event in audio_events]
if not args.expect_function_call and any(size != expected_bytes for size in packet_sizes):
raise AssertionError(f"audio deltas are not fixed 80 ms PCM16 packets: {packet_sizes}")
audio_pcm = np.frombuffer(audio, dtype="<i2").astype(np.float32) / 32768.0
audio_rms = float(np.sqrt(np.mean(np.square(audio_pcm)))) if audio_pcm.size else 0.0
if not args.expect_function_call and audio and audio_rms < args.minimum_audio_rms:
raise AssertionError(
f"model output RMS {audio_rms:.6f} is below {args.minimum_audio_rms:.6f}; "
"received packets contain only silence"
)
_write_events(output_dir / "events.jsonl", client)
if audio:
write_pcm16_wav(output_dir / "output.wav", audio, sample_rate_hz=OUTPUT_SAMPLE_RATE_HZ)
result = {
"ok": True,
"session_id": session_id,
"input_frames": frame_count,
"capabilities": capabilities,
"event_counts": {
event_type: client.events.count(event_type)
for event_type in sorted({str(event.get("type")) for event in client.events.events})
},
"audio_bytes": len(audio),
"audio_rms": audio_rms,
"function_events": function_events,
"function_items": function_items,
"output_dir": str(output_dir),
}
(output_dir / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
return result
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--url", default="ws://127.0.0.1:8125/v1/realtime")
parser.add_argument("--model", required=True)
parser.add_argument("--input-wav", required=True)
parser.add_argument("--input-channel", type=int, default=0)
parser.add_argument("--output-dir", default="/tmp/nemotron-voicechat-duplex")
parser.add_argument("--instructions", default=DEFAULT_INSTRUCTIONS)
parser.add_argument("--max-frames", type=int)
parser.add_argument("--minimum-audio-chunks", type=int, default=1)
parser.add_argument("--minimum-audio-rms", type=float, default=1e-4)
parser.add_argument("--expect-function-call", action="store_true")
parser.add_argument("--expected-function-name", default="generate_random_number")
parser.add_argument("--expected-function-arguments")
parser.add_argument("--function-output")
parser.add_argument("--expected-post-tool-text")
parser.add_argument("--no-realtime", action="store_true")
parser.add_argument("--drain-s", type=float, default=2.0)
parser.add_argument("--timeout-s", type=float, default=600.0)
return parser.parse_args(argv)
def main() -> None:
print(json.dumps(asyncio.run(run(parse_args())), ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
@@ -52,9 +52,8 @@ def get_prompt(prompt_type="zh"):
def _set_rocm_request_timeout(request_config: dict) -> None:
if current_omni_platform.is_rocm():
# A cold AITER MHA kernel build took 301 seconds on MI300X.
# Allow 600 seconds for the first request; later requests reuse the cache.
request_config["timeout"] = 600.0
# Cold AITER MHA builds on loaded MI300 agents can exceed 20 minutes.
request_config["timeout"] = 1800.0
tts_server_params = [
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import asyncio
import os
from pathlib import Path
import pytest
from huggingface_hub import snapshot_download
from tests.e2e.online_serving.nemotron_voicechat_realtime_duplex import parse_args, run
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniServerParams, get_model_prefix
from tests.helpers.stage_config import get_deploy_config_path
MODEL = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_MODEL", "nvidia/NVIDIA-NemotronLabs-VoiceChat-11B")
DEPLOY_CONFIG = get_deploy_config_path("nemotron_labs_voicechat_duplex.yaml")
TOKENIZER = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_LLM_PATH")
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize(
"omni_server",
[
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=DEPLOY_CONFIG,
env_dict={"NEMOTRON_VOICECHAT_LLM_PATH": TOKENIZER} if TOKENIZER else None,
),
id="native-duplex",
)
],
indirect=True,
)
def test_native_duplex_turn_taking_streams_model_audio(omni_server, tmp_path: Path) -> None:
model_prefix = get_model_prefix()
root = Path(model_prefix) / MODEL if model_prefix else Path(MODEL)
if not root.is_dir():
root = Path(snapshot_download(MODEL, local_files_only=True))
args = parse_args(
(
f"--url ws://{omni_server.host}:{omni_server.port}/v1/realtime --model {omni_server.model} "
f"--input-wav {root / 'turn_taking.wav'} --input-channel 0 --max-frames 190 "
f"--minimum-audio-chunks 48 --minimum-audio-rms 0.001 --no-realtime --timeout-s 300 "
f"--output-dir {tmp_path / 'native_duplex'}"
).split()
)
result = asyncio.run(run(args))
assert result["ok"] is True
assert result["input_frames"] == 190
assert result["event_counts"]["response.speak"] > 0
assert result["audio_bytes"] >= result["input_frames"] * 1764 * 2 // 4
@@ -551,6 +551,54 @@ async def test_native_realtime_protocol_conversation_item_create_commits_user_te
}
@pytest.mark.asyncio
async def test_function_output_is_recorded_only_after_runtime_delivery():
ws = TimedWebSocket()
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
protocol.bind_sender(ws.send_json)
protocol.encode_outbound_event(
{
"type": "conversation.item.created",
"item": {
"id": "item-call",
"type": "function_call",
"call_id": "call-1",
"name": "weather",
"arguments": "{}",
"status": "completed",
},
}
)
event = {
"type": "conversation.item.create",
"item": {
"id": "item-output",
"type": "function_call_output",
"call_id": "call-1",
"output": '{"temperature":20}',
},
}
first = await protocol._to_duplex_event(event)
retry = await protocol._to_duplex_event(event)
assert first is not None
assert retry is not None
assert "item-output" not in protocol._conversation_items
delivered_item = first["payload"]["item"]
protocol.encode_outbound_event(
{
"type": "conversation.item.created",
"item": delivered_item,
}
)
assert protocol._conversation_items["item-output"] == delivered_item
assert await protocol._to_duplex_event(event) is None
assert ws.sent[-1]["error"]["code"] == "invalid_function_call_output"
@pytest.mark.asyncio
async def test_native_realtime_protocol_audio_commit_requires_non_empty_buffer():
ws = TimedWebSocket()
@@ -5481,7 +5529,7 @@ async def test_failed_native_append_prevents_queued_append_from_running():
@pytest.mark.asyncio
async def test_failed_final_append_keeps_committed_audio_for_response_retry():
async def test_failed_final_append_discards_retained_audio_without_retry():
failed_result = {
"operation": "append",
"session_id": "sid-final-append-retry",
@@ -5490,15 +5538,7 @@ async def test_failed_final_append_keeps_committed_audio_for_response_retry():
"error_count": 1,
"stage_results": [],
}
successful_result = {
"operation": "append",
"session_id": "sid-final-append-retry",
"ok": True,
"unsupported_count": 0,
"error_count": 0,
"stage_results": [],
}
engine = FakeEngineClient(append_results=[failed_result, successful_result])
engine = FakeEngineClient(append_result=failed_result)
handler = OmniDuplexSessionHandler(
chat_service=FakeChatService(engine),
config_timeout_s=0.1,
@@ -5523,15 +5563,18 @@ async def test_failed_final_append_keeps_committed_audio_for_response_retry():
handler_task = asyncio.create_task(handler.handle_session(ws))
for _ in range(100):
if len(engine.appended) >= 2:
state = handler._serving_runtime_adapter.session_states.get("sid-final-append-retry")
if len(engine.appended) >= 1 and state is not None and state.committed_audio_payload is None:
break
await asyncio.sleep(0.01)
state = handler._serving_runtime_adapter.session_states["sid-final-append-retry"]
assert state.committed_audio_payload is None
assert state.committed_audio_reserved_bytes == 0
assert handler._registry.get("sid-final-append-retry").pending_input_bytes == 0
ws.put({"type": "session.close"})
await asyncio.wait_for(handler_task, timeout=2)
assert len(engine.appended) == 2
assert engine.appended[0][2]["audio"] == engine.appended[1][2]["audio"]
assert engine.append_operation_ids[0] == engine.append_operation_ids[1]
assert len(engine.appended) == 1
@pytest.mark.asyncio
@@ -225,3 +225,39 @@ def test_torchaudio_mel_filterbank_matches_librosa() -> None:
reference = librosa.filters.mel(sr=16000, n_fft=512, n_mels=128, fmin=0.0, fmax=8000.0, norm="slaney")
assert ta.shape == reference.shape
assert abs(ta - reference).max() < 1e-6
def test_code2wav_reuses_and_clears_per_request_streaming_cache() -> None:
import torch
from torch import nn
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_code2wav import (
NemotronVoiceChatCode2Wav,
)
class Codec(nn.Module):
def __init__(self) -> None:
super().__init__()
self.anchor = nn.Parameter(torch.zeros(()))
self.caches = []
def decode(self, codes, lengths, *, cache):
self.caches.append(cache)
wav = torch.zeros((1, int(codes.shape[1]) * 1764), device=codes.device)
return wav, lengths * 1764
model = NemotronVoiceChatCode2Wav.__new__(NemotronVoiceChatCode2Wav)
nn.Module.__init__(model)
model._sample_rate, model._num_quantizers, model._codebook_size, model._wav_per_frame = 22050, 31, 1024, 1764
model._duplex_codec_caches = {}
model.audio_codec = Codec()
def info(codes):
return [{"meta": {"codec_streaming": True, "request_id": "req"}, "codes": {"audio": codes}}]
model.forward(runtime_additional_information=info(torch.zeros((1, 31), dtype=torch.long)))
model.forward(runtime_additional_information=info(torch.zeros((2, 31), dtype=torch.long)))
assert model.audio_codec.caches[0] is model.audio_codec.caches[1]
assert "req" in model._duplex_codec_caches
model.on_requests_finished({"req"})
assert "req" not in model._duplex_codec_caches
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
@@ -285,9 +285,46 @@ def test_talker2code2wav_async_chunk_requires_prompt_len() -> None:
talker2code2wav_async_chunk(tm, {"codes": {"audio": torch.zeros(5, 31, dtype=torch.long)}}, req)
# =========================
# Talker drain-per-wake (async scheduling review P1/P2)
# =========================
def test_talker2code2wav_duplex_keeps_cumulative_history_across_wakes() -> None:
from vllm_omni.model_executor.stage_input_processors.nemotron_voicechat import (
talker2code2wav_async_chunk,
)
tm = _fake_transfer_manager(codec_chunk_frames=2)
prompt_len = 3
req = _streaming_request(
prompt_len=prompt_len,
generated=[],
info={"meta": {"codec_streaming": torch.tensor([True])}, "nvc_logical_prompt_len": prompt_len},
)
prompt_rows = torch.zeros((prompt_len - 1, 31), dtype=torch.long)
first = torch.full((1, 31), 101, dtype=torch.long)
second = torch.full((1, 31), 202, dtype=torch.long)
for rows, finished in ((prompt_rows, False), (torch.cat([prompt_rows, first]), False)):
payload = talker2code2wav_async_chunk(
tm,
{"codes": {"audio": rows}, "meta": {"codec_streaming": torch.tensor([True])}},
req,
is_finished=finished,
)
assert payload is not None and not bool(payload.meta.finished)
payload = talker2code2wav_async_chunk(
tm,
{
"codes": {"audio": torch.cat([prompt_rows, first, second])},
"meta": {"codec_streaming": torch.tensor([True])},
},
req,
is_finished=True,
)
assert payload is not None
assert not bool(payload.meta.finished)
assert not bool(payload.meta.is_segment_finished)
assert torch.equal(payload.codes.audio, torch.cat([first, second]))
assert tm.request_payload["req-0"]["nvc_frames_seen"] == 2
class _FakeTTSModelOutput:
@@ -342,7 +379,14 @@ def _bare_talker():
return talker
def _async_info(timeline: list[int], prompt_len: int, finished: bool, request_id: str = "req-0") -> dict:
def _async_info(
timeline: list[int],
prompt_len: int,
finished: bool,
request_id: str = "req-0",
*,
codec_streaming: bool = False,
) -> dict:
return {
"request_id": request_id,
"additional_information": {
@@ -351,6 +395,7 @@ def _async_info(timeline: list[int], prompt_len: int, finished: bool, request_id
"num_processed_tokens": prompt_len,
"finished": torch.tensor(finished),
"request_id": request_id,
"codec_streaming": codec_streaming,
},
},
}
@@ -373,7 +418,7 @@ def test_talker_drains_all_received_positions_per_wake() -> None:
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info)
assert session["step"] == prompt_len # t advanced 1 -> P in one wake
assert update["codes"]["audio"].shape == (prompt_len - 1, 31)
assert float(embeds[0, 0]) == 0.0 # not finished
assert float(embeds[0, 0]) == 0.0 # offline stream waits for upstream terminal
# Coalesced chunk: TWO new frame tokens arrive in ONE wake (delayed save
# thread merged two thinker steps). Both must be consumed by this wake.
@@ -383,14 +428,13 @@ def test_talker_drains_all_received_positions_per_wake() -> None:
assert update["codes"]["audio"].shape == (prompt_len + 1, 31)
assert float(embeds[0, 0]) == 0.0
# Zero-progress wake (no new positions, not finished): plain CONTINUE, no
# fabricated frames, no codes update.
# Zero-progress wake: never fabricate frames or end the offline stream
# before the thinker sends its terminal marker.
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info2)
assert session["step"] == prompt_len + 2
assert update == {}
assert float(embeds[0, 0]) == 0.0
# Final chunk: one more token + finished marker -> drain + stop flag.
info3 = _async_info([pad] * prompt_len + [7, 8, 9], prompt_len, finished=True)
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info3)
assert session["step"] == prompt_len + 3
@@ -398,6 +442,18 @@ def test_talker_drains_all_received_positions_per_wake() -> None:
assert float(embeds[0, 0]) == 1.0 # stop
def test_talker_native_codec_stream_ends_each_received_segment() -> None:
talker = _bare_talker()
ids = torch.zeros(1, dtype=torch.long)
info = _async_info([12, 12, 7], 2, finished=False, codec_streaming=True)
talker.preprocess(ids, None, _omni_is_prefill=True, **info)
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info)
assert update["codes"]["audio"].shape == (2, 31)
assert float(embeds[0, 0]) == 1.0
def test_talker_sync_mode_steps_once_per_wake() -> None:
"""The verified sync parity path keeps its one-TTS-step-per-engine-step cadence."""
talker = _bare_talker()
+1
View File
@@ -1832,6 +1832,7 @@ class TestPreferModelSamplerNoneFallback:
"hunyuan_image3",
"minicpmo_4_5",
"minimax_music3",
"nemotron_voicechat",
}
assert declarers == expected, (
"The set of models declaring `prefer_model_sampler` changed:\n"