[Model] Add native full-duplex Nemotron VoiceChat serving (#6089)
Signed-off-by: Sysy <1370724210@qq.com> Signed-off-by: sy03 <137072421@qq.com> Co-authored-by: sy03 <137072421@qq.com>
This commit is contained in:
1 parent
c2723979c0
commit
432b2c6ca3
40 files changed
+3560
-154
No files matched your search
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import (
|
||||
NemotronVoiceChatDataPlaneContext,
|
||||
NemotronVoiceChatDataPlaneSession,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
_RUNTIME = {
|
||||
"nvc_text_bos_id": 0,
|
||||
"nvc_text_eos_id": 1,
|
||||
"nvc_text_pad_id": 12,
|
||||
"nvc_function_sotc_id": 20,
|
||||
"nvc_function_eotc_id": 21,
|
||||
"nvc_tokenizer_ref": "test-tokenizer",
|
||||
}
|
||||
|
||||
|
||||
def _projector(encode_audio=lambda *_: None) -> NemotronVoiceChatDataPlaneSession:
|
||||
projector = NemotronVoiceChatDataPlaneSession(encode_audio)
|
||||
projector.configure_runtime(_RUNTIME, tokenizer=SimpleNamespace(decode=lambda *_args, **_kwargs: "text"))
|
||||
return projector
|
||||
|
||||
|
||||
def _stage0_output(request_id: str = "req-0", *, text_token: int = 12, function_token: int | None = None) -> object:
|
||||
metadata: dict[str, object] = {"nvc_text_token_ids": [text_token]}
|
||||
if function_token is not None:
|
||||
metadata["nvc_function_token"] = [function_token]
|
||||
completion = SimpleNamespace(multimodal_output=metadata)
|
||||
return SimpleNamespace(
|
||||
stage_id=0,
|
||||
request_output=SimpleNamespace(request_id=request_id, outputs=[completion]),
|
||||
)
|
||||
|
||||
|
||||
def test_function_channel_projects_completed_call_without_ending_speech() -> None:
|
||||
projector = _projector()
|
||||
projector._decode = lambda token_ids: (
|
||||
'[{"name":"weather","arguments":{"city":"Shanghai"}}]' if token_ids == [99] else ""
|
||||
)
|
||||
context = NemotronVoiceChatDataPlaneContext(epoch=0)
|
||||
|
||||
events = []
|
||||
for function_token in (20, 99, 21):
|
||||
events.extend(projector.project_output(_stage0_output(function_token=function_token), context=context))
|
||||
|
||||
listen = [event for event in events if event.get("is_listen") is True]
|
||||
function = [event for event in events if event.get("function_call") is True]
|
||||
assert len(listen) == 3
|
||||
assert len(function) == 1
|
||||
assert function[0]["name"] == "weather"
|
||||
assert function[0]["arguments"] == '{"city":"Shanghai"}'
|
||||
|
||||
projector._decode = lambda _token_ids: "not-json"
|
||||
bad_events = []
|
||||
for function_token in (20, 98, 21):
|
||||
bad_events.extend(projector.project_output(_stage0_output(function_token=function_token)))
|
||||
assert any(event.get("error_code") == "nemotron_function_call_parse_error" for event in bad_events)
|
||||
|
||||
|
||||
def test_new_epoch_request_does_not_inherit_partial_function_or_frame_state() -> None:
|
||||
projector = _projector(lambda *_: "audio")
|
||||
old_request = duplex_resource_request_id(DuplexFence("sid", epoch=1), "stage0")
|
||||
new_request = duplex_resource_request_id(DuplexFence("sid", epoch=2), "stage0")
|
||||
|
||||
projector.begin_request(old_request)
|
||||
list(projector.project_output(_stage0_output(old_request, text_token=42, function_token=20)))
|
||||
projector.begin_request(new_request)
|
||||
|
||||
# EOTC in the new epoch must not close the old epoch's partial function.
|
||||
new_events = list(projector.project_output(_stage0_output(new_request, text_token=1, function_token=21)))
|
||||
assert not [event for event in new_events if event.get("function_call") is True]
|
||||
assert not [event for event in new_events if event.get("end_of_turn") is True]
|
||||
|
||||
audio = SimpleNamespace(
|
||||
stage_id=2,
|
||||
request_output=SimpleNamespace(
|
||||
request_id=new_request,
|
||||
outputs=[
|
||||
SimpleNamespace(multimodal_output={"model_outputs": [np.ones(1764, dtype=np.float32)], "sr": [22050]})
|
||||
],
|
||||
),
|
||||
)
|
||||
audio_events = list(projector.project_output(audio))
|
||||
assert sum(event.get("end_of_turn") is True for event in audio_events) == 1
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import (
|
||||
NemotronVoiceChatPcmAppendBuffer,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _payload(samples: np.ndarray, *, sample_rate_hz: int = 16000, fmt: str = "pcm_f32le") -> dict[str, object]:
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(np.asarray(samples, dtype=np.float32).tobytes()).decode("ascii"),
|
||||
"format": fmt,
|
||||
"sample_rate_hz": sample_rate_hz,
|
||||
}
|
||||
|
||||
|
||||
def _append(buffer, samples, operation_id):
|
||||
return buffer.prepare_append(
|
||||
_payload(np.asarray(samples, dtype=np.float32)),
|
||||
operation_id=operation_id,
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
|
||||
def test_irregular_browser_packets_emit_exact_ordered_80ms_frame() -> None:
|
||||
buffer = NemotronVoiceChatPcmAppendBuffer()
|
||||
assert _append(buffer, np.arange(1000), "packet-1") is None
|
||||
|
||||
reservation = _append(buffer, np.arange(1000, 1280), "packet-2")
|
||||
|
||||
assert reservation is not None
|
||||
assert reservation.operation_id == "packet-2"
|
||||
decoded = np.frombuffer(base64.b64decode(reservation.payload["audio"]), dtype=np.float32)
|
||||
np.testing.assert_array_equal(decoded, np.arange(1280, dtype=np.float32))
|
||||
assert buffer.pending_byte_count == 0
|
||||
|
||||
|
||||
def test_append_rejects_invalid_frame_contract() -> None:
|
||||
cases = (
|
||||
(1281, 16000, "pcm_f32le", 0.0, "at most 1280"),
|
||||
(1, 8000, "pcm_f32le", 0.0, "sample_rate_hz"),
|
||||
(1, 16000, "audio/pcm", 0.0, "format"),
|
||||
(1, 16000, "pcm_f32le", np.nan, "finite"),
|
||||
)
|
||||
for samples, sample_rate_hz, fmt, value, message in cases:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
NemotronVoiceChatPcmAppendBuffer().prepare_append(
|
||||
_payload(np.full(samples, value, dtype=np.float32), sample_rate_hz=sample_rate_hz, fmt=fmt),
|
||||
operation_id="invalid",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_commit_rejects_more_than_one_frame_tail() -> None:
|
||||
buffer = NemotronVoiceChatPcmAppendBuffer()
|
||||
for index, samples in enumerate((1000, 281)):
|
||||
buffer.prepare_append(
|
||||
_payload(np.zeros(samples, dtype=np.float32)),
|
||||
operation_id=f"tail-{index}",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=False,
|
||||
)
|
||||
with pytest.raises(ValueError, match="more than one 80 ms frame"):
|
||||
buffer.prepare_commit(operation_id="commit", chunk_period_ms=80)
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import (
|
||||
NemotronVoiceChatDuplexRuntimeExtension,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import (
|
||||
NemotronVoiceChatServingRuntimeAdapter,
|
||||
_render_tool_response,
|
||||
)
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
|
||||
NemotronVoiceChatThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _frame() -> dict[str, object]:
|
||||
raw = np.zeros(1280, dtype=np.float32).tobytes()
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(raw).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
|
||||
|
||||
def _plan(extension, *, input_seq: int):
|
||||
return extension.plan_append(
|
||||
request_id="req",
|
||||
fence=DuplexFence("sid", incarnation=2, epoch=3),
|
||||
session_config={},
|
||||
runtime_config={
|
||||
"nvc_prompt_token_ids": [0, 42, 1],
|
||||
"nvc_text_pad_id": 12,
|
||||
"nvc_max_model_len": 8192,
|
||||
},
|
||||
seq=input_seq,
|
||||
turn_seq=input_seq,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload=_frame(),
|
||||
final=False,
|
||||
sampling_params=SamplingParams(),
|
||||
)
|
||||
|
||||
|
||||
def test_first_append_prefills_prompt_then_each_append_consumes_one_frame() -> None:
|
||||
extension = NemotronVoiceChatDuplexRuntimeExtension()
|
||||
|
||||
params = extension.configure_sampling_params(
|
||||
runtime_config={}, defaults=(SamplingParams(temperature=0.7, max_tokens=4),)
|
||||
)[0]
|
||||
assert params.temperature == 0.0 and params.max_tokens == 1 and params.top_p == 1.0 and params.top_k == 0
|
||||
|
||||
first = _plan(extension, input_seq=1)
|
||||
later = _plan(extension, input_seq=2)
|
||||
|
||||
assert first.prompt["prompt_token_ids"] == [0, 42, 1, 12]
|
||||
assert later.prompt["prompt_token_ids"] == [12]
|
||||
assert first.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 1
|
||||
assert later.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 2
|
||||
|
||||
|
||||
def test_append_rejects_stage0_context_overflow() -> None:
|
||||
with pytest.raises(ValueError, match="max_model_len"):
|
||||
_plan(NemotronVoiceChatDuplexRuntimeExtension(), input_seq=8190)
|
||||
|
||||
|
||||
def test_function_output_becomes_versioned_nvidia_channel_tokens() -> None:
|
||||
encoded: list[str] = []
|
||||
adapter = NemotronVoiceChatServingRuntimeAdapter(lambda *_: None)
|
||||
adapter._tokenizer = SimpleNamespace(
|
||||
encode=lambda text, **_kwargs: encoded.append(text) or [31, 32, 33],
|
||||
)
|
||||
|
||||
first = adapter.runtime_config_for_function_output(
|
||||
{"type": "function_call_output", "call_id": "call-1", "output": '{"result":20}'},
|
||||
{},
|
||||
)
|
||||
second = adapter.runtime_config_for_function_output(
|
||||
{"type": "function_call_output", "call_id": "call-2", "output": "plain text"},
|
||||
first,
|
||||
)
|
||||
|
||||
assert _render_tool_response('{"result":20}') == '<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>'
|
||||
assert encoded == [
|
||||
'<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>',
|
||||
'<TOOL_RESPONSE>["plain text"]</TOOL_RESPONSE>',
|
||||
]
|
||||
assert first["nvc_function_response_generation"] == 1
|
||||
assert second["nvc_function_response_generation"] == 2
|
||||
assert second["nvc_function_response_token_ids"] == [31, 32, 33]
|
||||
assert second["nvc_function_response_call_id"] == "call-2"
|
||||
assert second["nvc_function_response_batches"] == [
|
||||
{"generation": 1, "call_id": "call-1", "token_ids": [31, 32, 33]},
|
||||
{"generation": 2, "call_id": "call-2", "token_ids": [31, 32, 33]},
|
||||
]
|
||||
|
||||
session = {"function_response_generation": 0, "forced_function_tokens": []}
|
||||
NemotronVoiceChatThinkerForConditionalGeneration._sync_forced_function_response(session, second)
|
||||
|
||||
assert session["function_response_generation"] == 2
|
||||
assert session["forced_function_token"] == 31
|
||||
assert session["forced_function_tokens"] == [31, 32, 33, 31, 32, 33]
|
||||
@@ -0,0 +1,364 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
import wave
|
||||
from collections.abc import Sequence
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
import numpy as np
|
||||
from scipy.signal import resample_poly
|
||||
|
||||
from vllm_omni.experimental.fullduplex.client import RealtimeDuplexClient, wait_for, write_pcm16_wav
|
||||
|
||||
INPUT_SAMPLE_RATE_HZ = 16_000
|
||||
OUTPUT_SAMPLE_RATE_HZ = 22_050
|
||||
FRAME_SAMPLES = 1_280
|
||||
FRAME_PERIOD_S = FRAME_SAMPLES / INPUT_SAMPLE_RATE_HZ
|
||||
DEFAULT_FUNCTION_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "generate_random_number",
|
||||
"description": "Generate a random integer between min and max (inclusive).",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"min": {"type": "integer", "description": "Minimum value (inclusive)"},
|
||||
"max": {"type": "integer", "description": "Maximum value (inclusive)"},
|
||||
},
|
||||
"required": ["min", "max"],
|
||||
},
|
||||
}
|
||||
]
|
||||
DEFAULT_INSTRUCTIONS = "You are NVIDIA Voice Chat. Answer briefly. Start by greeting the user."
|
||||
DEFAULT_FUNCTION_INSTRUCTIONS = (
|
||||
"You are NVIDIA Voice Chat. If the user's request matches an available tool, "
|
||||
"you MUST call that tool instead of answering from your own knowledge. "
|
||||
"Use only argument values spoken by the user and never invent missing values."
|
||||
)
|
||||
|
||||
|
||||
def _url(base_url: str, model: str, session_id: str) -> str:
|
||||
parts = urlsplit(base_url)
|
||||
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
query.update(duplex="1", model=model, autostart="0", session_id=session_id)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
|
||||
|
||||
|
||||
def _read_wav(path: Path, *, input_channel: int = 0) -> np.ndarray:
|
||||
with wave.open(str(path), "rb") as wav_file:
|
||||
channels = wav_file.getnchannels()
|
||||
width = wav_file.getsampwidth()
|
||||
source_rate = wav_file.getframerate()
|
||||
raw = wav_file.readframes(wav_file.getnframes())
|
||||
dtypes = {1: np.uint8, 2: np.dtype("<i2"), 4: np.dtype("<i4")}
|
||||
if width not in dtypes:
|
||||
raise ValueError(f"unsupported input sample width: {width}")
|
||||
pcm = np.frombuffer(raw, dtype=dtypes[width]).astype(np.float32)
|
||||
pcm = (pcm - 128.0) / 128.0 if width == 1 else pcm / float(1 << (width * 8 - 1))
|
||||
if not 0 <= input_channel < channels:
|
||||
raise ValueError(f"input channel {input_channel} is outside WAV channel count {channels}")
|
||||
if channels > 1:
|
||||
pcm = pcm.reshape(-1, channels)[:, input_channel]
|
||||
if source_rate != INPUT_SAMPLE_RATE_HZ:
|
||||
divisor = math.gcd(source_rate, INPUT_SAMPLE_RATE_HZ)
|
||||
pcm = resample_poly(pcm, up=INPUT_SAMPLE_RATE_HZ // divisor, down=source_rate // divisor)
|
||||
return np.ascontiguousarray(pcm, dtype="<f4")
|
||||
|
||||
|
||||
async def _stream(client: RealtimeDuplexClient, pcm: np.ndarray, *, max_frames: int | None, realtime: bool) -> int:
|
||||
count = math.ceil(pcm.size / FRAME_SAMPLES)
|
||||
if max_frames is not None:
|
||||
count = min(count, max_frames)
|
||||
for seq in range(count):
|
||||
frame = pcm[seq * FRAME_SAMPLES : (seq + 1) * FRAME_SAMPLES]
|
||||
frame = np.pad(frame, (0, FRAME_SAMPLES - frame.size)).astype("<f4")
|
||||
await client.send(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(frame).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": INPUT_SAMPLE_RATE_HZ,
|
||||
"duration_ms": 80,
|
||||
"audio_end_ms": (seq + 1) * 80,
|
||||
}
|
||||
)
|
||||
if realtime:
|
||||
await asyncio.sleep(FRAME_PERIOD_S)
|
||||
return count
|
||||
|
||||
|
||||
def _events(client: RealtimeDuplexClient, event_type: str) -> list[dict[str, object]]:
|
||||
return [event for event in client.events.events if event.get("type") == event_type]
|
||||
|
||||
|
||||
async def _return_function_output_when_ready(
|
||||
client: RealtimeDuplexClient,
|
||||
*,
|
||||
output: str,
|
||||
timeout_s: float,
|
||||
) -> tuple[str, int]:
|
||||
await wait_for(
|
||||
lambda: bool(client.events.errors()) or client.events.count("response.function_call_arguments.done") > 0,
|
||||
timeout_s=timeout_s,
|
||||
label="function call to execute",
|
||||
)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"function call failed before tool execution: {client.events.errors()}")
|
||||
function_done = _events(client, "response.function_call_arguments.done")[-1]
|
||||
call_id = function_done.get("call_id")
|
||||
if not isinstance(call_id, str) or not call_id:
|
||||
raise AssertionError(f"completed function call has no call_id: {function_done}")
|
||||
event_count_before_output = len(client.events.events)
|
||||
await client.send(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "function_call_output",
|
||||
"call_id": call_id,
|
||||
"output": output,
|
||||
},
|
||||
}
|
||||
)
|
||||
return call_id, event_count_before_output
|
||||
|
||||
|
||||
def _write_events(path: Path, client: RealtimeDuplexClient) -> None:
|
||||
path.write_text(
|
||||
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _managed_client(client: RealtimeDuplexClient, *, timeout_s: float):
|
||||
async with client:
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if client.events.count("session.created") and not client.events.count("session.closed"):
|
||||
with suppress(Exception):
|
||||
await client.close_session(timeout_s=min(timeout_s, 30.0))
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> dict[str, object]:
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
if args.expect_function_call and args.instructions == DEFAULT_INSTRUCTIONS:
|
||||
instructions = DEFAULT_FUNCTION_INSTRUCTIONS
|
||||
else:
|
||||
instructions = args.instructions
|
||||
tools = DEFAULT_FUNCTION_TOOLS if args.expect_function_call else None
|
||||
session_id = f"nemotron-voicechat-{uuid.uuid4().hex}"
|
||||
client = RealtimeDuplexClient(_url(args.url, args.model, session_id))
|
||||
async with _managed_client(client, timeout_s=args.timeout_s):
|
||||
session_payload: dict[str, object] = {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"input_audio_format": "pcm_f32le",
|
||||
"output_audio_format": "pcm16",
|
||||
"instructions": instructions,
|
||||
"idle_timeout_s": args.timeout_s,
|
||||
"turn_detection": None,
|
||||
"extra_body": {"auto_response": True},
|
||||
}
|
||||
if tools is not None:
|
||||
session_payload["tools"] = tools
|
||||
await client.send({"type": "session.update", "session": session_payload})
|
||||
await wait_for(
|
||||
lambda: client.events.count("session.created") > 0 or bool(client.events.errors()),
|
||||
timeout_s=args.timeout_s,
|
||||
label="session.created",
|
||||
)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"session setup failed: {client.events.errors()}")
|
||||
created = _events(client, "session.created")[-1]
|
||||
session = created.get("session")
|
||||
capabilities = session.get("capabilities") if isinstance(session, dict) else None
|
||||
expected = dict(
|
||||
implementation_level="model_native_duplex",
|
||||
chunk_period_ms=80,
|
||||
supports_core_resumable_request=True,
|
||||
supports_core_kv_lease=False,
|
||||
supports_multi_session=False,
|
||||
)
|
||||
if not isinstance(capabilities, dict) or any(capabilities.get(key) != value for key, value in expected.items()):
|
||||
raise AssertionError(f"unexpected capabilities: {capabilities}")
|
||||
|
||||
function_output_task = (
|
||||
asyncio.create_task(
|
||||
_return_function_output_when_ready(
|
||||
client,
|
||||
output=args.function_output,
|
||||
timeout_s=args.timeout_s,
|
||||
)
|
||||
)
|
||||
if args.function_output is not None
|
||||
else None
|
||||
)
|
||||
pcm = _read_wav(Path(args.input_wav), input_channel=args.input_channel)
|
||||
frame_count = await _stream(client, pcm, max_frames=args.max_frames, realtime=not args.no_realtime)
|
||||
completed_responses_at_commit = client.events.count("response.done")
|
||||
await client.send({"type": "input_audio_buffer.commit", "final": True})
|
||||
await wait_for(
|
||||
lambda: (
|
||||
bool(client.events.errors())
|
||||
or (
|
||||
client.events.count("response.function_call_arguments.done") > 0
|
||||
if args.expect_function_call
|
||||
else (
|
||||
client.events.count("response.audio.delta") >= args.minimum_audio_chunks
|
||||
and (client.events.count("response.done") > completed_responses_at_commit)
|
||||
)
|
||||
)
|
||||
),
|
||||
timeout_s=args.timeout_s,
|
||||
label="model output",
|
||||
)
|
||||
await asyncio.sleep(args.drain_s)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"Realtime session emitted errors: {client.events.errors()}")
|
||||
done_events = _events(client, "response.done")
|
||||
if not args.expect_function_call:
|
||||
response = done_events[-1].get("response") if done_events else None
|
||||
status = response.get("status") if isinstance(response, dict) else None
|
||||
if status != "completed":
|
||||
raise AssertionError(f"response did not complete successfully: {done_events[-1:]}")
|
||||
|
||||
function_events = [
|
||||
event for event in client.events.events if str(event.get("type", "")).startswith("response.function_call")
|
||||
]
|
||||
function_items = [
|
||||
event
|
||||
for event in _events(client, "response.output_item.done")
|
||||
if isinstance(event.get("item"), dict) and event["item"].get("type") == "function_call"
|
||||
]
|
||||
if args.expect_function_call and not any(
|
||||
event.get("type") == "response.function_call_arguments.done" for event in function_events
|
||||
):
|
||||
raise AssertionError(f"no completed function call: {function_events}")
|
||||
if args.expect_function_call:
|
||||
matching_items = [
|
||||
event for event in function_items if event["item"].get("name") == args.expected_function_name
|
||||
]
|
||||
if not matching_items:
|
||||
raise AssertionError(f"expected {args.expected_function_name!r}, got {function_items}")
|
||||
function_item = matching_items[-1]["item"]
|
||||
try:
|
||||
function_arguments = json.loads(str(function_item.get("arguments", "")))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AssertionError(f"function arguments are not JSON: {function_item}") from exc
|
||||
if args.expected_function_arguments is not None:
|
||||
expected_arguments = json.loads(args.expected_function_arguments)
|
||||
if function_arguments != expected_arguments:
|
||||
raise AssertionError(
|
||||
f"function arguments differ: expected={expected_arguments!r}, actual={function_arguments!r}"
|
||||
)
|
||||
|
||||
if args.function_output is not None:
|
||||
call_id = function_item.get("call_id")
|
||||
if not isinstance(call_id, str) or not call_id:
|
||||
raise AssertionError(f"function item has no call_id: {function_item}")
|
||||
assert function_output_task is not None
|
||||
returned_call_id, event_count_before_output = await function_output_task
|
||||
assert returned_call_id == call_id
|
||||
|
||||
def tool_result_completed() -> bool:
|
||||
later = client.events.events[event_count_before_output:]
|
||||
transcript = "".join(
|
||||
str(event.get("delta", ""))
|
||||
for event in later
|
||||
if event.get("type") == "response.audio_transcript.delta"
|
||||
).lower()
|
||||
return (
|
||||
any(event.get("type") == "response.audio.delta" for event in later)
|
||||
and any(event.get("type") == "response.done" for event in later)
|
||||
and (args.expected_post_tool_text is None or args.expected_post_tool_text.lower() in transcript)
|
||||
)
|
||||
|
||||
await wait_for(
|
||||
lambda: bool(client.events.errors()) or tool_result_completed(),
|
||||
timeout_s=args.timeout_s,
|
||||
label="completed response after function output",
|
||||
)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"function output failed: {client.events.errors()}")
|
||||
await asyncio.sleep(args.drain_s)
|
||||
|
||||
audio = client.events.audio_bytes()
|
||||
audio_events = _events(client, "response.audio.delta")
|
||||
rates = {event.get("sample_rate_hz") for event in audio_events}
|
||||
if not args.expect_function_call and audio and rates != {OUTPUT_SAMPLE_RATE_HZ}:
|
||||
raise AssertionError(f"unexpected output sample rates: {rates}")
|
||||
if not args.expect_function_call and args.minimum_audio_chunks and not audio:
|
||||
raise AssertionError("model produced no audio")
|
||||
expected_bytes = 2 * OUTPUT_SAMPLE_RATE_HZ * expected["chunk_period_ms"] // 1000
|
||||
packet_sizes = [len(base64.b64decode(str(event.get("delta", "")), validate=True)) for event in audio_events]
|
||||
if not args.expect_function_call and any(size != expected_bytes for size in packet_sizes):
|
||||
raise AssertionError(f"audio deltas are not fixed 80 ms PCM16 packets: {packet_sizes}")
|
||||
audio_pcm = np.frombuffer(audio, dtype="<i2").astype(np.float32) / 32768.0
|
||||
audio_rms = float(np.sqrt(np.mean(np.square(audio_pcm)))) if audio_pcm.size else 0.0
|
||||
if not args.expect_function_call and audio and audio_rms < args.minimum_audio_rms:
|
||||
raise AssertionError(
|
||||
f"model output RMS {audio_rms:.6f} is below {args.minimum_audio_rms:.6f}; "
|
||||
"received packets contain only silence"
|
||||
)
|
||||
_write_events(output_dir / "events.jsonl", client)
|
||||
if audio:
|
||||
write_pcm16_wav(output_dir / "output.wav", audio, sample_rate_hz=OUTPUT_SAMPLE_RATE_HZ)
|
||||
result = {
|
||||
"ok": True,
|
||||
"session_id": session_id,
|
||||
"input_frames": frame_count,
|
||||
"capabilities": capabilities,
|
||||
"event_counts": {
|
||||
event_type: client.events.count(event_type)
|
||||
for event_type in sorted({str(event.get("type")) for event in client.events.events})
|
||||
},
|
||||
"audio_bytes": len(audio),
|
||||
"audio_rms": audio_rms,
|
||||
"function_events": function_events,
|
||||
"function_items": function_items,
|
||||
"output_dir": str(output_dir),
|
||||
}
|
||||
|
||||
(output_dir / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return result
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:8125/v1/realtime")
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--input-wav", required=True)
|
||||
parser.add_argument("--input-channel", type=int, default=0)
|
||||
parser.add_argument("--output-dir", default="/tmp/nemotron-voicechat-duplex")
|
||||
parser.add_argument("--instructions", default=DEFAULT_INSTRUCTIONS)
|
||||
parser.add_argument("--max-frames", type=int)
|
||||
parser.add_argument("--minimum-audio-chunks", type=int, default=1)
|
||||
parser.add_argument("--minimum-audio-rms", type=float, default=1e-4)
|
||||
parser.add_argument("--expect-function-call", action="store_true")
|
||||
parser.add_argument("--expected-function-name", default="generate_random_number")
|
||||
parser.add_argument("--expected-function-arguments")
|
||||
parser.add_argument("--function-output")
|
||||
parser.add_argument("--expected-post-tool-text")
|
||||
parser.add_argument("--no-realtime", action="store_true")
|
||||
parser.add_argument("--drain-s", type=float, default=2.0)
|
||||
parser.add_argument("--timeout-s", type=float, default=600.0)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(json.dumps(asyncio.run(run(parse_args())), ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -52,9 +52,8 @@ def get_prompt(prompt_type="zh"):
|
||||
|
||||
def _set_rocm_request_timeout(request_config: dict) -> None:
|
||||
if current_omni_platform.is_rocm():
|
||||
# A cold AITER MHA kernel build took 301 seconds on MI300X.
|
||||
# Allow 600 seconds for the first request; later requests reuse the cache.
|
||||
request_config["timeout"] = 600.0
|
||||
# Cold AITER MHA builds on loaded MI300 agents can exceed 20 minutes.
|
||||
request_config["timeout"] = 1800.0
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from tests.e2e.online_serving.nemotron_voicechat_realtime_duplex import parse_args, run
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams, get_model_prefix
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_MODEL", "nvidia/NVIDIA-NemotronLabs-VoiceChat-11B")
|
||||
DEPLOY_CONFIG = get_deploy_config_path("nemotron_labs_voicechat_duplex.yaml")
|
||||
TOKENIZER = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_LLM_PATH")
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
[
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=DEPLOY_CONFIG,
|
||||
env_dict={"NEMOTRON_VOICECHAT_LLM_PATH": TOKENIZER} if TOKENIZER else None,
|
||||
),
|
||||
id="native-duplex",
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_native_duplex_turn_taking_streams_model_audio(omni_server, tmp_path: Path) -> None:
|
||||
model_prefix = get_model_prefix()
|
||||
root = Path(model_prefix) / MODEL if model_prefix else Path(MODEL)
|
||||
if not root.is_dir():
|
||||
root = Path(snapshot_download(MODEL, local_files_only=True))
|
||||
args = parse_args(
|
||||
(
|
||||
f"--url ws://{omni_server.host}:{omni_server.port}/v1/realtime --model {omni_server.model} "
|
||||
f"--input-wav {root / 'turn_taking.wav'} --input-channel 0 --max-frames 190 "
|
||||
f"--minimum-audio-chunks 48 --minimum-audio-rms 0.001 --no-realtime --timeout-s 300 "
|
||||
f"--output-dir {tmp_path / 'native_duplex'}"
|
||||
).split()
|
||||
)
|
||||
result = asyncio.run(run(args))
|
||||
assert result["ok"] is True
|
||||
assert result["input_frames"] == 190
|
||||
assert result["event_counts"]["response.speak"] > 0
|
||||
assert result["audio_bytes"] >= result["input_frames"] * 1764 * 2 // 4
|
||||
Reference in new issue
Block a user