[Model] Add native full-duplex Nemotron VoiceChat serving (#6089)
Signed-off-by: Sysy <1370724210@qq.com> Signed-off-by: sy03 <137072421@qq.com> Co-authored-by: sy03 <137072421@qq.com>
This commit is contained in:
1 parent
c2723979c0
commit
432b2c6ca3
40 files changed
+3560
-154
No files matched your search
@@ -198,6 +198,7 @@ def test_resumable_segment_boundary_keeps_pre_transition_send_watermark() -> Non
|
||||
inter_stage_output,
|
||||
session,
|
||||
True,
|
||||
new_token_ids=[42],
|
||||
confirmed_num_computed_tokens=26,
|
||||
)
|
||||
|
||||
@@ -427,3 +428,33 @@ def test_ready_async_chunk_prompt_replacement_releases_stale_kv_once() -> None:
|
||||
assert sched.chunk_transfer_adapter.replaced_streaming_prompt_ids == set()
|
||||
assert sched.chunk_transfer_adapter.requests_with_ready_chunks == {session.request_id}
|
||||
assert sched.chunk_transfer_adapter.requests_num_chunks_sent == {}
|
||||
|
||||
|
||||
def test_chunk_segment_cleanup_keeps_requeued_resumable_receiver() -> None:
|
||||
"""A WAITING_FOR_CHUNK stop must not delete its newly parked session."""
|
||||
session = _make_request()
|
||||
session.resumable = True
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
|
||||
def queue(*requests):
|
||||
result = MagicMock()
|
||||
result.requests = set(requests)
|
||||
result.add_request.side_effect = result.requests.add
|
||||
result.remove_requests.side_effect = result.requests.difference_update
|
||||
return result
|
||||
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched.running = []
|
||||
sched.waiting = queue()
|
||||
sched.skipped_waiting = queue(session)
|
||||
sched.num_waiting_for_streaming_input = 1
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(segment_finished_requests={session.request_id})
|
||||
|
||||
sched._resume_downstream_chunk_receiver(session)
|
||||
sched._remove_stopped_requests_from_queues(set(), {session})
|
||||
|
||||
assert session.status == RequestStatus.WAITING
|
||||
assert session in sched.waiting.requests
|
||||
assert session not in sched.skipped_waiting.requests
|
||||
assert sched.num_waiting_for_streaming_input == 0
|
||||
assert session.request_id not in sched.chunk_transfer_adapter.segment_finished_requests
|
||||
@@ -1209,6 +1209,67 @@ def test_cleanup_preserves_pending_save(build_adapter):
|
||||
assert len(adapter._pending_save_reqs) == 1
|
||||
|
||||
|
||||
def test_abort_invalidates_queued_sender_task_before_external_id_reuse(build_adapter):
|
||||
adapter, connector = build_adapter(stage_id=1)
|
||||
request = _req("req-old", RequestStatus.WAITING, external_req_id="ext-reused")
|
||||
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct(
|
||||
codes=CodesStruct(audio=torch.tensor([1], dtype=torch.long))
|
||||
)
|
||||
|
||||
adapter.save_async(multimodal_output=None, request=request)
|
||||
stale_task = adapter._pending_save_reqs.popleft()
|
||||
adapter.cleanup(request.request_id, request.external_req_id)
|
||||
adapter._send_single_request(stale_task)
|
||||
|
||||
connector.put.assert_not_called()
|
||||
assert request.external_req_id not in adapter.put_req_chunk
|
||||
|
||||
replacement = _req("req-new", RequestStatus.WAITING, external_req_id=request.external_req_id)
|
||||
adapter.save_async(multimodal_output=None, request=replacement)
|
||||
current_task = adapter._pending_save_reqs.popleft()
|
||||
adapter._send_single_request(current_task)
|
||||
|
||||
connector.put.assert_called_once()
|
||||
assert connector.put.call_args.kwargs["put_key"] == "ext-reused_1_0"
|
||||
|
||||
|
||||
def test_finish_requests_does_not_wait_for_inflight_send(build_adapter):
|
||||
adapter, connector = build_adapter(stage_id=1)
|
||||
first = _req("req-first", RequestStatus.WAITING, external_req_id="ext-first")
|
||||
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct()
|
||||
put_started = threading.Event()
|
||||
release_put = threading.Event()
|
||||
|
||||
def blocking_put(**kwargs):
|
||||
put_started.set()
|
||||
release_put.wait(timeout=2)
|
||||
return True, 1, {}
|
||||
|
||||
connector.put.side_effect = blocking_put
|
||||
adapter.save_async(multimodal_output=None, request=first)
|
||||
task = adapter._pending_save_reqs.popleft()
|
||||
sender = threading.Thread(target=adapter._send_single_request, args=(task,))
|
||||
sender.start()
|
||||
assert put_started.wait(timeout=1)
|
||||
cleaner = threading.Thread(
|
||||
target=adapter.finish_requests,
|
||||
args=([first.request_id], RequestStatus.FINISHED_ABORTED, {first.request_id: first}),
|
||||
)
|
||||
cleaner.start()
|
||||
try:
|
||||
cleaner.join(timeout=0.5)
|
||||
assert not cleaner.is_alive()
|
||||
assert task["sender_token"].cancelled
|
||||
assert first.external_req_id in adapter._sender_tokens
|
||||
finally:
|
||||
release_put.set()
|
||||
sender.join(timeout=1)
|
||||
cleaner.join(timeout=1)
|
||||
assert not sender.is_alive()
|
||||
assert first.external_req_id not in adapter._sender_tokens
|
||||
assert first.external_req_id not in adapter.put_req_chunk
|
||||
|
||||
|
||||
def test_cleanup_only_affects_target_request(build_adapter):
|
||||
"""Cleanup for one request must not affect another request's state."""
|
||||
adapter, _ = build_adapter(stage_id=1)
|
||||
@@ -2216,3 +2277,43 @@ def test_expiry_is_per_request(build_adapter):
|
||||
|
||||
assert adapter.collect_timed_out_request_ids(timeout_s=600.0) == {"stalled"}
|
||||
assert "healthy" in adapter._waiting_since
|
||||
|
||||
|
||||
def test_abort_clears_native_codec_state_before_external_id_reuse(build_adapter):
|
||||
from vllm_omni.model_executor.stage_input_processors.nemotron_voicechat import (
|
||||
talker2code2wav_async_chunk,
|
||||
)
|
||||
|
||||
adapter, _ = build_adapter(stage_id=1, connector_extra={"codec_chunk_frames": 1})
|
||||
external_req_id = "ext-native-abort"
|
||||
first = _req("req-native-abort", RequestStatus.WAITING, external_req_id=external_req_id)
|
||||
first.resumable = True
|
||||
first.model_intermediate_buffer = None
|
||||
first.additional_information = {"meta": {"codec_streaming": True}, "nvc_logical_prompt_len": 1}
|
||||
first_frame = torch.full((1, 31), 101, dtype=torch.long)
|
||||
|
||||
talker2code2wav_async_chunk(
|
||||
adapter,
|
||||
{"codes": {"audio": first_frame}, "meta": {"codec_streaming": True}},
|
||||
first,
|
||||
is_finished=False,
|
||||
)
|
||||
|
||||
adapter.requests_num_chunks_sent[external_req_id] = 1
|
||||
adapter.finish_requests([first.request_id], RequestStatus.FINISHED_ABORTED, {first.request_id: first})
|
||||
|
||||
assert external_req_id not in adapter.request_payload
|
||||
assert external_req_id not in adapter.requests_num_chunks_sent
|
||||
|
||||
replacement = _req("req-native-abort", RequestStatus.WAITING, external_req_id=external_req_id)
|
||||
replacement.resumable = True
|
||||
replacement.model_intermediate_buffer = None
|
||||
replacement.additional_information = first.additional_information
|
||||
replacement_payload = talker2code2wav_async_chunk(
|
||||
adapter,
|
||||
{"codes": {"audio": torch.full((1, 31), 202, dtype=torch.long)}, "meta": {"codec_streaming": True}},
|
||||
replacement,
|
||||
is_finished=False,
|
||||
)
|
||||
|
||||
assert torch.equal(replacement_payload.codes.audio, torch.full((1, 31), 202, dtype=torch.long))
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import (
|
||||
NemotronVoiceChatDataPlaneContext,
|
||||
NemotronVoiceChatDataPlaneSession,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
_RUNTIME = {
|
||||
"nvc_text_bos_id": 0,
|
||||
"nvc_text_eos_id": 1,
|
||||
"nvc_text_pad_id": 12,
|
||||
"nvc_function_sotc_id": 20,
|
||||
"nvc_function_eotc_id": 21,
|
||||
"nvc_tokenizer_ref": "test-tokenizer",
|
||||
}
|
||||
|
||||
|
||||
def _projector(encode_audio=lambda *_: None) -> NemotronVoiceChatDataPlaneSession:
|
||||
projector = NemotronVoiceChatDataPlaneSession(encode_audio)
|
||||
projector.configure_runtime(_RUNTIME, tokenizer=SimpleNamespace(decode=lambda *_args, **_kwargs: "text"))
|
||||
return projector
|
||||
|
||||
|
||||
def _stage0_output(request_id: str = "req-0", *, text_token: int = 12, function_token: int | None = None) -> object:
|
||||
metadata: dict[str, object] = {"nvc_text_token_ids": [text_token]}
|
||||
if function_token is not None:
|
||||
metadata["nvc_function_token"] = [function_token]
|
||||
completion = SimpleNamespace(multimodal_output=metadata)
|
||||
return SimpleNamespace(
|
||||
stage_id=0,
|
||||
request_output=SimpleNamespace(request_id=request_id, outputs=[completion]),
|
||||
)
|
||||
|
||||
|
||||
def test_function_channel_projects_completed_call_without_ending_speech() -> None:
|
||||
projector = _projector()
|
||||
projector._decode = lambda token_ids: (
|
||||
'[{"name":"weather","arguments":{"city":"Shanghai"}}]' if token_ids == [99] else ""
|
||||
)
|
||||
context = NemotronVoiceChatDataPlaneContext(epoch=0)
|
||||
|
||||
events = []
|
||||
for function_token in (20, 99, 21):
|
||||
events.extend(projector.project_output(_stage0_output(function_token=function_token), context=context))
|
||||
|
||||
listen = [event for event in events if event.get("is_listen") is True]
|
||||
function = [event for event in events if event.get("function_call") is True]
|
||||
assert len(listen) == 3
|
||||
assert len(function) == 1
|
||||
assert function[0]["name"] == "weather"
|
||||
assert function[0]["arguments"] == '{"city":"Shanghai"}'
|
||||
|
||||
projector._decode = lambda _token_ids: "not-json"
|
||||
bad_events = []
|
||||
for function_token in (20, 98, 21):
|
||||
bad_events.extend(projector.project_output(_stage0_output(function_token=function_token)))
|
||||
assert any(event.get("error_code") == "nemotron_function_call_parse_error" for event in bad_events)
|
||||
|
||||
|
||||
def test_new_epoch_request_does_not_inherit_partial_function_or_frame_state() -> None:
|
||||
projector = _projector(lambda *_: "audio")
|
||||
old_request = duplex_resource_request_id(DuplexFence("sid", epoch=1), "stage0")
|
||||
new_request = duplex_resource_request_id(DuplexFence("sid", epoch=2), "stage0")
|
||||
|
||||
projector.begin_request(old_request)
|
||||
list(projector.project_output(_stage0_output(old_request, text_token=42, function_token=20)))
|
||||
projector.begin_request(new_request)
|
||||
|
||||
# EOTC in the new epoch must not close the old epoch's partial function.
|
||||
new_events = list(projector.project_output(_stage0_output(new_request, text_token=1, function_token=21)))
|
||||
assert not [event for event in new_events if event.get("function_call") is True]
|
||||
assert not [event for event in new_events if event.get("end_of_turn") is True]
|
||||
|
||||
audio = SimpleNamespace(
|
||||
stage_id=2,
|
||||
request_output=SimpleNamespace(
|
||||
request_id=new_request,
|
||||
outputs=[
|
||||
SimpleNamespace(multimodal_output={"model_outputs": [np.ones(1764, dtype=np.float32)], "sr": [22050]})
|
||||
],
|
||||
),
|
||||
)
|
||||
audio_events = list(projector.project_output(audio))
|
||||
assert sum(event.get("end_of_turn") is True for event in audio_events) == 1
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import (
|
||||
NemotronVoiceChatPcmAppendBuffer,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _payload(samples: np.ndarray, *, sample_rate_hz: int = 16000, fmt: str = "pcm_f32le") -> dict[str, object]:
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(np.asarray(samples, dtype=np.float32).tobytes()).decode("ascii"),
|
||||
"format": fmt,
|
||||
"sample_rate_hz": sample_rate_hz,
|
||||
}
|
||||
|
||||
|
||||
def _append(buffer, samples, operation_id):
|
||||
return buffer.prepare_append(
|
||||
_payload(np.asarray(samples, dtype=np.float32)),
|
||||
operation_id=operation_id,
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
|
||||
def test_irregular_browser_packets_emit_exact_ordered_80ms_frame() -> None:
|
||||
buffer = NemotronVoiceChatPcmAppendBuffer()
|
||||
assert _append(buffer, np.arange(1000), "packet-1") is None
|
||||
|
||||
reservation = _append(buffer, np.arange(1000, 1280), "packet-2")
|
||||
|
||||
assert reservation is not None
|
||||
assert reservation.operation_id == "packet-2"
|
||||
decoded = np.frombuffer(base64.b64decode(reservation.payload["audio"]), dtype=np.float32)
|
||||
np.testing.assert_array_equal(decoded, np.arange(1280, dtype=np.float32))
|
||||
assert buffer.pending_byte_count == 0
|
||||
|
||||
|
||||
def test_append_rejects_invalid_frame_contract() -> None:
|
||||
cases = (
|
||||
(1281, 16000, "pcm_f32le", 0.0, "at most 1280"),
|
||||
(1, 8000, "pcm_f32le", 0.0, "sample_rate_hz"),
|
||||
(1, 16000, "audio/pcm", 0.0, "format"),
|
||||
(1, 16000, "pcm_f32le", np.nan, "finite"),
|
||||
)
|
||||
for samples, sample_rate_hz, fmt, value, message in cases:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
NemotronVoiceChatPcmAppendBuffer().prepare_append(
|
||||
_payload(np.full(samples, value, dtype=np.float32), sample_rate_hz=sample_rate_hz, fmt=fmt),
|
||||
operation_id="invalid",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_commit_rejects_more_than_one_frame_tail() -> None:
|
||||
buffer = NemotronVoiceChatPcmAppendBuffer()
|
||||
for index, samples in enumerate((1000, 281)):
|
||||
buffer.prepare_append(
|
||||
_payload(np.zeros(samples, dtype=np.float32)),
|
||||
operation_id=f"tail-{index}",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=False,
|
||||
)
|
||||
with pytest.raises(ValueError, match="more than one 80 ms frame"):
|
||||
buffer.prepare_commit(operation_id="commit", chunk_period_ms=80)
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import (
|
||||
NemotronVoiceChatDuplexRuntimeExtension,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import (
|
||||
NemotronVoiceChatServingRuntimeAdapter,
|
||||
_render_tool_response,
|
||||
)
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
|
||||
NemotronVoiceChatThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _frame() -> dict[str, object]:
|
||||
raw = np.zeros(1280, dtype=np.float32).tobytes()
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(raw).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
|
||||
|
||||
def _plan(extension, *, input_seq: int):
|
||||
return extension.plan_append(
|
||||
request_id="req",
|
||||
fence=DuplexFence("sid", incarnation=2, epoch=3),
|
||||
session_config={},
|
||||
runtime_config={
|
||||
"nvc_prompt_token_ids": [0, 42, 1],
|
||||
"nvc_text_pad_id": 12,
|
||||
"nvc_max_model_len": 8192,
|
||||
},
|
||||
seq=input_seq,
|
||||
turn_seq=input_seq,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload=_frame(),
|
||||
final=False,
|
||||
sampling_params=SamplingParams(),
|
||||
)
|
||||
|
||||
|
||||
def test_first_append_prefills_prompt_then_each_append_consumes_one_frame() -> None:
|
||||
extension = NemotronVoiceChatDuplexRuntimeExtension()
|
||||
|
||||
params = extension.configure_sampling_params(
|
||||
runtime_config={}, defaults=(SamplingParams(temperature=0.7, max_tokens=4),)
|
||||
)[0]
|
||||
assert params.temperature == 0.0 and params.max_tokens == 1 and params.top_p == 1.0 and params.top_k == 0
|
||||
|
||||
first = _plan(extension, input_seq=1)
|
||||
later = _plan(extension, input_seq=2)
|
||||
|
||||
assert first.prompt["prompt_token_ids"] == [0, 42, 1, 12]
|
||||
assert later.prompt["prompt_token_ids"] == [12]
|
||||
assert first.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 1
|
||||
assert later.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 2
|
||||
|
||||
|
||||
def test_append_rejects_stage0_context_overflow() -> None:
|
||||
with pytest.raises(ValueError, match="max_model_len"):
|
||||
_plan(NemotronVoiceChatDuplexRuntimeExtension(), input_seq=8190)
|
||||
|
||||
|
||||
def test_function_output_becomes_versioned_nvidia_channel_tokens() -> None:
|
||||
encoded: list[str] = []
|
||||
adapter = NemotronVoiceChatServingRuntimeAdapter(lambda *_: None)
|
||||
adapter._tokenizer = SimpleNamespace(
|
||||
encode=lambda text, **_kwargs: encoded.append(text) or [31, 32, 33],
|
||||
)
|
||||
|
||||
first = adapter.runtime_config_for_function_output(
|
||||
{"type": "function_call_output", "call_id": "call-1", "output": '{"result":20}'},
|
||||
{},
|
||||
)
|
||||
second = adapter.runtime_config_for_function_output(
|
||||
{"type": "function_call_output", "call_id": "call-2", "output": "plain text"},
|
||||
first,
|
||||
)
|
||||
|
||||
assert _render_tool_response('{"result":20}') == '<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>'
|
||||
assert encoded == [
|
||||
'<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>',
|
||||
'<TOOL_RESPONSE>["plain text"]</TOOL_RESPONSE>',
|
||||
]
|
||||
assert first["nvc_function_response_generation"] == 1
|
||||
assert second["nvc_function_response_generation"] == 2
|
||||
assert second["nvc_function_response_token_ids"] == [31, 32, 33]
|
||||
assert second["nvc_function_response_call_id"] == "call-2"
|
||||
assert second["nvc_function_response_batches"] == [
|
||||
{"generation": 1, "call_id": "call-1", "token_ids": [31, 32, 33]},
|
||||
{"generation": 2, "call_id": "call-2", "token_ids": [31, 32, 33]},
|
||||
]
|
||||
|
||||
session = {"function_response_generation": 0, "forced_function_tokens": []}
|
||||
NemotronVoiceChatThinkerForConditionalGeneration._sync_forced_function_response(session, second)
|
||||
|
||||
assert session["function_response_generation"] == 2
|
||||
assert session["forced_function_token"] == 31
|
||||
assert session["forced_function_tokens"] == [31, 32, 33, 31, 32, 33]
|
||||
@@ -0,0 +1,364 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
import wave
|
||||
from collections.abc import Sequence
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
import numpy as np
|
||||
from scipy.signal import resample_poly
|
||||
|
||||
from vllm_omni.experimental.fullduplex.client import RealtimeDuplexClient, wait_for, write_pcm16_wav
|
||||
|
||||
INPUT_SAMPLE_RATE_HZ = 16_000
|
||||
OUTPUT_SAMPLE_RATE_HZ = 22_050
|
||||
FRAME_SAMPLES = 1_280
|
||||
FRAME_PERIOD_S = FRAME_SAMPLES / INPUT_SAMPLE_RATE_HZ
|
||||
DEFAULT_FUNCTION_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "generate_random_number",
|
||||
"description": "Generate a random integer between min and max (inclusive).",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"min": {"type": "integer", "description": "Minimum value (inclusive)"},
|
||||
"max": {"type": "integer", "description": "Maximum value (inclusive)"},
|
||||
},
|
||||
"required": ["min", "max"],
|
||||
},
|
||||
}
|
||||
]
|
||||
DEFAULT_INSTRUCTIONS = "You are NVIDIA Voice Chat. Answer briefly. Start by greeting the user."
|
||||
DEFAULT_FUNCTION_INSTRUCTIONS = (
|
||||
"You are NVIDIA Voice Chat. If the user's request matches an available tool, "
|
||||
"you MUST call that tool instead of answering from your own knowledge. "
|
||||
"Use only argument values spoken by the user and never invent missing values."
|
||||
)
|
||||
|
||||
|
||||
def _url(base_url: str, model: str, session_id: str) -> str:
|
||||
parts = urlsplit(base_url)
|
||||
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
query.update(duplex="1", model=model, autostart="0", session_id=session_id)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
|
||||
|
||||
|
||||
def _read_wav(path: Path, *, input_channel: int = 0) -> np.ndarray:
|
||||
with wave.open(str(path), "rb") as wav_file:
|
||||
channels = wav_file.getnchannels()
|
||||
width = wav_file.getsampwidth()
|
||||
source_rate = wav_file.getframerate()
|
||||
raw = wav_file.readframes(wav_file.getnframes())
|
||||
dtypes = {1: np.uint8, 2: np.dtype("<i2"), 4: np.dtype("<i4")}
|
||||
if width not in dtypes:
|
||||
raise ValueError(f"unsupported input sample width: {width}")
|
||||
pcm = np.frombuffer(raw, dtype=dtypes[width]).astype(np.float32)
|
||||
pcm = (pcm - 128.0) / 128.0 if width == 1 else pcm / float(1 << (width * 8 - 1))
|
||||
if not 0 <= input_channel < channels:
|
||||
raise ValueError(f"input channel {input_channel} is outside WAV channel count {channels}")
|
||||
if channels > 1:
|
||||
pcm = pcm.reshape(-1, channels)[:, input_channel]
|
||||
if source_rate != INPUT_SAMPLE_RATE_HZ:
|
||||
divisor = math.gcd(source_rate, INPUT_SAMPLE_RATE_HZ)
|
||||
pcm = resample_poly(pcm, up=INPUT_SAMPLE_RATE_HZ // divisor, down=source_rate // divisor)
|
||||
return np.ascontiguousarray(pcm, dtype="<f4")
|
||||
|
||||
|
||||
async def _stream(client: RealtimeDuplexClient, pcm: np.ndarray, *, max_frames: int | None, realtime: bool) -> int:
|
||||
count = math.ceil(pcm.size / FRAME_SAMPLES)
|
||||
if max_frames is not None:
|
||||
count = min(count, max_frames)
|
||||
for seq in range(count):
|
||||
frame = pcm[seq * FRAME_SAMPLES : (seq + 1) * FRAME_SAMPLES]
|
||||
frame = np.pad(frame, (0, FRAME_SAMPLES - frame.size)).astype("<f4")
|
||||
await client.send(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(frame).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": INPUT_SAMPLE_RATE_HZ,
|
||||
"duration_ms": 80,
|
||||
"audio_end_ms": (seq + 1) * 80,
|
||||
}
|
||||
)
|
||||
if realtime:
|
||||
await asyncio.sleep(FRAME_PERIOD_S)
|
||||
return count
|
||||
|
||||
|
||||
def _events(client: RealtimeDuplexClient, event_type: str) -> list[dict[str, object]]:
|
||||
return [event for event in client.events.events if event.get("type") == event_type]
|
||||
|
||||
|
||||
async def _return_function_output_when_ready(
|
||||
client: RealtimeDuplexClient,
|
||||
*,
|
||||
output: str,
|
||||
timeout_s: float,
|
||||
) -> tuple[str, int]:
|
||||
await wait_for(
|
||||
lambda: bool(client.events.errors()) or client.events.count("response.function_call_arguments.done") > 0,
|
||||
timeout_s=timeout_s,
|
||||
label="function call to execute",
|
||||
)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"function call failed before tool execution: {client.events.errors()}")
|
||||
function_done = _events(client, "response.function_call_arguments.done")[-1]
|
||||
call_id = function_done.get("call_id")
|
||||
if not isinstance(call_id, str) or not call_id:
|
||||
raise AssertionError(f"completed function call has no call_id: {function_done}")
|
||||
event_count_before_output = len(client.events.events)
|
||||
await client.send(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "function_call_output",
|
||||
"call_id": call_id,
|
||||
"output": output,
|
||||
},
|
||||
}
|
||||
)
|
||||
return call_id, event_count_before_output
|
||||
|
||||
|
||||
def _write_events(path: Path, client: RealtimeDuplexClient) -> None:
|
||||
path.write_text(
|
||||
"".join(json.dumps(event, ensure_ascii=False) + "\n" for event in client.events.events),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _managed_client(client: RealtimeDuplexClient, *, timeout_s: float):
|
||||
async with client:
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if client.events.count("session.created") and not client.events.count("session.closed"):
|
||||
with suppress(Exception):
|
||||
await client.close_session(timeout_s=min(timeout_s, 30.0))
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> dict[str, object]:
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
if args.expect_function_call and args.instructions == DEFAULT_INSTRUCTIONS:
|
||||
instructions = DEFAULT_FUNCTION_INSTRUCTIONS
|
||||
else:
|
||||
instructions = args.instructions
|
||||
tools = DEFAULT_FUNCTION_TOOLS if args.expect_function_call else None
|
||||
session_id = f"nemotron-voicechat-{uuid.uuid4().hex}"
|
||||
client = RealtimeDuplexClient(_url(args.url, args.model, session_id))
|
||||
async with _managed_client(client, timeout_s=args.timeout_s):
|
||||
session_payload: dict[str, object] = {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"input_audio_format": "pcm_f32le",
|
||||
"output_audio_format": "pcm16",
|
||||
"instructions": instructions,
|
||||
"idle_timeout_s": args.timeout_s,
|
||||
"turn_detection": None,
|
||||
"extra_body": {"auto_response": True},
|
||||
}
|
||||
if tools is not None:
|
||||
session_payload["tools"] = tools
|
||||
await client.send({"type": "session.update", "session": session_payload})
|
||||
await wait_for(
|
||||
lambda: client.events.count("session.created") > 0 or bool(client.events.errors()),
|
||||
timeout_s=args.timeout_s,
|
||||
label="session.created",
|
||||
)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"session setup failed: {client.events.errors()}")
|
||||
created = _events(client, "session.created")[-1]
|
||||
session = created.get("session")
|
||||
capabilities = session.get("capabilities") if isinstance(session, dict) else None
|
||||
expected = dict(
|
||||
implementation_level="model_native_duplex",
|
||||
chunk_period_ms=80,
|
||||
supports_core_resumable_request=True,
|
||||
supports_core_kv_lease=False,
|
||||
supports_multi_session=False,
|
||||
)
|
||||
if not isinstance(capabilities, dict) or any(capabilities.get(key) != value for key, value in expected.items()):
|
||||
raise AssertionError(f"unexpected capabilities: {capabilities}")
|
||||
|
||||
function_output_task = (
|
||||
asyncio.create_task(
|
||||
_return_function_output_when_ready(
|
||||
client,
|
||||
output=args.function_output,
|
||||
timeout_s=args.timeout_s,
|
||||
)
|
||||
)
|
||||
if args.function_output is not None
|
||||
else None
|
||||
)
|
||||
pcm = _read_wav(Path(args.input_wav), input_channel=args.input_channel)
|
||||
frame_count = await _stream(client, pcm, max_frames=args.max_frames, realtime=not args.no_realtime)
|
||||
completed_responses_at_commit = client.events.count("response.done")
|
||||
await client.send({"type": "input_audio_buffer.commit", "final": True})
|
||||
await wait_for(
|
||||
lambda: (
|
||||
bool(client.events.errors())
|
||||
or (
|
||||
client.events.count("response.function_call_arguments.done") > 0
|
||||
if args.expect_function_call
|
||||
else (
|
||||
client.events.count("response.audio.delta") >= args.minimum_audio_chunks
|
||||
and (client.events.count("response.done") > completed_responses_at_commit)
|
||||
)
|
||||
)
|
||||
),
|
||||
timeout_s=args.timeout_s,
|
||||
label="model output",
|
||||
)
|
||||
await asyncio.sleep(args.drain_s)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"Realtime session emitted errors: {client.events.errors()}")
|
||||
done_events = _events(client, "response.done")
|
||||
if not args.expect_function_call:
|
||||
response = done_events[-1].get("response") if done_events else None
|
||||
status = response.get("status") if isinstance(response, dict) else None
|
||||
if status != "completed":
|
||||
raise AssertionError(f"response did not complete successfully: {done_events[-1:]}")
|
||||
|
||||
function_events = [
|
||||
event for event in client.events.events if str(event.get("type", "")).startswith("response.function_call")
|
||||
]
|
||||
function_items = [
|
||||
event
|
||||
for event in _events(client, "response.output_item.done")
|
||||
if isinstance(event.get("item"), dict) and event["item"].get("type") == "function_call"
|
||||
]
|
||||
if args.expect_function_call and not any(
|
||||
event.get("type") == "response.function_call_arguments.done" for event in function_events
|
||||
):
|
||||
raise AssertionError(f"no completed function call: {function_events}")
|
||||
if args.expect_function_call:
|
||||
matching_items = [
|
||||
event for event in function_items if event["item"].get("name") == args.expected_function_name
|
||||
]
|
||||
if not matching_items:
|
||||
raise AssertionError(f"expected {args.expected_function_name!r}, got {function_items}")
|
||||
function_item = matching_items[-1]["item"]
|
||||
try:
|
||||
function_arguments = json.loads(str(function_item.get("arguments", "")))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AssertionError(f"function arguments are not JSON: {function_item}") from exc
|
||||
if args.expected_function_arguments is not None:
|
||||
expected_arguments = json.loads(args.expected_function_arguments)
|
||||
if function_arguments != expected_arguments:
|
||||
raise AssertionError(
|
||||
f"function arguments differ: expected={expected_arguments!r}, actual={function_arguments!r}"
|
||||
)
|
||||
|
||||
if args.function_output is not None:
|
||||
call_id = function_item.get("call_id")
|
||||
if not isinstance(call_id, str) or not call_id:
|
||||
raise AssertionError(f"function item has no call_id: {function_item}")
|
||||
assert function_output_task is not None
|
||||
returned_call_id, event_count_before_output = await function_output_task
|
||||
assert returned_call_id == call_id
|
||||
|
||||
def tool_result_completed() -> bool:
|
||||
later = client.events.events[event_count_before_output:]
|
||||
transcript = "".join(
|
||||
str(event.get("delta", ""))
|
||||
for event in later
|
||||
if event.get("type") == "response.audio_transcript.delta"
|
||||
).lower()
|
||||
return (
|
||||
any(event.get("type") == "response.audio.delta" for event in later)
|
||||
and any(event.get("type") == "response.done" for event in later)
|
||||
and (args.expected_post_tool_text is None or args.expected_post_tool_text.lower() in transcript)
|
||||
)
|
||||
|
||||
await wait_for(
|
||||
lambda: bool(client.events.errors()) or tool_result_completed(),
|
||||
timeout_s=args.timeout_s,
|
||||
label="completed response after function output",
|
||||
)
|
||||
if client.events.errors():
|
||||
raise AssertionError(f"function output failed: {client.events.errors()}")
|
||||
await asyncio.sleep(args.drain_s)
|
||||
|
||||
audio = client.events.audio_bytes()
|
||||
audio_events = _events(client, "response.audio.delta")
|
||||
rates = {event.get("sample_rate_hz") for event in audio_events}
|
||||
if not args.expect_function_call and audio and rates != {OUTPUT_SAMPLE_RATE_HZ}:
|
||||
raise AssertionError(f"unexpected output sample rates: {rates}")
|
||||
if not args.expect_function_call and args.minimum_audio_chunks and not audio:
|
||||
raise AssertionError("model produced no audio")
|
||||
expected_bytes = 2 * OUTPUT_SAMPLE_RATE_HZ * expected["chunk_period_ms"] // 1000
|
||||
packet_sizes = [len(base64.b64decode(str(event.get("delta", "")), validate=True)) for event in audio_events]
|
||||
if not args.expect_function_call and any(size != expected_bytes for size in packet_sizes):
|
||||
raise AssertionError(f"audio deltas are not fixed 80 ms PCM16 packets: {packet_sizes}")
|
||||
audio_pcm = np.frombuffer(audio, dtype="<i2").astype(np.float32) / 32768.0
|
||||
audio_rms = float(np.sqrt(np.mean(np.square(audio_pcm)))) if audio_pcm.size else 0.0
|
||||
if not args.expect_function_call and audio and audio_rms < args.minimum_audio_rms:
|
||||
raise AssertionError(
|
||||
f"model output RMS {audio_rms:.6f} is below {args.minimum_audio_rms:.6f}; "
|
||||
"received packets contain only silence"
|
||||
)
|
||||
_write_events(output_dir / "events.jsonl", client)
|
||||
if audio:
|
||||
write_pcm16_wav(output_dir / "output.wav", audio, sample_rate_hz=OUTPUT_SAMPLE_RATE_HZ)
|
||||
result = {
|
||||
"ok": True,
|
||||
"session_id": session_id,
|
||||
"input_frames": frame_count,
|
||||
"capabilities": capabilities,
|
||||
"event_counts": {
|
||||
event_type: client.events.count(event_type)
|
||||
for event_type in sorted({str(event.get("type")) for event in client.events.events})
|
||||
},
|
||||
"audio_bytes": len(audio),
|
||||
"audio_rms": audio_rms,
|
||||
"function_events": function_events,
|
||||
"function_items": function_items,
|
||||
"output_dir": str(output_dir),
|
||||
}
|
||||
|
||||
(output_dir / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return result
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:8125/v1/realtime")
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--input-wav", required=True)
|
||||
parser.add_argument("--input-channel", type=int, default=0)
|
||||
parser.add_argument("--output-dir", default="/tmp/nemotron-voicechat-duplex")
|
||||
parser.add_argument("--instructions", default=DEFAULT_INSTRUCTIONS)
|
||||
parser.add_argument("--max-frames", type=int)
|
||||
parser.add_argument("--minimum-audio-chunks", type=int, default=1)
|
||||
parser.add_argument("--minimum-audio-rms", type=float, default=1e-4)
|
||||
parser.add_argument("--expect-function-call", action="store_true")
|
||||
parser.add_argument("--expected-function-name", default="generate_random_number")
|
||||
parser.add_argument("--expected-function-arguments")
|
||||
parser.add_argument("--function-output")
|
||||
parser.add_argument("--expected-post-tool-text")
|
||||
parser.add_argument("--no-realtime", action="store_true")
|
||||
parser.add_argument("--drain-s", type=float, default=2.0)
|
||||
parser.add_argument("--timeout-s", type=float, default=600.0)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print(json.dumps(asyncio.run(run(parse_args())), ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -52,9 +52,8 @@ def get_prompt(prompt_type="zh"):
|
||||
|
||||
def _set_rocm_request_timeout(request_config: dict) -> None:
|
||||
if current_omni_platform.is_rocm():
|
||||
# A cold AITER MHA kernel build took 301 seconds on MI300X.
|
||||
# Allow 600 seconds for the first request; later requests reuse the cache.
|
||||
request_config["timeout"] = 600.0
|
||||
# Cold AITER MHA builds on loaded MI300 agents can exceed 20 minutes.
|
||||
request_config["timeout"] = 1800.0
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
from tests.e2e.online_serving.nemotron_voicechat_realtime_duplex import parse_args, run
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams, get_model_prefix
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_MODEL", "nvidia/NVIDIA-NemotronLabs-VoiceChat-11B")
|
||||
DEPLOY_CONFIG = get_deploy_config_path("nemotron_labs_voicechat_duplex.yaml")
|
||||
TOKENIZER = os.environ.get("VLLM_TEST_NEMOTRON_VOICECHAT_LLM_PATH")
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
[
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=DEPLOY_CONFIG,
|
||||
env_dict={"NEMOTRON_VOICECHAT_LLM_PATH": TOKENIZER} if TOKENIZER else None,
|
||||
),
|
||||
id="native-duplex",
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_native_duplex_turn_taking_streams_model_audio(omni_server, tmp_path: Path) -> None:
|
||||
model_prefix = get_model_prefix()
|
||||
root = Path(model_prefix) / MODEL if model_prefix else Path(MODEL)
|
||||
if not root.is_dir():
|
||||
root = Path(snapshot_download(MODEL, local_files_only=True))
|
||||
args = parse_args(
|
||||
(
|
||||
f"--url ws://{omni_server.host}:{omni_server.port}/v1/realtime --model {omni_server.model} "
|
||||
f"--input-wav {root / 'turn_taking.wav'} --input-channel 0 --max-frames 190 "
|
||||
f"--minimum-audio-chunks 48 --minimum-audio-rms 0.001 --no-realtime --timeout-s 300 "
|
||||
f"--output-dir {tmp_path / 'native_duplex'}"
|
||||
).split()
|
||||
)
|
||||
result = asyncio.run(run(args))
|
||||
assert result["ok"] is True
|
||||
assert result["input_frames"] == 190
|
||||
assert result["event_counts"]["response.speak"] > 0
|
||||
assert result["audio_bytes"] >= result["input_frames"] * 1764 * 2 // 4
|
||||
@@ -551,6 +551,54 @@ async def test_native_realtime_protocol_conversation_item_create_commits_user_te
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_function_output_is_recorded_only_after_runtime_delivery():
|
||||
ws = TimedWebSocket()
|
||||
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
|
||||
protocol.bind_sender(ws.send_json)
|
||||
protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "conversation.item.created",
|
||||
"item": {
|
||||
"id": "item-call",
|
||||
"type": "function_call",
|
||||
"call_id": "call-1",
|
||||
"name": "weather",
|
||||
"arguments": "{}",
|
||||
"status": "completed",
|
||||
},
|
||||
}
|
||||
)
|
||||
event = {
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"id": "item-output",
|
||||
"type": "function_call_output",
|
||||
"call_id": "call-1",
|
||||
"output": '{"temperature":20}',
|
||||
},
|
||||
}
|
||||
|
||||
first = await protocol._to_duplex_event(event)
|
||||
retry = await protocol._to_duplex_event(event)
|
||||
|
||||
assert first is not None
|
||||
assert retry is not None
|
||||
assert "item-output" not in protocol._conversation_items
|
||||
|
||||
delivered_item = first["payload"]["item"]
|
||||
protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "conversation.item.created",
|
||||
"item": delivered_item,
|
||||
}
|
||||
)
|
||||
|
||||
assert protocol._conversation_items["item-output"] == delivered_item
|
||||
assert await protocol._to_duplex_event(event) is None
|
||||
assert ws.sent[-1]["error"]["code"] == "invalid_function_call_output"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_realtime_protocol_audio_commit_requires_non_empty_buffer():
|
||||
ws = TimedWebSocket()
|
||||
@@ -5481,7 +5529,7 @@ async def test_failed_native_append_prevents_queued_append_from_running():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_final_append_keeps_committed_audio_for_response_retry():
|
||||
async def test_failed_final_append_discards_retained_audio_without_retry():
|
||||
failed_result = {
|
||||
"operation": "append",
|
||||
"session_id": "sid-final-append-retry",
|
||||
@@ -5490,15 +5538,7 @@ async def test_failed_final_append_keeps_committed_audio_for_response_retry():
|
||||
"error_count": 1,
|
||||
"stage_results": [],
|
||||
}
|
||||
successful_result = {
|
||||
"operation": "append",
|
||||
"session_id": "sid-final-append-retry",
|
||||
"ok": True,
|
||||
"unsupported_count": 0,
|
||||
"error_count": 0,
|
||||
"stage_results": [],
|
||||
}
|
||||
engine = FakeEngineClient(append_results=[failed_result, successful_result])
|
||||
engine = FakeEngineClient(append_result=failed_result)
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
@@ -5523,15 +5563,18 @@ async def test_failed_final_append_keeps_committed_audio_for_response_retry():
|
||||
|
||||
handler_task = asyncio.create_task(handler.handle_session(ws))
|
||||
for _ in range(100):
|
||||
if len(engine.appended) >= 2:
|
||||
state = handler._serving_runtime_adapter.session_states.get("sid-final-append-retry")
|
||||
if len(engine.appended) >= 1 and state is not None and state.committed_audio_payload is None:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
state = handler._serving_runtime_adapter.session_states["sid-final-append-retry"]
|
||||
assert state.committed_audio_payload is None
|
||||
assert state.committed_audio_reserved_bytes == 0
|
||||
assert handler._registry.get("sid-final-append-retry").pending_input_bytes == 0
|
||||
ws.put({"type": "session.close"})
|
||||
await asyncio.wait_for(handler_task, timeout=2)
|
||||
|
||||
assert len(engine.appended) == 2
|
||||
assert engine.appended[0][2]["audio"] == engine.appended[1][2]["audio"]
|
||||
assert engine.append_operation_ids[0] == engine.append_operation_ids[1]
|
||||
assert len(engine.appended) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -225,3 +225,39 @@ def test_torchaudio_mel_filterbank_matches_librosa() -> None:
|
||||
reference = librosa.filters.mel(sr=16000, n_fft=512, n_mels=128, fmin=0.0, fmax=8000.0, norm="slaney")
|
||||
assert ta.shape == reference.shape
|
||||
assert abs(ta - reference).max() < 1e-6
|
||||
|
||||
|
||||
def test_code2wav_reuses_and_clears_per_request_streaming_cache() -> None:
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_code2wav import (
|
||||
NemotronVoiceChatCode2Wav,
|
||||
)
|
||||
|
||||
class Codec(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchor = nn.Parameter(torch.zeros(()))
|
||||
self.caches = []
|
||||
|
||||
def decode(self, codes, lengths, *, cache):
|
||||
self.caches.append(cache)
|
||||
wav = torch.zeros((1, int(codes.shape[1]) * 1764), device=codes.device)
|
||||
return wav, lengths * 1764
|
||||
|
||||
model = NemotronVoiceChatCode2Wav.__new__(NemotronVoiceChatCode2Wav)
|
||||
nn.Module.__init__(model)
|
||||
model._sample_rate, model._num_quantizers, model._codebook_size, model._wav_per_frame = 22050, 31, 1024, 1764
|
||||
model._duplex_codec_caches = {}
|
||||
model.audio_codec = Codec()
|
||||
|
||||
def info(codes):
|
||||
return [{"meta": {"codec_streaming": True, "request_id": "req"}, "codes": {"audio": codes}}]
|
||||
|
||||
model.forward(runtime_additional_information=info(torch.zeros((1, 31), dtype=torch.long)))
|
||||
model.forward(runtime_additional_information=info(torch.zeros((2, 31), dtype=torch.long)))
|
||||
assert model.audio_codec.caches[0] is model.audio_codec.caches[1]
|
||||
assert "req" in model._duplex_codec_caches
|
||||
model.on_requests_finished({"req"})
|
||||
assert "req" not in model._duplex_codec_caches
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -285,9 +285,46 @@ def test_talker2code2wav_async_chunk_requires_prompt_len() -> None:
|
||||
talker2code2wav_async_chunk(tm, {"codes": {"audio": torch.zeros(5, 31, dtype=torch.long)}}, req)
|
||||
|
||||
|
||||
# =========================
|
||||
# Talker drain-per-wake (async scheduling review P1/P2)
|
||||
# =========================
|
||||
def test_talker2code2wav_duplex_keeps_cumulative_history_across_wakes() -> None:
|
||||
from vllm_omni.model_executor.stage_input_processors.nemotron_voicechat import (
|
||||
talker2code2wav_async_chunk,
|
||||
)
|
||||
|
||||
tm = _fake_transfer_manager(codec_chunk_frames=2)
|
||||
prompt_len = 3
|
||||
req = _streaming_request(
|
||||
prompt_len=prompt_len,
|
||||
generated=[],
|
||||
info={"meta": {"codec_streaming": torch.tensor([True])}, "nvc_logical_prompt_len": prompt_len},
|
||||
)
|
||||
prompt_rows = torch.zeros((prompt_len - 1, 31), dtype=torch.long)
|
||||
first = torch.full((1, 31), 101, dtype=torch.long)
|
||||
second = torch.full((1, 31), 202, dtype=torch.long)
|
||||
|
||||
for rows, finished in ((prompt_rows, False), (torch.cat([prompt_rows, first]), False)):
|
||||
payload = talker2code2wav_async_chunk(
|
||||
tm,
|
||||
{"codes": {"audio": rows}, "meta": {"codec_streaming": torch.tensor([True])}},
|
||||
req,
|
||||
is_finished=finished,
|
||||
)
|
||||
assert payload is not None and not bool(payload.meta.finished)
|
||||
|
||||
payload = talker2code2wav_async_chunk(
|
||||
tm,
|
||||
{
|
||||
"codes": {"audio": torch.cat([prompt_rows, first, second])},
|
||||
"meta": {"codec_streaming": torch.tensor([True])},
|
||||
},
|
||||
req,
|
||||
is_finished=True,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert not bool(payload.meta.finished)
|
||||
assert not bool(payload.meta.is_segment_finished)
|
||||
assert torch.equal(payload.codes.audio, torch.cat([first, second]))
|
||||
assert tm.request_payload["req-0"]["nvc_frames_seen"] == 2
|
||||
|
||||
|
||||
class _FakeTTSModelOutput:
|
||||
@@ -342,7 +379,14 @@ def _bare_talker():
|
||||
return talker
|
||||
|
||||
|
||||
def _async_info(timeline: list[int], prompt_len: int, finished: bool, request_id: str = "req-0") -> dict:
|
||||
def _async_info(
|
||||
timeline: list[int],
|
||||
prompt_len: int,
|
||||
finished: bool,
|
||||
request_id: str = "req-0",
|
||||
*,
|
||||
codec_streaming: bool = False,
|
||||
) -> dict:
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"additional_information": {
|
||||
@@ -351,6 +395,7 @@ def _async_info(timeline: list[int], prompt_len: int, finished: bool, request_id
|
||||
"num_processed_tokens": prompt_len,
|
||||
"finished": torch.tensor(finished),
|
||||
"request_id": request_id,
|
||||
"codec_streaming": codec_streaming,
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -373,7 +418,7 @@ def test_talker_drains_all_received_positions_per_wake() -> None:
|
||||
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info)
|
||||
assert session["step"] == prompt_len # t advanced 1 -> P in one wake
|
||||
assert update["codes"]["audio"].shape == (prompt_len - 1, 31)
|
||||
assert float(embeds[0, 0]) == 0.0 # not finished
|
||||
assert float(embeds[0, 0]) == 0.0 # offline stream waits for upstream terminal
|
||||
|
||||
# Coalesced chunk: TWO new frame tokens arrive in ONE wake (delayed save
|
||||
# thread merged two thinker steps). Both must be consumed by this wake.
|
||||
@@ -383,14 +428,13 @@ def test_talker_drains_all_received_positions_per_wake() -> None:
|
||||
assert update["codes"]["audio"].shape == (prompt_len + 1, 31)
|
||||
assert float(embeds[0, 0]) == 0.0
|
||||
|
||||
# Zero-progress wake (no new positions, not finished): plain CONTINUE, no
|
||||
# fabricated frames, no codes update.
|
||||
# Zero-progress wake: never fabricate frames or end the offline stream
|
||||
# before the thinker sends its terminal marker.
|
||||
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info2)
|
||||
assert session["step"] == prompt_len + 2
|
||||
assert update == {}
|
||||
assert float(embeds[0, 0]) == 0.0
|
||||
|
||||
# Final chunk: one more token + finished marker -> drain + stop flag.
|
||||
info3 = _async_info([pad] * prompt_len + [7, 8, 9], prompt_len, finished=True)
|
||||
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info3)
|
||||
assert session["step"] == prompt_len + 3
|
||||
@@ -398,6 +442,18 @@ def test_talker_drains_all_received_positions_per_wake() -> None:
|
||||
assert float(embeds[0, 0]) == 1.0 # stop
|
||||
|
||||
|
||||
def test_talker_native_codec_stream_ends_each_received_segment() -> None:
|
||||
talker = _bare_talker()
|
||||
ids = torch.zeros(1, dtype=torch.long)
|
||||
info = _async_info([12, 12, 7], 2, finished=False, codec_streaming=True)
|
||||
talker.preprocess(ids, None, _omni_is_prefill=True, **info)
|
||||
|
||||
_, embeds, update = talker.preprocess(ids, None, _omni_is_prefill=False, **info)
|
||||
|
||||
assert update["codes"]["audio"].shape == (2, 31)
|
||||
assert float(embeds[0, 0]) == 1.0
|
||||
|
||||
|
||||
def test_talker_sync_mode_steps_once_per_wake() -> None:
|
||||
"""The verified sync parity path keeps its one-TTS-step-per-engine-step cadence."""
|
||||
talker = _bare_talker()
|
||||
|
||||
@@ -1832,6 +1832,7 @@ class TestPreferModelSamplerNoneFallback:
|
||||
"hunyuan_image3",
|
||||
"minicpmo_4_5",
|
||||
"minimax_music3",
|
||||
"nemotron_voicechat",
|
||||
}
|
||||
assert declarers == expected, (
|
||||
"The set of models declaring `prefer_model_sampler` changed:\n"
|
||||
|
||||
Reference in new issue
Block a user