[Perf][NemotronVoiceChat] Fast defaults for all three deploy profiles: 2.8x offline, realtime full-duplex audio (#6354)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yuekai ZhangandClaude Fable 5 authored and GitHub committed 2026-09-03 18:49:19 -07:00
1 parent e43c787544
commit 4179b0cae1
22 files changed
+2606 -195

No files matched your search

@@ -705,6 +705,10 @@ class DuplexEARTTS(nn.Module):
generation_config=None,
ignore_eos_flag_stop=True,
request_id=None,
current_subword_embed=None,
cache_position=None,
attention_mask=None,
position_ids=None,
):
"""
Runs a single autoregressive prediction step to infer audio codec codes.
@@ -756,6 +760,20 @@ class DuplexEARTTS(nn.Module):
# request_id is only used by vLLM backend — skip for native PyTorch inference
if request_id is not None:
inputs["request_id"] = request_id
# Fast-path extras (see RVQEARTTSModel.forward): a precomputed subword
# embedding bypasses the CAS encoder (host-sync free); cache_position
# enables StaticCache decoding; an explicit (4D) attention_mask and
# position_ids keep the mask and RoPE fully tensor-driven, which CUDA
# graph capture requires (the HF-derived paths read python-int cache
# lengths that would be baked into the captured graph).
if current_subword_embed is not None:
inputs["subword_embeds"] = current_subword_embed
if cache_position is not None:
inputs["cache_position"] = cache_position
if attention_mask is not None:
inputs["attention_mask"] = attention_mask
if position_ids is not None:
inputs["position_ids"] = position_ids
outputs = self.tts_model(**inputs)
@@ -1141,14 +1141,24 @@ class RVQEARTTSModel(nn.Module):
subword_ids: Tensor | None,
subword_mask: Tensor | None,
uncond_dec_flag: Tensor,
subword_embeds: Tensor | None = None,
) -> Tensor:
"""Computes the final conditioning tensor by combining all sources."""
"""Computes the final conditioning tensor by combining all sources.
``subword_embeds`` short-circuits the CharAwareSubwordEncoder with a
precomputed embedding (same shape as its output). Besides skipping the
char-level backbone, this path is free of host syncs
(``subword_mask.any()`` / the encoder's ``.cpu().tolist()``), which
makes it CUDA-graph capturable.
"""
cond = torch.zeros((1, 1, self.hidden_size), device=uncond_dec_flag.device)
if self.embed_context is not None and context_hidden_state is not None:
cond = cond + self.embed_context(context_hidden_state)
if self.embed_subword is not None and subword_ids is not None:
if subword_embeds is not None:
cond = cond + subword_embeds
elif self.embed_subword is not None and subword_ids is not None:
# Infer subword mask from context if not provided
if subword_mask is None and context_hidden_state is not None:
subword_mask = torch.any(context_hidden_state != 0, dim=-1)
@@ -1244,6 +1254,8 @@ class RVQEARTTSModel(nn.Module):
ignore_eos_flag_stop: bool = False,
asr_speech_tokens_emb: Tensor | None = None,
audio_prompt_latent: Tensor | None = None,
subword_embeds: Tensor | None = None,
cache_position: Tensor | None = None,
) -> RVQEARTTSOutput:
"""
Performs a forward pass handling training, generation, or single-step inference.
@@ -1352,13 +1364,17 @@ class RVQEARTTSModel(nn.Module):
subword_ids = torch.cat([subword_ids] * 2, 0)
if subword_mask is not None:
subword_mask = torch.cat([subword_mask] * 2, 0)
if subword_embeds is not None:
subword_embeds = torch.cat([subword_embeds] * 2, 0)
if asr_speech_tokens_emb is not None:
asr_speech_tokens_emb = torch.cat([asr_speech_tokens_emb] * 2, 0)
uncond_dec_flag = torch.cat([uncond_dec_flag, torch.ones_like(uncond_dec_flag)], 0)
# Prepare conditioning
cond = self._prepare_conditioning(context_hidden_state, subword_ids, subword_mask, uncond_dec_flag)
cond = self._prepare_conditioning(
context_hidden_state, subword_ids, subword_mask, uncond_dec_flag, subword_embeds=subword_embeds
)
if asr_speech_tokens_emb is not None:
cond = cond + asr_speech_tokens_emb
@@ -1368,13 +1384,20 @@ class RVQEARTTSModel(nn.Module):
else:
inputs_embeds = code_embeds + cond
# Main backbone pass
# Main backbone pass. cache_position is threaded only when provided so
# backbones without the parameter keep working (it is required for
# StaticCache-based decoding, where the write index cannot be derived
# from the cache contents).
backbone_kwargs: dict[str, Any] = {}
if cache_position is not None:
backbone_kwargs["cache_position"] = cache_position
backbone_outputs = self.backbone(
inputs_embeds=inputs_embeds,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=past_key_values,
use_cache=use_cache,
**backbone_kwargs,
)
hidden_states = backbone_outputs.last_hidden_state
@@ -1564,7 +1587,9 @@ class RVQEARTTSModel(nn.Module):
else:
eos_flag = lm_logits.argmax(-1) == 1
if torch.all(eos_flag) and not ignore_eos_flag_stop:
# Check the cheap Python flag first: evaluating torch.all() forces a
# host sync per frame even when the result is ignored.
if not ignore_eos_flag_stop and torch.all(eos_flag):
return None, lm_logits, eos_flag
else:
lm_logits = None
@@ -1573,17 +1598,18 @@ class RVQEARTTSModel(nn.Module):
# Initialize the full code tensor
code = torch.zeros((b, t, d), dtype=torch.long, device=device) + self.config.codebook_size
# 3. Set up the iterative denoising schedule for the continuous part
rates = torch.linspace(0.0, 1.0, num_iter + 1, device=device)[:-1].unsqueeze(-1)
masking_rates = get_masking_rate(rates, exponent=exponent)
num_maskings = torch.ceil(masking_rates * self.config.num_quantizers).long()
ks = num_maskings - F.pad(num_maskings[1:], [0, 0, 0, 1])
# 3. Set up the iterative denoising schedule for the continuous part.
# The schedule is a pure function of (num_iter, exponent,
# num_quantizers) — compute it once as Python ints and cache it, so the
# per-frame loop below runs without any GPU->CPU sync (`.item()` /
# `torch.all(k == 0)` on a device tensor would stall the dispatch
# pipeline several times per 80 ms frame).
ks = self._unmask_schedule(num_iter, float(exponent))
# 4. Iteratively unmask the continuous part of the code
cnt = 0
for i, k in enumerate(ks):
if torch.all(k == 0):
if k == 0:
continue
# Prepare input for the MoG head
@@ -1593,7 +1619,7 @@ class RVQEARTTSModel(nn.Module):
mog_input_embeds = self.embed_code(self.depthsum_embedding(code).to(self.embed_code.weight.dtype))
if self.config.random_target_masking:
mog_input_embeds += self.embed_target_mask(cnt + k - 1)
mog_input_embeds += self.embed_target_mask(torch.tensor(cnt + k - 1, device=device))
if guidance_scale_i > 0.0:
mog_input_embeds = torch.cat(
[mog_input_embeds + hidden_states, mog_input_embeds + uncond_hidden_states], 0
@@ -1607,10 +1633,26 @@ class RVQEARTTSModel(nn.Module):
top_p_or_k=top_p_or_k_i,
)
z = mog_mu + torch.exp(mog_logs) * torch.randn_like(mog_mu) * noise_scale_i
code = depthsum_encoding_step(self.rvq_embs, z, code, cnt, k[0].item())
cnt += k[0].item()
code = depthsum_encoding_step(self.rvq_embs, z, code, cnt, k)
cnt += k
return code, lm_logits, eos_flag
def _unmask_schedule(self, num_iter: int, exponent: float) -> list[int]:
"""Per-iteration unmask counts as Python ints (cached; sync-free loop)."""
cache = getattr(self, "_unmask_schedule_cache", None)
if cache is None:
cache = {}
self._unmask_schedule_cache = cache
key = (int(num_iter), float(exponent), int(self.config.num_quantizers))
ks = cache.get(key)
if ks is None:
rates = torch.linspace(0.0, 1.0, num_iter + 1)[:-1].unsqueeze(-1)
masking_rates = get_masking_rate(rates, exponent=exponent)
num_maskings = torch.ceil(masking_rates * self.config.num_quantizers).long()
ks = (num_maskings - F.pad(num_maskings[1:], [0, 0, 0, 1])).reshape(-1).tolist()
cache[key] = ks
return ks
def load_state_dict(self, state_dict, strict: bool = True):
try:
super().load_state_dict(state_dict, strict=strict)
@@ -82,9 +82,22 @@ class NemotronVoiceChatCode2Wav(nn.Module):
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
# Incremental streaming decode (hf_overrides.use_incremental_codec_cache):
# keep a CausalConv1dCache per request so each async chunk decodes ONLY
# its new frames — O(T) codec work — instead of re-decoding the whole
# cumulative prefix per chunk (O(T^2)). This is the codec-cache path
# NVIDIA's own serving stack runs (S2S_USE_CODEC_CACHE); the default
# stays the prefix re-decode, whose concatenation is bit-equal to the
# one-shot decode of the final stack.
self._use_codec_cache = bool(getattr(self.config, "use_incremental_codec_cache", False))
self.requires_request_ids = self._use_codec_cache
# Shared per-request causal-conv caches; ALSO used by duplex serving,
# whose per-segment payloads mark themselves with meta.codec_streaming
# + meta.request_id (independent of the hf_override).
self._stream_caches: dict[str, Any] = {}
# Constructed in load_weights (owns the tts_model.audio_codec.* subtree).
self.audio_codec: nn.Module | None = None
self._duplex_codec_caches: dict[str, object] = {}
# ------------------------------------------------------------------
# Runner-facing placeholder hooks.
@@ -130,6 +143,7 @@ class NemotronVoiceChatCode2Wav(nn.Module):
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
request_ids: list[str] | None = None,
**kwargs: Any,
) -> OmniOutput:
sr_tensor = torch.tensor(self._sample_rate, dtype=torch.int32)
@@ -189,25 +203,40 @@ class NemotronVoiceChatCode2Wav(nn.Module):
left_context_frames = max(int(runtime_info["meta.left_context_size"]), 0)
if left_context_frames >= frames:
continue # terminal empty chunk: all frames already emitted
lens = torch.tensor([frames], device=device, dtype=torch.long)
is_stream_chunk = (isinstance(meta, Mapping) and "left_context_size" in meta) or (
isinstance(runtime_info, Mapping) and "meta.left_context_size" in runtime_info
)
request_id = str(request_ids[i]) if request_ids is not None and i < len(request_ids) else None
codec_cache = None
decode_codes = codes
if codec_streaming:
# Duplex serving: the per-segment payload carries ONLY the new
# frames; continuity lives in the per-request causal-conv cache
# keyed by the talker-forwarded meta.request_id.
if not isinstance(cache_request_id, str) or not cache_request_id:
raise ValueError("Nemotron VoiceChat incremental codec input lacks meta.request_id")
from vllm_omni.model_executor.models.nemotron_voicechat.nemo_vendored.ear_tts_vae_codec import (
CausalConv1dCache,
)
codec_cache = self._duplex_codec_caches.setdefault(cache_request_id, CausalConv1dCache())
codec_cache = self._stream_caches.setdefault(cache_request_id, CausalConv1dCache())
elif self._use_codec_cache and is_stream_chunk and request_id is not None:
# Async-chunk streaming: cumulative payload; decode ONLY the
# frames past the already-emitted prefix, with the causal-conv
# left context carried in the per-request cache.
from vllm_omni.model_executor.models.nemotron_voicechat.nemo_vendored.ear_tts_vae_codec import (
CausalConv1dCache,
)
codec_cache = self._stream_caches.setdefault(request_id, CausalConv1dCache())
decode_codes = codes[left_context_frames:]
lens = torch.tensor([decode_codes.shape[0]], device=device, dtype=torch.long)
# NeMo decodes the codec strictly in fp32.
with torch.autocast(device_type=device.type, enabled=False):
wav, wav_len = self.audio_codec.decode(
codes.unsqueeze(0),
lens,
cache=codec_cache,
)
wav, wav_len = self.audio_codec.decode(decode_codes.unsqueeze(0), lens, cache=codec_cache)
wav = wav.reshape(-1)[: int(wav_len.reshape(-1)[0])]
if left_context_frames > 0 and not codec_streaming:
if left_context_frames > 0 and codec_cache is None:
# Prefix re-decode path: slice off the samples already emitted.
wav = wav[left_context_frames * self._wav_per_frame :]
audios[i] = wav.detach().to(dtype=torch.float32).cpu()
logger.debug(
@@ -222,15 +251,15 @@ class NemotronVoiceChatCode2Wav(nn.Module):
multimodal_outputs={"model_outputs": audios, "sr": srs},
)
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
for request_id in finished_req_ids:
self._duplex_codec_caches.pop(str(request_id), None)
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
raise TypeError(f"NemotronVoiceChatCode2Wav expected OmniOutput, got {type(model_outputs)}")
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
for request_id in finished_req_ids:
self._stream_caches.pop(str(request_id), None)
# ------------------------------------------------------------------
# Weights.
# ------------------------------------------------------------------
@@ -113,13 +113,71 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
self.has_postprocess = False
self.requires_full_prefix_cached_hidden_states = False
# Stateful per-request execution is only validated at batch size 1
# (the shipped offline scope); fail fast on silent multi-request use.
# The vendored (eager / captured-step) paths share single-session
# buffers (StaticCache, capture-time addresses) and stay batch=1; the
# NATIVE path keeps all per-request state in _sessions (paged KV,
# per-request MoG on each request's own hidden slice) and supports
# multi-session batching (validated at max_num_seqs<=4). The native
# flag is read below, so peek at it here for the guard.
max_num_seqs = int(getattr(vllm_config.scheduler_config, "max_num_seqs", 1))
if max_num_seqs != 1:
if max_num_seqs != 1 and not bool(getattr(self.config, "use_native_talker", False)):
raise NotImplementedError(
f"NemotronVoiceChat talker supports max_num_seqs=1 only (got {max_num_seqs}); "
"the vendored EAR-TTS session state is not batch-safe."
f"NemotronVoiceChat talker supports max_num_seqs=1 only (got {max_num_seqs}) on the "
"vendored paths; the EAR-TTS session buffers are not batch-safe. Use "
"hf_overrides.use_native_talker for multi-session batching."
)
# CUDA-graph fast path (hf_overrides.use_talker_cuda_graphs). Off by
# default: the eager path is the NeMo bit-parity acceptance path, while
# the captured step replays the same math ~3x faster with a
# graph-managed RNG (audio equivalent but not bit-identical).
self._use_cuda_graphs = bool(getattr(self.config, "use_talker_cuda_graphs", False))
self._graph_max_cache_len = int(getattr(self.config, "talker_graph_max_cache_len", 4096))
self._step_graph: Any | None = None
# Native-vLLM path (hf_overrides.use_native_talker, experimental): the
# Gemma3 backbone runs as a real vLLM model on PagedAttention (KV and
# per-step attention cost scale with the ACTUAL session length; vLLM's
# own decode CUDA graphs apply), mirroring NVIDIA's deployment. The
# per-frame MoG sampling runs on the backbone's hidden state in
# make_omni_output. Classifier-free guidance is NOT applied yet on
# this path (single conditional stream; the paired-request CFG design
# is a documented follow-up), so audio quality trails the vendored
# CFG paths — use for development only.
self._use_native_backbone = bool(getattr(self.config, "use_native_talker", False))
# Classifier-free guidance on the native path
# (hf_overrides.native_talker_guidance): the conditional stream stays
# on vLLM paged KV; the unconditional stream mirrors it on the vendored
# HF backbone (see talker_native.UncondStream) and the pair is blended
# per NeMo's generate_step math. Off by default (measured ASR-equivalent
# at this checkpoint's guidance_scale=0.2, but kept available for
# listening-test parity with the vendored paths).
self._native_guidance = self._use_native_backbone and bool(
getattr(self.config, "native_talker_guidance", False)
)
# Captured MoG sampling step (built lazily on the first decode frame;
# falls back to eager sampling if capture fails).
self._mog_graph: Any | None = None
self._uncond_graph: Any | None = None
self.backbone: nn.Module | None = None
if self._use_native_backbone:
if self._use_cuda_graphs:
raise ValueError(
"NemotronVoiceChat talker: use_native_talker and use_talker_cuda_graphs are mutually exclusive."
)
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
synthesize_backbone_config,
)
tts_config = dict(getattr(self.config, "tts_cfg", {}) or {}).get("tts_config") or {}
backbone_cfg = synthesize_backbone_config(dict(tts_config))
self.backbone = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=backbone_cfg,
prefix=maybe_prefix(prefix, "backbone"),
architectures=["Gemma3ForCausalLM"],
)
# Vendored DuplexEARTTS; constructed in load_weights.
@@ -141,6 +199,16 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
inputs_embeds: torch.Tensor | None = None,
**_: Any,
) -> torch.Tensor:
if self._use_native_backbone:
# Real Gemma3 forward on PagedAttention; preprocess injected the
# fused embeddings, make_omni_output turns the hidden state into
# codes (MoG) and the CONTINUE/STOP flag.
return self.backbone(
input_ids=None,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
# The per-frame compute happens in preprocess; the "hidden state" only
# transports the stop flag (column 0) to compute_logits.
if inputs_embeds is not None:
@@ -161,6 +229,8 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
if self._use_native_backbone:
return self._native_make_omni_output(model_outputs, **kwargs)
hidden = model_outputs
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information") or []
codes_list: list[torch.Tensor] = []
@@ -275,6 +345,36 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
)
return timeline
@staticmethod
def _expected_total_tokens(info: dict[str, Any]) -> int | None:
"""Producer-announced FINAL timeline length (meta.expected_total_tokens)."""
meta = info.get("meta")
value = meta.get("expected_total_tokens") if isinstance(meta, dict) else None
if value is None:
value = info.get("meta.expected_total_tokens")
return int(value) if value is not None else None
def _graph_session_usable(self, timeline: torch.Tensor, expected_len: int | None) -> bool:
"""Whether the CUDA-graph fast path can serve this session.
``expected_len`` is the known FINAL timeline length (sync mode, or a
frame-locked async producer's meta.expected_total_tokens); the graph
picks the smallest fitting cache bucket from it, since per-step
attention cost scales with the bucket size. ``None`` = unbounded
(duplex) -> largest bucket.
"""
if self._step_graph is None:
return False
needed = max(int(timeline.numel()), expected_len or 0)
if not self._step_graph.fits(needed):
logger.warning(
"NemotronVoiceChat talker: timeline of %d positions exceeds the CUDA-graph "
"capacity; using the eager per-frame step for this request.",
needed,
)
return False
return self._step_graph.ensure_captured(expected_len)
def _init_session(self, request_id: str, info: dict[str, Any], device: torch.device) -> dict[str, Any]:
assert self.tts is not None
timeline = self._timeline(info).to(device)
@@ -283,15 +383,6 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
f"NemotronVoiceChat talker timeline for request {request_id} has "
f"{timeline.numel()} positions; need at least 2 (prompt + one frame)."
)
# Warmup: pre-baked speaker latent + init prompt forward (NeMo
# offline_inference lines around get_init_inputs/first forward).
self.tts.set_init_inputs(speaker_name=self._speaker_name)
init_inputs = self.tts.get_init_inputs(B=1)
guidance_enabled = self._guidance_enabled()
generation_config = self.tts._get_generation_config(guidance_enabled)
init_inputs.update({"use_cache": True, "past_key_values": None, "guidance_enabled": guidance_enabled})
with torch.inference_mode():
outputs = self.tts.tts_model(**init_inputs)
prompt_len = self._logical_prompt_len(info)
if prompt_len is None:
raise ValueError(
@@ -304,17 +395,17 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
codec_streaming = scalar_bool(meta.get("codec_streaming")) if isinstance(meta, dict) else False
codec_streaming = codec_streaming or scalar_bool(info.get("meta.codec_streaming"))
codec_request_id = meta.get("request_id") if isinstance(meta, dict) else None
# Warmup: pre-baked speaker latent (NeMo offline_inference lines around
# get_init_inputs); the init prompt forward runs in the eager branch
# below, or inside the CUDA-graph session's own StaticCache prefill.
self.tts.set_init_inputs(speaker_name=self._speaker_name)
guidance_enabled = self._guidance_enabled()
session = {
"timeline": timeline,
"prompt_len": prompt_len,
"step": 1, # NeMo's loop starts at t=1
"past_key_values": outputs.past_key_values,
"code": init_inputs["code"][:, -1:],
"first_context_subword_id": init_inputs["subword_ids"][:, -1].unsqueeze(-1),
"current_subword_mask": torch.ones(1, 1, device=device, dtype=torch.bool),
"tts_initialized": False,
"guidance_enabled": guidance_enabled,
"generation_config": generation_config,
"codes_rows": [],
"codec_streaming": codec_streaming,
"codec_request_id": codec_request_id,
@@ -324,14 +415,434 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
"sync_mode": info.get("nvc_text_timeline") is not None,
"upstream_finished": self._upstream_finished(info) or info.get("nvc_text_timeline") is not None,
}
if session["sync_mode"]:
expected_len: int | None = int(timeline.numel())
else:
expected_len = self._expected_total_tokens(info)
if self._graph_session_usable(timeline, expected_len):
self._step_graph.start_session(timeline, expected_len=expected_len)
session["use_graph"] = True
else:
init_inputs = self.tts.get_init_inputs(B=1)
generation_config = self.tts._get_generation_config(guidance_enabled)
init_inputs.update({"use_cache": True, "past_key_values": None, "guidance_enabled": guidance_enabled})
with torch.inference_mode():
outputs = self.tts.tts_model(**init_inputs)
session.update(
{
"use_graph": False,
"past_key_values": outputs.past_key_values,
"code": init_inputs["code"][:, -1:],
"first_context_subword_id": init_inputs["subword_ids"][:, -1].unsqueeze(-1),
"tts_initialized": False,
"generation_config": generation_config,
}
)
self._sessions[request_id] = session
return session
# ------------------------------------------------------------------
# Native-vLLM path (experimental; see __init__).
# ------------------------------------------------------------------
def _encode_cas_rows(self, ids: torch.Tensor) -> torch.Tensor:
"""Batched CharAwareSubwordEncoder rows for timeline tokens ([n, H])."""
model = self.tts.tts_model
mask = torch.ones(1, ids.numel(), dtype=torch.bool, device=ids.device)
with torch.inference_mode():
return model.embed_subword(ids.unsqueeze(0), mask)[0]
def _init_native_session(self, request_id: str, info: dict[str, Any], device: torch.device) -> dict[str, Any]:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
build_prefill_embeds,
)
timeline = self._timeline(info).to(device)
prompt_len = self._logical_prompt_len(info)
if prompt_len is None:
raise ValueError(
"NemotronVoiceChat talker request is missing its logical prompt length "
"('nvc_logical_prompt_len' or async meta.num_processed_tokens); the code2wav "
"producer cannot trim the prompt-region codes without it."
)
meta = info.get("meta")
codec_streaming = scalar_bool(meta.get("codec_streaming")) if isinstance(meta, dict) else False
codec_streaming = codec_streaming or scalar_bool(info.get("meta.codec_streaming"))
codec_request_id = meta.get("request_id") if isinstance(meta, dict) else None
sync_mode = info.get("nvc_text_timeline") is not None
if not sync_mode and not codec_streaming:
raise NotImplementedError(
"NemotronVoiceChat native talker supports the sync pipeline and duplex "
"(resumable) serving; the offline async-chunk pipeline would free-run past "
"the received timeline (non-resumable requests cannot park per segment) — "
"use the captured-step fast_streaming yaml for offline streaming."
)
self.tts.set_init_inputs(speaker_name=self._speaker_name)
init_inputs = self.tts.get_init_inputs(B=1)
init_len = int(init_inputs["code"].shape[1])
model = self.tts.tts_model
with torch.inference_mode():
prefill = build_prefill_embeds(
model,
code=init_inputs["code"],
subword_ids=init_inputs["subword_ids"],
subword_mask=init_inputs["subword_mask"],
audio_mask=init_inputs["audio_mask"],
audio_prompt_latent=init_inputs.get("audio_prompt_latent"),
uncond=False,
)[0] # [init_len, H]
uncond_stream = None
if self._native_guidance:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
UncondStepGraph,
UncondStream,
)
with torch.inference_mode():
uncond_prefill = build_prefill_embeds(
model,
code=init_inputs["code"],
subword_ids=init_inputs["subword_ids"],
subword_mask=init_inputs["subword_mask"],
audio_mask=init_inputs["audio_mask"],
audio_prompt_latent=init_inputs.get("audio_prompt_latent"),
uncond=True,
).to(self._dtype)
# Captured StaticCache step when the graph is free (one session at
# a time; sync sessions pick the smallest fitting bucket, duplex
# takes the largest). Concurrent sessions step eagerly.
if self._uncond_graph is None:
self._uncond_graph = UncondStepGraph(
model,
hidden_size=int(uncond_prefill.shape[-1]),
dtype=self._dtype,
device=timeline.device,
max_cache_len=self._graph_max_cache_len,
)
total_positions = (init_len + int(timeline.numel()) + 8) if sync_mode else None
if self._uncond_graph.start_session(request_id, uncond_prefill, total_positions):
uncond_stream = self._uncond_graph
else:
uncond_stream = UncondStream(model)
uncond_stream.prefill(uncond_prefill)
session = {
"use_native": True,
"timeline": timeline,
"cas_table": self._encode_cas_rows(timeline),
"prompt_len": int(prompt_len),
"init_len": init_len,
"prefill_embeds": prefill,
"step": 1, # NeMo's loop starts at t=1
"code": init_inputs["code"][:, -1:],
# Kept for KV-recompute replay: step t=1's feedback code, before
# session["code"] starts tracking the latest sampled frame.
"initial_code": init_inputs["code"][:, -1:],
# CFG: native_talker_guidance mirrors the stream on the vendored
# HF backbone (UncondStream); otherwise a single conditional
# stream (measured ASR-equivalent at guidance_scale=0.2; see yaml).
"generation_config": self.tts._get_generation_config(self._native_guidance),
"uncond_stream": uncond_stream,
"uncond_hidden": None,
# NeMo's inference_force_speech_silence_on_eos (default on): EOS
# text frames feed the silence codec frame back instead of the
# sampled codes.
"silence_codes": (
self.tts.codec_silence_tokens.reshape(1, 1, -1).to(device=timeline.device, dtype=torch.long)
if bool(self.tts.cfg.get("inference_force_speech_silence_on_eos", True))
else None
),
"text_eos_id": int(self.tts.text_eos_id),
"codes_rows": [],
"pending_step": None,
"codec_streaming": codec_streaming,
"codec_request_id": codec_request_id,
"sync_mode": info.get("nvc_text_timeline") is not None,
"upstream_finished": self._upstream_finished(info) or info.get("nvc_text_timeline") is not None,
}
self._sessions[request_id] = session
return session
def _refresh_native_timeline(self, session: dict[str, Any], info: dict[str, Any], device: torch.device) -> None:
"""Adopt a longer timeline from the latest chunk; CAS-encode new rows."""
latest = self._try_timeline(info)
old_len = int(session["timeline"].numel())
if latest is not None and latest.numel() > old_len:
latest = latest.to(device)
new_rows = self._encode_cas_rows(latest[old_len:].to(torch.long))
session["timeline"] = latest
session["cas_table"] = torch.cat([session["cas_table"], new_rows.to(session["cas_table"].dtype)], dim=0)
if self._upstream_finished(info):
session["upstream_finished"] = True
def _native_preprocess(
self,
input_ids: torch.Tensor,
info: dict[str, Any],
device: torch.device,
span: int,
request_id: str,
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
build_decode_embeds,
)
# NOTE: resumable (duplex) segments report _omni_is_prefill=True on
# EVERY wake (each appended placeholder extends the prompt), so the
# branch is decided by the engine POSITION, never by the prefill flag:
# positions [0, init_len) carry the speaker-prompt prefill embeddings,
# every later position consumes exactly one timeline step.
session = self._sessions.get(request_id)
if session is None:
session = self._init_native_session(request_id, info, device)
else:
self._refresh_native_timeline(session, info, device)
offset = max(0, int(info.get("_omni_num_computed_tokens", 0) or 0))
init_len = int(session["init_len"])
step = int(session["step"])
total = int(session["timeline"].numel())
prefill_head: torch.Tensor | None = None
if offset < init_len:
# Speaker-prompt prefill region (possibly chunked across steps).
if offset + span <= init_len:
prefill: torch.Tensor = session["prefill_embeds"]
return input_ids, prefill[offset : offset + span].to(self._dtype), {}
# A span crossing the boundary is only legal while a KV-cache
# recomputation replays already-generated positions in bulk; a
# LIVE first timeline step sharing a chunk with the prefill means
# a timeline chunk outpaced the speaker prefill.
if step <= 1:
raise ValueError(
f"NemotronVoiceChat native talker got a span [{offset}, {offset + span}) crossing "
f"the speaker-prompt boundary ({init_len}); the stage prompt must be exactly "
f"{init_len} placeholder tokens (set hf_overrides.talker_init_len / "
f"async_chunk_prewarm_prompt_len to {init_len} in the deploy yaml). A timeline "
"chunk extended this request before its speaker prefill finished — timeline "
"positions are strictly sequential and cannot share a step with the prefill."
)
prefill_head = session["prefill_embeds"][offset:init_len]
span -= init_len - offset
offset = init_len
t_first = offset - init_len + 1
t_last = t_first + span - 1
if t_last > step:
raise RuntimeError(
f"NemotronVoiceChat native talker was scheduled timeline steps {t_first}..{t_last} but "
f"the session is at step {step}; the per-frame code feedback is strictly sequential. "
"The duplex producer drips one new timeline token per chunk — a position past the "
"session step means chunks outpaced engine steps (raise the frame budget or "
"investigate the engine stall)."
)
if t_last >= total:
raise RuntimeError(f"NemotronVoiceChat native talker stepped past the timeline: step {t_last} of {total}.")
replay_last = min(t_last, step - 1)
if replay_last >= t_first and len(session["codes_rows"]) < replay_last - 1:
raise RuntimeError(
f"NemotronVoiceChat native talker cannot replay steps {t_first}..{replay_last}: only "
f"{len(session['codes_rows'])} stored code rows."
)
replay_rows: list[torch.Tensor] = []
if replay_last >= t_first:
# KV-cache recomputation replays already-generated engine
# positions. Rebuild their decode embeds deterministically from
# the stored code history: no resampling (pending_step is only set
# for a live step) and the manual unconditional stream is not
# advanced (it was never rewound).
if not session.get("replaying"):
session["replaying"] = True
logger.warning(
"NemotronVoiceChat native talker replaying timeline steps %d..%d after KV recompute.",
t_first,
replay_last,
)
replay_rows = [self._native_replay_embeds(session, rt) for rt in range(t_first, replay_last + 1)]
if t_last < step:
# Pure replay chunk: no live position this step.
embeds = torch.cat(replay_rows, dim=0)
if prefill_head is not None:
embeds = torch.cat([prefill_head.to(embeds.dtype), embeds], dim=0)
return input_ids, embeds.to(self._dtype), {}
session.pop("replaying", None)
t = step
model = self.tts.tts_model
with torch.inference_mode():
prev_codes = session["code"]
if session["silence_codes"] is not None:
# NeMo's inference_force_speech_silence_on_eos: on an EOS text
# frame the code feedback becomes the silence frame, so the
# turn ends in clean silence instead of a trailing artifact.
# Tensor-level select — no host sync on the timeline token.
is_eos = session["timeline"][t : t + 1].reshape(1, 1, 1) == session["text_eos_id"]
prev_codes = torch.where(is_eos, session["silence_codes"].expand_as(prev_codes), prev_codes)
embeds = build_decode_embeds(
model,
prev_codes=prev_codes,
subword_embed=session["cas_table"][t : t + 1].unsqueeze(0),
uncond=False,
)[0] # [1, H]
if session["uncond_stream"] is not None:
# Advance the unconditional stream in lockstep: same prev codes,
# null conditioning. Its hidden row pairs with this step's
# conditional row in make_omni_output.
uncond_embed = build_decode_embeds(
model,
prev_codes=prev_codes,
subword_embed=session["cas_table"][t : t + 1].unsqueeze(0),
uncond=True,
).to(self._dtype)
session["uncond_hidden"] = session["uncond_stream"].step(uncond_embed)
session["pending_step"] = t
embeds = embeds.to(self._dtype)
if replay_rows:
embeds = torch.cat([*replay_rows, embeds], dim=0)
if prefill_head is not None:
embeds = torch.cat([prefill_head.to(embeds.dtype), embeds], dim=0)
return input_ids, embeds, {}
def _native_replay_embeds(self, session: dict[str, Any], t: int) -> torch.Tensor:
"""Decode embeds for an already-generated step ``t`` (KV recompute).
Reconstructs the exact feedback the original step saw: the codes
sampled at step ``t-1`` (or the speaker-prompt code for ``t == 1``)
with the same silence-on-EOS substitution.
"""
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
build_decode_embeds,
)
model = self.tts.tts_model
with torch.inference_mode():
template: torch.Tensor = session["code"]
if t == 1:
prev_codes = session["initial_code"]
else:
prev_codes = session["codes_rows"][t - 2].reshape(template.shape).to(dtype=template.dtype)
if session["silence_codes"] is not None:
is_eos = session["timeline"][t : t + 1].reshape(1, 1, 1) == session["text_eos_id"]
prev_codes = torch.where(is_eos, session["silence_codes"].expand_as(prev_codes), prev_codes)
embeds = build_decode_embeds(
model,
prev_codes=prev_codes,
subword_embed=session["cas_table"][t : t + 1].unsqueeze(0),
uncond=False,
)[0]
return embeds.to(self._dtype)
def _native_make_omni_output(self, hidden: torch.Tensor, **kwargs: Any) -> OmniOutput:
infos = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information") or []
spans = kwargs.get("request_token_spans")
flag = torch.zeros(hidden.shape[0], 2, dtype=torch.float32, device=hidden.device)
# Batch-aligned per-request lists: the generic mm splitter
# (to_payload_element) hands list element ``idx`` to request ``idx``
# but clones a shared tensor to EVERY request, so a single shared
# payload would leak the last request's codes into the others'
# streams. Requests with nothing to ship this step keep placeholder
# entries.
empty_codes = torch.empty(0, dtype=torch.long)
audio_list: list[torch.Tensor] = [empty_codes for _ in infos]
prompt_len_list: list[torch.Tensor] = [torch.zeros(1, dtype=torch.long) for _ in infos]
streaming_list: list[torch.Tensor] = [torch.zeros(1, dtype=torch.bool) for _ in infos]
emitted = False
def _emit(index: int, session_state: dict[str, Any]) -> None:
nonlocal emitted
audio_list[index] = torch.cat(session_state["codes_rows"], dim=0)
# 1-D tensors on purpose: the wire-payload builder indexes
# element.shape[0] (see the eager make_omni_output).
prompt_len_list[index] = torch.tensor([session_state["prompt_len"]], dtype=torch.long)
streaming_list[index] = torch.tensor([bool(session_state["codec_streaming"])], dtype=torch.bool)
emitted = True
for i, info in enumerate(infos):
if not isinstance(info, dict):
continue
request_id = info.get("request_id")
session = self._sessions.get(str(request_id)) if request_id is not None else None
if not session or not session.get("use_native"):
continue
pending = session.get("pending_step")
if pending is None:
# Prefill forward: no new codes this step, but keep the
# request's cumulative history in its own slot so the
# splitter never falls back to a shared tensor.
if session.get("codes_rows"):
_emit(i, session)
continue
session["pending_step"] = None
if spans is None:
raise RuntimeError("NemotronVoiceChat native talker needs request_token_spans from the runner.")
start, end = spans[i]
model = self.tts.tts_model
if session["uncond_stream"] is not None:
# CFG: pair the conditional row with this frame's uncond row
# ([cond; uncond] order — generate_step chunks in that order)
# and let generate_step apply NeMo's hidden-space blends
# (lm_head EOS blend + the post-MLP blend inside MoGHead.infer).
uncond_hidden = session["uncond_hidden"]
if uncond_hidden is None:
raise RuntimeError(
"NemotronVoiceChat native talker CFG desynchronized: no unconditional "
"hidden row for this frame (preprocess/make_omni_output mismatch)."
)
session["uncond_hidden"] = None
pair = torch.cat(
[hidden[end - 1 : end].reshape(1, 1, -1), uncond_hidden.reshape(1, 1, -1)],
dim=0,
).to(self._dtype)
if self._mog_graph is None:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
MoGStepGraph,
)
self._mog_graph = MoGStepGraph(model, session["generation_config"], batch=2)
codes = self._mog_graph.run(pair)
else:
if self._mog_graph is None:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
MoGStepGraph,
)
self._mog_graph = MoGStepGraph(model, session["generation_config"], batch=1)
codes = self._mog_graph.run(hidden[end - 1 : end].reshape(1, 1, -1))
session["code"] = codes
session["codes_rows"].append(codes.reshape(1, -1).to(torch.long))
session["step"] = int(pending) + 1
# Duplex (codec_streaming) sessions stop per segment when the
# received timeline is exhausted and resume with KV intact when
# the next thinker frame arrives; sync sessions stop at the end.
finished = int(session["step"]) >= int(session["timeline"].numel()) and self._segment_stops_at_exhaustion(
session
)
if finished:
flag[start:end, 0] = 1.0
_emit(i, session)
# Belt and braces: also mirror into the live per-request buffer so
# the flush-time full-payload producer can read it from
# additional_information as well as from the wire payload.
info["codes"] = {"audio": audio_list[i]}
info["meta"] = {
"nvc_tts_step": int(session["step"]),
"nvc_logical_prompt_len": int(session["prompt_len"]),
"codec_streaming": bool(session["codec_streaming"]),
"request_id": session["codec_request_id"],
}
outputs: dict[str, Any] = {}
if emitted:
outputs = {
"codes": {"audio": audio_list},
"meta": {
"nvc_logical_prompt_len": prompt_len_list,
"codec_streaming": streaming_list,
},
}
return OmniOutput(text_hidden_states=flag, multimodal_outputs=outputs)
def _refresh_session_timeline(self, session: dict[str, Any], info: dict[str, Any], device: torch.device) -> None:
"""Adopt a longer timeline from the latest async-chunk payload."""
latest = self._try_timeline(info)
if latest is not None and latest.numel() > int(session["timeline"].numel()):
session["timeline"] = latest.to(device)
if session.get("use_graph"):
self._step_graph.extend_timeline(session["timeline"])
if self._upstream_finished(info):
session["upstream_finished"] = True
@@ -374,15 +885,23 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
session["code"] = code
session["past_key_values"] = past_key_values
session["step"] = t + 1
# Native codec streaming uses one resumable scheduler segment per
# received timeline suffix. The pre-existing offline async-chunk path,
# however, must stay alive until the thinker sends its terminal marker;
# stopping at an intermediate cumulative snapshot starves Stage 2.
finished = (t + 1) >= int(session["timeline"].numel()) and (
bool(session["codec_streaming"]) or bool(session["upstream_finished"])
)
finished = (t + 1) >= int(session["timeline"].numel()) and self._segment_stops_at_exhaustion(session)
return code.reshape(1, -1).to(torch.long), finished
@staticmethod
def _segment_stops_at_exhaustion(session: dict[str, Any]) -> bool:
"""Whether exhausting the received timeline should emit the stop token.
Duplex serving submits RESUMABLE talker requests and marks them with
``codec_streaming``: exhausting the timeline ends the current scheduler
segment, and the parked request resumes (TTS state intact) when the
next thinker frame arrives. Offline requests are NOT resumable — a
stop token would end them permanently — so mid-stream exhaustion emits
CONTINUE instead, and the stop waits for the upstream's final chunk
(``meta.finished``; sync mode ships the whole timeline up front, so it
counts as finished from the start)."""
return bool(session["codec_streaming"]) or bool(session["upstream_finished"])
def preprocess(
self,
input_ids: torch.Tensor,
@@ -394,6 +913,9 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
span = int(input_ids.shape[0])
request_id = require_request_id(info, "talker")
if self._use_native_backbone:
return self._native_preprocess(input_ids, info, device, span, request_id)
embeds = torch.zeros((span, self._hidden), device=device, dtype=self._dtype)
if request_id not in self._sessions:
# The request id, not vLLM's per-segment prefill flag, is the
@@ -404,47 +926,56 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
session = self._sessions[request_id]
self._refresh_session_timeline(session, info, device)
# Sync mode: one NeMo TTS step per engine step (the verified parity
# path, unchanged). Async-chunk mode: DRAIN every received timeline
# position on each wake. The engine tokens are placeholders
# (CONTINUE/STOP) and the codes ride the cumulative payload, so
# multiple TTS steps per engine step are safe — and necessary:
# DRAIN every received timeline position on each wake, in both modes.
# The engine tokens are placeholders (CONTINUE/STOP) and the codes ride
# the cumulative payload, so multiple TTS steps per engine step are
# safe — and necessary:
# * sync mode ships the WHOLE timeline up front, so the first decode
# wake runs every TTS step back-to-back instead of paying a full
# engine round trip (scheduler + sampler + IPC) per 80 ms frame;
# the codes are computed in the same order as the stepwise loop,
# so the output is bit-identical,
# * the save_async background thread can coalesce thinker steps into
# one chunk (fewer wakes than new positions), and
# * the prompt-region PAD steps (logical_prompt_len - 1 of them) all
# become available with chunk 0 and must not cost one upstream
# chunk each.
# Rows stay on the GPU (tiny [1, Q] longs): a per-frame .cpu() would
# stall the dispatch pipeline once per frame; the stage producers move
# the final payload to CPU when shipping.
finished = False
steps_run = 0
new_code_rows: list[torch.Tensor] = []
codec_streaming = bool(session["codec_streaming"])
while int(session["step"]) < int(session["timeline"].numel()):
codes_row, finished = self._step_session(session)
if codec_streaming:
# One D2H copy per wake is enough; avoid synchronizing after every drained TTS step.
new_code_rows.append(codes_row)
else:
session["codes_rows"].append(codes_row.cpu())
steps_run += 1
if session["sync_mode"]:
break
total = int(session["timeline"].numel())
if session.get("use_graph"):
steps_run = total - int(session["step"])
if steps_run > 0:
self._step_graph.run_steps(steps_run)
session["step"] = total
finished = self._segment_stops_at_exhaustion(session)
else:
while int(session["step"]) < total:
codes_row, finished = self._step_session(session)
session["codes_rows"].append(codes_row)
steps_run += 1
if steps_run == 0:
# Zero-progress wake (duplicate/terminal-marker chunk): no TTS step
# was run, so never fabricate a frame. Stop the current scheduler
# segment only for native codec streaming, or once the ordinary
# offline async producer has sent its terminal marker.
finished = int(session["step"]) >= int(session["timeline"].numel()) and (
bool(session["codec_streaming"]) or bool(session["upstream_finished"])
# was run, so never fabricate a frame. Stop only when this session
# may end at exhaustion (see _segment_stops_at_exhaustion);
# otherwise emit CONTINUE and wait for more upstream input.
finished = int(session["step"]) >= int(session["timeline"].numel()) and self._segment_stops_at_exhaustion(
session
)
if finished:
embeds[:, 0] = 1.0
return input_ids, embeds, {}
if codec_streaming:
# The code2wav producer owns suffix slicing and therefore requires
# cumulative history on every wake. Keep the history on CPU and do
# one D2H transfer for all rows produced by this wake.
session["codes_rows"].append(torch.cat(new_code_rows, dim=0).cpu())
cumulative = torch.cat(session["codes_rows"], dim=0)
# The code2wav producers own the suffix slicing and therefore receive
# the cumulative history on every wake. Rows stay on the GPU (tiny
# [1, Q] longs; a per-step .cpu() would stall dispatch once per frame);
# the producers move the shipped payload to CPU.
if session.get("use_graph"):
cumulative = self._step_graph.codes_rows(1, int(session["step"]))
else:
cumulative = torch.cat(session["codes_rows"], dim=0)
if finished:
embeds[:, 0] = 1.0 # stop flag -> compute_logits emits the stop token
info_update: dict[str, Any] = {
@@ -466,6 +997,8 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
for request_id in finished_req_ids:
self._sessions.pop(str(request_id), None)
if self._uncond_graph is not None:
self._uncond_graph.release(str(request_id))
# ------------------------------------------------------------------
# Weights.
@@ -497,22 +1030,49 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
tts = DuplexEARTTS({"data": tts_data, "model": tts_model_cfg})
prefix = "tts_model."
backbone_prefix = "tts_model.tts_model.backbone."
state: dict[str, torch.Tensor] = {}
backbone_weights: list[tuple[str, torch.Tensor]] = []
for name, tensor in weights:
if not name.startswith(prefix):
continue
if self._use_native_backbone and name.startswith(backbone_prefix):
# The backbone runs as a native vLLM Gemma3 model instead.
backbone_weights.append((name, tensor))
if not self._native_guidance:
continue
# CFG keeps a second copy in the vendored module: the
# unconditional stream decodes on the HF backbone.
target = name.removeprefix(prefix)
if target.startswith("audio_codec."):
state[target] = tensor.to(dtype=torch.float32)
else:
state[target] = tensor.to(dtype=self._dtype)
if self._use_native_backbone:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
iter_backbone_weights,
)
loaded = self.backbone.load_weights(
iter_backbone_weights(backbone_weights, hidden_size=self._hidden, dtype=self._dtype)
)
logger.info("NemotronVoiceChat native talker backbone loaded %d vLLM tensors", len(loaded))
if not state:
raise ValueError(
"No 'tts_model.*' tensors found in the checkpoint; the NemotronVoiceChat "
"unified model.safetensors is required for the talker stage."
)
if self._use_native_backbone and not self._native_guidance:
# The HF backbone inside the vendored model is unused on the native
# path (only the embedding/CAS/MoG modules are consulted); drop it
# before loading so its ~2.4 GiB of fp32 parameters are freed and
# its (rerouted) weights are not reported missing. With CFG the
# backbone stays: the unconditional stream decodes on it.
tts.tts_model.backbone = nn.Identity()
missing, unexpected = tts.load_state_dict(state, strict=False)
real_missing = [m for m in missing if not m.startswith("audio_codec.")]
if self._use_native_backbone and not self._native_guidance:
real_missing = [m for m in real_missing if not m.startswith("tts_model.backbone.")]
if real_missing:
raise ValueError(f"NemotronVoiceChat talker is missing TTS weights: {sorted(real_missing)[:10]} ...")
if unexpected:
@@ -533,4 +1093,40 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
self._speaker_name,
self._guidance_enabled(),
)
if self._use_cuda_graphs:
from vllm_omni.model_executor.models.nemotron_voicechat.talker_graph import (
TalkerStepGraph,
)
step_graph = TalkerStepGraph(
self.tts,
dtype=self._dtype,
device=torch.device(device) if not isinstance(device, torch.device) else device,
max_cache_len=self._graph_max_cache_len,
guidance_enabled=self._guidance_enabled(),
)
supported, reason = step_graph.config_supported()
if supported:
self._step_graph = step_graph
else:
logger.warning(
"NemotronVoiceChat talker CUDA graphs requested but unsupported (%s); "
"using the eager per-frame step.",
reason,
)
if self._use_native_backbone:
# Capture the per-frame MoG sampling graph at load time; a lazy
# first-frame capture would otherwise cost the first session a
# few hundred ms of first-packet latency.
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
MoGStepGraph,
)
mog_batch = 2 if self._native_guidance else 1
self._mog_graph = MoGStepGraph(
self.tts.tts_model,
self.tts._get_generation_config(self._native_guidance),
batch=mog_batch,
)
self._mog_graph.ensure_captured(torch.zeros(mog_batch, 1, self._hidden, dtype=self._dtype, device=device))
return {name for name, _ in self.named_parameters()}
@@ -61,43 +61,74 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
# Mel columns kept ahead of the earliest column a perception chunk consumes,
# so reflect-pad / preemphasis boundary effects at the trimmed rolling-window
# head (which only reach the first ~2 columns) never touch a consumed column.
_DUPLEX_MEL_MARGIN_COLS = 8
_LM_ARCHITECTURE = "NemotronHForCausalLM"
def slice_perception_streaming_mel(
processed_signal: torch.Tensor,
def _streaming_cfg_at(value: Any, index: int) -> int:
return int(value[index] if isinstance(value, list | tuple) else value)
def perception_chunk_geometry(
frame_idx: int,
stream_total_cols: int,
streaming_cfg: Any,
) -> tuple[torch.Tensor, int]:
"""Slice one 80 ms mel chunk using NeMo's cache-manager geometry."""
def _at(value: Any, index: int) -> int:
return int(value[index] if isinstance(value, list | tuple) else value)
chunk_size_first = _at(streaming_cfg.chunk_size, 0)
chunk_size = _at(streaming_cfg.chunk_size, 1)
shift_size_first = _at(streaming_cfg.shift_size, 0)
shift_size = _at(streaming_cfg.shift_size, 1)
pre_encode_first = _at(streaming_cfg.pre_encode_cache_size, 0)
pre_encode_size = _at(streaming_cfg.pre_encode_cache_size, 1)
if frame_idx == 0:
mel_chunk = processed_signal[:, :, :chunk_size_first]
if pre_encode_first > 0:
mel_chunk = torch.nn.functional.pad(mel_chunk, (pre_encode_first, 0))
return mel_chunk, 0
) -> tuple[int, int, int]:
"""Stream-global (cache_start, chunk_start, chunk_end) mel columns for ``frame_idx > 0``."""
chunk_size = _streaming_cfg_at(streaming_cfg.chunk_size, 1)
shift_size_first = _streaming_cfg_at(streaming_cfg.shift_size, 0)
shift_size = _streaming_cfg_at(streaming_cfg.shift_size, 1)
pre_encode_size = _streaming_cfg_at(streaming_cfg.pre_encode_cache_size, 1)
chunk_start = shift_size_first + (frame_idx - 1) * shift_size
chunk_end = chunk_start + chunk_size
# Match NeMo's tail-boundary rule. With one 80 ms frame per call this
# normally leaves the expected trailing STFT columns unused.
offset = chunk_size - shift_size_first
if chunk_end > processed_signal.shape[-1] - offset:
chunk_end = processed_signal.shape[-1] - offset
if chunk_end > stream_total_cols - offset:
chunk_end = stream_total_cols - offset
chunk_start = chunk_end - chunk_size
main_chunk = processed_signal[:, :, chunk_start:chunk_end]
cache_start = max(0, chunk_start - pre_encode_size)
cache_mel = processed_signal[:, :, cache_start:chunk_start]
return cache_start, chunk_start, chunk_end
def slice_perception_streaming_mel(
processed_signal: torch.Tensor,
frame_idx: int,
streaming_cfg: Any,
*,
window_col0: int = 0,
) -> tuple[torch.Tensor, int]:
"""Slice one 80 ms mel chunk using NeMo's cache-manager geometry.
``processed_signal`` may be the mel of a trailing window of the stream
instead of the full history; ``window_col0`` is the stream-global column
index of its first column (the window must end at the stream tail).
"""
chunk_size_first = _streaming_cfg_at(streaming_cfg.chunk_size, 0)
pre_encode_first = _streaming_cfg_at(streaming_cfg.pre_encode_cache_size, 0)
pre_encode_size = _streaming_cfg_at(streaming_cfg.pre_encode_cache_size, 1)
if frame_idx == 0:
if window_col0 != 0:
raise ValueError("perception stream start requires an untrimmed mel window")
mel_chunk = processed_signal[:, :, :chunk_size_first]
if pre_encode_first > 0:
mel_chunk = torch.nn.functional.pad(mel_chunk, (pre_encode_first, 0))
return mel_chunk, 0
stream_total_cols = window_col0 + int(processed_signal.shape[-1])
cache_start, chunk_start, chunk_end = perception_chunk_geometry(frame_idx, stream_total_cols, streaming_cfg)
if cache_start < window_col0:
raise ValueError(
f"perception mel window starts at column {window_col0} but frame {frame_idx} needs column {cache_start}"
)
main_chunk = processed_signal[:, :, chunk_start - window_col0 : chunk_end - window_col0]
cache_mel = processed_signal[:, :, cache_start - window_col0 : chunk_start - window_col0]
if cache_mel.shape[-1] < pre_encode_size:
cache_mel = torch.nn.functional.pad(
cache_mel,
@@ -300,12 +331,14 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
# Stateful per-request execution is only validated at batch size 1
# (the shipped offline scope); fail fast on silent multi-request use.
# Multi-session batching: all per-request state (audio frames, fused
# prefill embeds, function-token feedback) lives in _sessions keyed by
# request id; the runner drives preprocess per request and hands
# postprocess each request's own hidden-state slice, and the NemotronH
# backbone batches natively on paged KV. Validated at max_num_seqs<=4.
max_num_seqs = int(getattr(vllm_config.scheduler_config, "max_num_seqs", 1))
if max_num_seqs != 1:
raise NotImplementedError(
f"NemotronVoiceChat thinker supports max_num_seqs=1 only (got {max_num_seqs}); "
"multi-request batching of the per-request fusion state is not implemented."
)
if max_num_seqs > 1:
logger.info("NemotronVoiceChat thinker: multi-session batching enabled (max_num_seqs=%d)", max_num_seqs)
# request_id -> per-request state (audio frames, prefill embeds, counters).
self._sessions: dict[str, dict[str, Any]] = {}
@@ -367,11 +400,16 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
if isinstance(model_outputs, OmniOutput):
return model_outputs
multimodal_outputs: dict[str, torch.Tensor] = {}
if self._use_function_head and model_outputs.numel():
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information")
single_request = isinstance(info_dicts, list) and len(info_dicts) == 1
if self._use_function_head and model_outputs.numel() and single_request:
# Only a single-request batch can attribute the batch's last row
# to a request. With several concurrent requests this key is
# omitted and postprocess computes each request's token from its
# own hidden-state slice instead (and applies any forced token).
with torch.inference_mode():
function_token = self.function_head(model_outputs[-1:, :].to(self._dtype)).argmax(dim=-1)
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information")
info = info_dicts[0] if isinstance(info_dicts, list) and len(info_dicts) == 1 else None
info = info_dicts[0]
request_id = info.get("request_id") if isinstance(info, dict) else None
session = self._sessions.get(request_id) if isinstance(request_id, str) else None
forced_token = session.get("forced_function_token") if isinstance(session, dict) else None
@@ -474,25 +512,51 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
)
audio = torch.cat([audio, frame_audio])
else:
# Stream start. A session normally starts at input sequence 1;
# seeing a later sequence means model-side session state was
# dropped mid-stream (request abort racing an already-queued step,
# or a stage-request re-open). Restart perception from this frame
# instead of failing the engine step.
if source_input_seq != 1:
raise ValueError(
f"Nemotron VoiceChat cache-aware perception must start at input sequence 1, got {source_input_seq}"
logger.warning(
"Nemotron VoiceChat perception session reset mid-stream; restarting at input sequence %d.",
source_input_seq,
)
session["duplex_seq_base"] = source_input_seq - 1
session["duplex_window_col0"] = 0
audio = frame_audio
frame_idx = source_input_seq - 1 - int(session.get("duplex_seq_base", 0))
encoder = self.perception.encoder
streaming_cfg = encoder.streaming_cfg
window_col0 = int(session.get("duplex_window_col0", 0))
if frame_idx > 0:
# Bound the rolling audio window: each mel column is local (STFT
# centered on hop-aligned samples, no cross-stream normalization),
# so only the columns the chunk slice consumes need to exist.
# Keep a margin ahead of cache_start so reflect-pad/preemphasis
# boundary effects at the trimmed window head never reach a
# consumed column.
hop = int(getattr(self._streaming_preprocessor.featurizer, "hop_length", 160))
total_samples = window_col0 * hop + int(audio.numel())
stream_total_cols = total_samples // hop + 1
cache_start, _, _ = perception_chunk_geometry(frame_idx, stream_total_cols, streaming_cfg)
desired_col0 = max(0, cache_start - _DUPLEX_MEL_MARGIN_COLS)
if desired_col0 > window_col0:
audio = audio[(desired_col0 - window_col0) * hop :]
window_col0 = desired_col0
with torch.inference_mode():
processed_signal, _ = self._streaming_preprocessor(
input_signal=audio.unsqueeze(0),
length=torch.tensor([audio.numel()], device=device, dtype=torch.long),
)
frame_idx = source_input_seq - 1
mel_chunk, drop_extra_pre_encoded = slice_perception_streaming_mel(
processed_signal,
frame_idx,
streaming_cfg,
window_col0=window_col0,
)
if frame_idx == 0:
caches = encoder.get_initial_cache_state(
@@ -538,6 +602,7 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
)
stable = stable[0].to(self._dtype)
session["duplex_audio"] = audio
session["duplex_window_col0"] = window_col0
session["perception_cache_last_channel"] = cache_channel
session["perception_cache_last_time"] = cache_time
session["perception_cache_last_channel_len"] = cache_channel_len
@@ -599,12 +664,14 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
self._sync_forced_function_response(session, runtime_config)
if source_input_seq <= 1:
offset = max(0, int(info.get("_omni_num_computed_tokens", 0) or 0))
prefill = session["prefill_embeds"]
span = int(input_ids.shape[0])
if offset + span > int(prefill.shape[0]):
raise ValueError("Nemotron VoiceChat duplex first append exceeds its fused prompt")
offset = max(0, int(info.get("_omni_num_computed_tokens", 0) or 0))
span = int(input_ids.shape[0])
prefill = session["prefill_embeds"]
if offset + span <= int(prefill.shape[0]):
# Engine positions inside the fused prompt prefill (possibly
# chunked). The position decides, not the duplex input sequence:
# after a mid-stream session reset the re-opened request prefills
# its prompt again at a sequence far beyond 1.
return (
input_ids,
prefill[offset : offset + span].to(self._dtype),
@@ -887,14 +954,18 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
if isinstance(request_id, str):
session = self._sessions.get(request_id)
if session is not None:
# StreamingUpdate replaces the runner payload at each append;
# retain the function channel in model-owned session state.
session["func_token"] = int(token.item())
forced = session.pop("forced_function_token", None)
if isinstance(forced, int):
queue = session.get("forced_function_tokens")
if not isinstance(queue, list) or not queue or queue.pop(0) != forced:
raise RuntimeError("Nemotron VoiceChat forced function-token queue lost alignment")
# Apply the override here too: with several concurrent
# requests make_omni_output omits the shared token, so the
# token above is this request's own computed one.
token = torch.full_like(token, forced)
# StreamingUpdate replaces the runner payload at each append;
# retain the function channel in model-owned session state.
session["func_token"] = int(token.item())
update: dict[str, Any] = {"nvc_prev_function_token": token}
if os.environ.get("NEMOTRON_VOICECHAT_DEBUG_FUNCTION_TIMELINE", "0") == "1":
existing = kwargs.get("nvc_function_tokens")
@@ -0,0 +1,329 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CUDA-graph fast path for the NemotronVoiceChat talker per-frame step.
The eager talker step is launch-bound: one 80 ms frame costs ~3200 kernel
launches (28-layer Gemma3 backbone + 5 MoG unmasking passes + a 31-level
depthsum loop) for ~9 ms of actual GPU work, so the eager wall time is
~25-34 ms/frame. This module captures the WHOLE step into one CUDA graph so
the drain loop degenerates to ``graph.replay()`` per frame (~12 ms/frame,
GPU-bound). This mirrors NVIDIA's own deployment of this model, which serves
EAR-TTS through vLLM decode CUDA graphs with CFG baked into the capture.
What makes the step capturable:
* ``StaticCache`` (preallocated KV, write index via ``cache_position``)
instead of a growing HF ``DynamicCache``;
* the CharAwareSubwordEncoder — dynamic char-length shapes plus per-call
``.cpu().tolist()`` / ``subword_mask.any()`` host syncs — is hoisted OUT of
the step: the frame-locked text timeline is known ahead of the audio, so its
subword embeddings are precomputed in one batched call per session and the
in-graph step just gathers a row (``subword_embeds`` fast path in the
vendored model);
* the code feedback, output row write, step counter, and cache position all
advance INSIDE the graph, so a replay is self-contained.
The captured state machine (all device tensors, no host round trips):
cur_id = timeline_ids[step_ctr] # for EOS-silence gating
cur_emb = cas_table[step_ctr] # precomputed CAS row
code = one EAR-TTS step (backbone + MoG, CFG batch=2, StaticCache)
code_buf, codes_out[step_ctr] = code # feedback + output
step_ctr += 1; cache_pos += 1
MoG sampling noise draws from the CUDA graph-safe Philox generator, which
advances per replay — audio is NOT bit-identical to the eager path (nor is the
eager path bit-stable across process RNG states); the frame-locked text channel
is unaffected. Sessions longer than ``max_cache_len`` fall back to the eager
path (the talker decides per session).
"""
from __future__ import annotations
from typing import Any
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
class TalkerStepGraph:
"""Owns the StaticCache, the per-session buffers, and the captured graph.
Sessions are strictly serial (the talker enforces ``max_num_seqs=1``); the
same buffers and capture are reused across requests via ``start_session``.
"""
def __init__(
self,
tts: Any,
*,
dtype: torch.dtype,
device: torch.device,
max_cache_len: int,
guidance_enabled: bool,
) -> None:
self.tts = tts
self.model = tts.tts_model
self.device = device
self.dtype = dtype
self.max_cache_len = int(max_cache_len)
self.guidance_enabled = bool(guidance_enabled)
self.generation_config = tts._get_generation_config(self.guidance_enabled)
# Attention in StaticCache mode runs over the FULL padded cache length,
# so per-step cost scales with the bucket size (~11 ms/step at 512
# positions vs ~20 ms/step at 4096 in fp32 on H100). Sessions with a
# known timeline (sync mode) pick the smallest fitting bucket; growing
# (async) sessions use the largest, since codes already shipped cannot
# be regenerated under a different cache length mid-request.
self._bucket_lens = [b for b in (512, 1024, 2048) if b < self.max_cache_len] + [self.max_cache_len]
self._buckets: dict[int, dict[str, Any]] = {}
self._active_bucket: int | None = None
self._broken = False
self._init_len: int | None = None
self._timeline_len = 0
hidden = int(self.model.hidden_size)
quantizers = int(self.model.config.num_quantizers)
with torch.inference_mode():
self._timeline_ids = torch.zeros(self.max_cache_len, dtype=torch.long, device=device)
self._cas_table = torch.zeros(self.max_cache_len, hidden, dtype=dtype, device=device)
self._codes_out = torch.zeros(self.max_cache_len, quantizers, dtype=torch.long, device=device)
self._code_buf = torch.zeros(1, 1, quantizers, dtype=torch.long, device=device)
self._step_ctr = torch.ones(1, dtype=torch.long, device=device)
self._cache_pos = torch.zeros(1, dtype=torch.long, device=device)
self._ones_mask = torch.ones(1, 1, dtype=torch.bool, device=device)
# Prepared 4D attention mask over the cache slots (True = attend).
# Passing a ready 4D mask makes transformers' mask pipeline return
# it as-is; the HF-derived mask (and anything else that reads the
# cache's python-int lengths) would otherwise bake the capture-time
# position into the graph. Per-bucket graphs slice a leading view.
self._attn_mask = torch.zeros(1, 1, 1, self.max_cache_len, dtype=torch.bool, device=device)
self._cache: Any = None # the ACTIVE bucket's StaticCache
# ------------------------------------------------------------------
# Support / capacity checks.
# ------------------------------------------------------------------
def config_supported(self) -> tuple[bool, str]:
"""Static-config gate for the graph path (independent of the request)."""
if self.device.type != "cuda":
return False, f"device {self.device} is not CUDA"
if not self.guidance_enabled:
return False, "classifier-free guidance disabled (capture assumes the CFG batch layout)"
if self.tts.cfg.tts_config.context_hidden_size is not None:
return False, "context-LM conditioning enabled (prev-subword context is not captured)"
if self.model.embed_subword is None:
return False, "no CharAwareSubwordEncoder to precompute"
sliding_window = getattr(self.model.backbone.config, "sliding_window", None)
if sliding_window is not None and self.max_cache_len >= int(sliding_window):
# The single prepared mask stands in for both the full and the
# sliding mask, which is only equivalent while no position can
# fall outside the window.
return False, (
f"max_cache_len {self.max_cache_len} >= backbone sliding_window {sliding_window}; "
"the prepared causal mask would diverge from the sliding mask"
)
return True, ""
def fits(self, timeline_len: int) -> bool:
init_len = self._init_len if self._init_len is not None else 64
return timeline_len <= self.max_cache_len - init_len - 1
def _bucket_len_for(self, timeline_len: int | None) -> int:
"""Smallest bucket that fits ``init_len + timeline_len + 1`` positions.
``None`` (growing async timeline) selects the largest bucket: codes
already shipped cannot be regenerated under a different cache length.
"""
if timeline_len is None:
return self._bucket_lens[-1]
init_len = self._init_len if self._init_len is not None else 64
needed = init_len + timeline_len + 1
for bucket in self._bucket_lens:
if needed <= bucket:
return bucket
return self._bucket_lens[-1]
# ------------------------------------------------------------------
# Capture.
# ------------------------------------------------------------------
def _get_bucket(self, bucket_len: int) -> dict[str, Any]:
bucket = self._buckets.get(bucket_len)
if bucket is None:
from transformers import StaticCache
bucket = {
"cache": StaticCache(
config=self.model.backbone.config,
max_batch_size=2 if self.guidance_enabled else 1,
max_cache_len=bucket_len,
device=self.device,
dtype=self.dtype,
),
"graph": None,
}
self._buckets[bucket_len] = bucket
return bucket
def _init_forward(self) -> None:
"""Eager speaker-prompt prefill into the StaticCache (per session)."""
init_inputs = self.tts.get_init_inputs(B=1)
init_len = int(init_inputs["code"].shape[1])
if self._init_len is None:
self._init_len = init_len
elif init_len != self._init_len:
raise RuntimeError(
f"NemotronVoiceChat talker graph: init prompt length changed ({self._init_len} -> {init_len}); "
"the captured graph assumes a fixed speaker prompt."
)
init_inputs.update(
{
"use_cache": True,
"past_key_values": self._cache,
"guidance_enabled": self.guidance_enabled,
"cache_position": torch.arange(init_len, device=self.device),
}
)
self.model(**init_inputs)
self._cache_pos.fill_(init_len)
self._code_buf.copy_(init_inputs["code"][:, -1:])
# Seed the prepared mask: the speaker-prompt slots are attendable, the
# per-step slot bits are set inside the captured step.
self._attn_mask.zero_()
self._attn_mask[..., :init_len] = True
def _one_step(self) -> None:
"""The captured body: one EAR-TTS frame step + in-graph state advance."""
cur_id = self._timeline_ids.index_select(0, self._step_ctr).reshape(1, 1)
cur_emb = self._cas_table.index_select(0, self._step_ctr).reshape(1, 1, -1)
# Open this step's own cache slot before attending (a token attends to
# itself), then run the step with fully tensor-driven mask/positions.
self._attn_mask.index_fill_(3, self._cache_pos, True)
code, _ = self.tts.infer_codes_one_step(
current_subword_id=cur_id,
# prev_subword_id only feeds the context-LM channel, which
# config_supported() guarantees is disabled.
prev_subword_id=cur_id,
current_subword_mask=self._ones_mask,
prev_audio_tokens=self._code_buf,
past_key_values=self._cache,
guidance_enabled=self.guidance_enabled,
generation_config=self.generation_config,
ignore_eos_flag_stop=True,
current_subword_embed=cur_emb,
cache_position=self._cache_pos,
attention_mask=self._attn_mask[..., : self._active_bucket],
position_ids=self._cache_pos.reshape(1, 1),
)
self._code_buf.copy_(code)
self._codes_out.index_copy_(0, self._step_ctr, code.reshape(1, -1))
self._step_ctr.add_(1)
self._cache_pos.add_(1)
def ensure_captured(self, timeline_len: int | None = None) -> bool:
"""Warm up and capture the bucket for this timeline (throwaway session)."""
if self._broken:
return False
bucket_len = self._bucket_len_for(timeline_len)
bucket = self._get_bucket(bucket_len)
if bucket["graph"] is not None:
return True
try:
with torch.inference_mode():
# Throwaway session: buffer VALUES are irrelevant for capture,
# only shapes/addresses matter. PAD-filled timeline rows and a
# zero CAS table are fine.
self._cache = bucket["cache"]
self._active_bucket = bucket_len
self._cache.reset()
self._step_ctr.fill_(1)
self._cache_pos.zero_()
self._init_forward()
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side):
for _ in range(3):
self._one_step()
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
self._one_step()
bucket["graph"] = graph
logger.info(
"NemotronVoiceChat talker CUDA graph captured (bucket=%d, init_len=%d)",
bucket_len,
self._init_len,
)
return True
except Exception:
self._broken = True
logger.exception(
"NemotronVoiceChat talker CUDA graph capture failed; falling back to the eager per-frame step."
)
return False
# ------------------------------------------------------------------
# Session lifecycle.
# ------------------------------------------------------------------
def start_session(self, timeline: torch.Tensor, *, expected_len: int | None) -> None:
"""Reset all captured state and prefill for a new request.
``expected_len`` is the FINAL timeline length when known up front
(sync mode ships the whole timeline; frame-locked async producers ship
meta.expected_total_tokens) and selects the smallest fitting bucket;
``None`` (unbounded growth, e.g. duplex) selects the largest bucket.
"""
bucket_len = self._bucket_len_for(expected_len)
bucket = self._buckets[bucket_len]
assert bucket["graph"] is not None, "ensure_captured() must succeed before start_session()"
with torch.inference_mode():
self._cache = bucket["cache"]
self._active_bucket = bucket_len
self._cache.reset()
self._step_ctr.fill_(1)
self._cache_pos.zero_()
self._codes_out.zero_()
self._timeline_len = 0
self._init_forward()
self.extend_timeline(timeline)
def extend_timeline(self, timeline: torch.Tensor) -> None:
"""Adopt a (possibly longer) timeline; CAS-encode only the new rows."""
new_len = int(timeline.numel())
if new_len <= self._timeline_len:
return
if not self.fits(new_len):
raise RuntimeError(
f"NemotronVoiceChat talker graph: timeline of {new_len} positions exceeds the "
f"captured capacity ({self.max_cache_len - (self._init_len or 0) - 1} frames). Raise "
"hf_overrides.talker_graph_max_cache_len or disable hf_overrides.use_talker_cuda_graphs."
)
active = self._active_bucket
if active is not None and (self._init_len or 0) + new_len + 1 > active:
raise RuntimeError(
f"NemotronVoiceChat talker graph: timeline of {new_len} positions exceeds the active "
f"cache bucket ({active} positions). Raise hf_overrides.talker_graph_max_cache_len or "
"disable hf_overrides.use_talker_cuda_graphs."
)
start = self._timeline_len
with torch.inference_mode():
new_ids = timeline[start:new_len].to(device=self.device, dtype=torch.long).reshape(-1)
self._timeline_ids[start:new_len] = new_ids
mask = torch.ones(1, new_ids.numel(), dtype=torch.bool, device=self.device)
# One batched CAS call per chunk of new tokens (the encoder's host
# syncs and dynamic char shapes stay OUTSIDE the captured step).
embeds = self.model.embed_subword(new_ids.unsqueeze(0), mask)[0]
self._cas_table[start:new_len] = embeds.to(self.dtype)
self._timeline_len = new_len
def run_steps(self, num_steps: int) -> None:
graph = self._buckets[self._active_bucket]["graph"]
for _ in range(num_steps):
graph.replay()
def codes_rows(self, first_step: int, last_step: int) -> torch.Tensor:
"""Codes for NeMo steps ``first_step..last_step-1`` (row t = step t)."""
with torch.inference_mode():
return self._codes_out[first_step:last_step].clone()
@@ -0,0 +1,429 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Native-vLLM building blocks for the NemotronVoiceChat talker.
The eager/CUDA-graph talker executes the vendored HF Gemma3 backbone inside the
per-request ``preprocess`` hook, next to (not inside) the vLLM engine. The
native path (``hf_overrides.use_native_talker``) instead runs the backbone as a
real vLLM model — PagedAttention KV cache (capacity and per-step attention cost
scale with the ACTUAL session length, no StaticCache buckets), vLLM's own decode
CUDA graphs, and, later, multi-session batching. This mirrors NVIDIA's own
deployment, whose EAR-TTS is served by a vLLM engine.
This module hosts the pieces that are independent of the talker class:
* ``synthesize_backbone_config``: the checkpoint stores the backbone as a NeMo
``tts_config.backbone_config`` dict; vLLM needs a real ``Gemma3TextConfig``.
The embedding table is never used (the fused embeddings are built from
codes + text conditioning), so a tiny dummy vocab keeps it negligible.
* ``iter_backbone_weights``: remaps the checkpoint's
``tts_model.tts_model.backbone.*`` HF-Gemma3 tensors onto the names vLLM's
``Gemma3ForCausalLM`` loader expects, and fabricates the dummy
``model.embed_tokens.weight``.
* ``build_prefill_embeds`` / ``build_decode_embeds``: the per-token fused input
embeddings, numerically verified against the vendored
``RVQEARTTSModel.forward`` embedding section for both CFG roles (the
speaker-prompt prefill must use the checkpoint's pre-baked
``audio_prompt_latent``, not the frozen projection).
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn.functional as F
_DUMMY_VOCAB_SIZE = 8
def synthesize_backbone_config(tts_config: dict[str, Any]) -> Any:
"""Build a ``Gemma3TextConfig`` from the checkpoint's NeMo backbone dict."""
from transformers import AutoConfig
backbone_type = tts_config.get("backbone_type")
if backbone_type != "gemma3_text":
raise NotImplementedError(
f"NemotronVoiceChat native talker supports a gemma3_text backbone only (got {backbone_type!r})."
)
backbone_cfg = dict(tts_config.get("backbone_config") or {})
backbone_cfg.pop("use_cache", None)
config = AutoConfig.for_model(
backbone_type,
**backbone_cfg,
vocab_size=_DUMMY_VOCAB_SIZE,
tie_word_embeddings=True,
)
return config
def iter_backbone_weights(
weights: Iterable[tuple[str, torch.Tensor]],
*,
hidden_size: int,
dtype: torch.dtype,
) -> Iterable[tuple[str, torch.Tensor]]:
"""Yield vLLM-Gemma3 weights from the unified checkpoint stream.
Checkpoint names: ``tts_model.tts_model.backbone.<hf gemma3 names>``.
vLLM ``Gemma3ForCausalLM`` expects ``model.<hf gemma3 names>`` plus an
embedding table (tied lm_head); the embedding is never consulted (fused
embeddings are injected via ``inputs_embeds``), so a dummy is fabricated.
"""
prefix = "tts_model.tts_model.backbone."
for name, tensor in weights:
if name.startswith(prefix):
yield "model." + name.removeprefix(prefix), tensor.to(dtype=dtype)
yield (
"model.embed_tokens.weight",
torch.zeros(_DUMMY_VOCAB_SIZE, hidden_size, dtype=dtype),
)
def build_prefill_embeds(
model: Any,
*,
code: torch.Tensor,
subword_ids: torch.Tensor,
subword_mask: torch.Tensor,
audio_mask: torch.Tensor,
audio_prompt_latent: torch.Tensor | None,
uncond: bool,
) -> torch.Tensor:
"""Speaker-prompt prefill embeddings for one CFG role.
Mirrors the ``audio_mask`` inference branch of the vendored
``RVQEARTTSModel.forward`` (verified to ~4e-6 max abs diff against the
embeddings the vendored forward feeds its backbone). ``model`` is the
vendored ``RVQEARTTSModel``. Returns ``[1, T, H]``.
"""
shifted_code = F.pad(code[:, :-1], [0, 0, 1, 0])
code_embed = model.depthsum_embedding(shifted_code)
bos_mask = (audio_mask & (~F.pad(audio_mask[:, :-1], [1, 0]))).unsqueeze(-1)
pre_bos_mask = bos_mask.cumsum(dim=1) == 0
code_embed = model.embed_code(code_embed)
if model.config.get("use_audio_prompt_frozen_projection", False):
if audio_prompt_latent is None:
weight = model.audio_prompt_projection_W.to(code_embed.device, code_embed.dtype)
audio_prompt_latent = torch.nn.functional.linear(code_embed, weight.T)
code_embed = torch.where(pre_bos_mask, audio_prompt_latent, code_embed)
code_embeds = code_embed + bos_mask * model.bos_emb
flag = torch.full((code.size(0), 1, 1), bool(uncond), dtype=torch.bool, device=code.device)
cond = model._prepare_conditioning(None, subword_ids, subword_mask, flag)
if model.config.use_gated_fusion_for_text_audio:
return model.gated_fusion_audio_text(code_embeds, cond)
return code_embeds + cond
class MoGStepGraph:
"""One captured CUDA graph for the per-frame MoG sampling step.
On the native path the backbone runs under vLLM's own decode CUDA graphs,
but the MoG iterative unmasking (``RVQEARTTSModel.generate_step``: 5
effective MoG-head passes + a 31-level depthsum quantization loop) would
otherwise run eagerly in ``make_omni_output`` — ~1000 kernel launches for
~2 ms of GPU work per 80 ms frame. All shapes are static and the unmask
schedule is Python ints, so the whole step captures into one graph whose
replay is a single launch. MoG noise draws from the CUDA-graph-safe Philox
stream (fresh randomness per replay, not bit-identical to eager).
``batch`` is the leading hidden dim (1 for the single conditional stream;
2 once paired-CFG lands). Capture failures fall back to eager permanently.
"""
def __init__(self, model: Any, generation_config: dict[str, Any], *, batch: int = 1) -> None:
self.model = model
self.generation_config = dict(generation_config)
self.batch = int(batch)
self._graph: torch.cuda.CUDAGraph | None = None
self._broken = False
self._hidden_buf: torch.Tensor | None = None
self._code_buf: torch.Tensor | None = None
def _step(self) -> None:
codes, _, _ = self.model.generate_step(self._hidden_buf, ignore_eos_flag_stop=True, **self.generation_config)
self._code_buf.copy_(codes)
def ensure_captured(self, hidden: torch.Tensor) -> bool:
if self._graph is not None:
return True
if self._broken:
return False
try:
with torch.inference_mode():
self._hidden_buf = hidden.clone()
# Under CFG (guidance_scale set) generate_step chunks the
# [cond; uncond] pair and returns codes for the effective
# batch only.
out_batch = self.batch
if self.generation_config.get("guidance_scale") is not None:
out_batch = max(self.batch // 2, 1)
self._code_buf = torch.zeros(
out_batch, 1, int(self.model.config.num_quantizers), dtype=torch.long, device=hidden.device
)
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(side):
for _ in range(3):
self._step()
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
self._step()
self._graph = graph
return True
except Exception:
self._broken = True
self._graph = None
import logging
logging.getLogger(__name__).exception(
"NemotronVoiceChat native talker: MoG step graph capture failed; sampling eagerly."
)
return False
def run(self, hidden: torch.Tensor) -> torch.Tensor:
"""MoG-sample one frame from ``hidden`` ([batch, 1, H]) -> codes [batch, 1, Q]."""
if self.ensure_captured(hidden):
with torch.inference_mode():
self._hidden_buf.copy_(hidden)
self._graph.replay()
return self._code_buf.clone()
with torch.inference_mode():
codes, _, _ = self.model.generate_step(hidden, ignore_eos_flag_stop=True, **self.generation_config)
return codes
def build_decode_embeds(
model: Any,
*,
prev_codes: torch.Tensor,
subword_embed: torch.Tensor,
uncond: bool,
) -> torch.Tensor:
"""One decode-step fused embedding for one CFG role (``[1, 1, H]``).
``subword_embed`` is the precomputed CharAwareSubwordEncoder row for the
current timeline token (host-sync free; see the CAS table precompute).
"""
code_embeds = model.embed_code(model.depthsum_embedding(prev_codes))
flag = torch.full((prev_codes.size(0), 1, 1), bool(uncond), dtype=torch.bool, device=prev_codes.device)
cond = model._prepare_conditioning(None, None, None, flag, subword_embeds=subword_embed)
if model.config.use_gated_fusion_for_text_audio:
return model.gated_fusion_audio_text(code_embeds, cond)
return code_embeds + cond
class UncondStream:
"""The unconditional CFG stream, run on the vendored HF Gemma3 backbone.
NeMo applies classifier-free guidance by duplicating every backbone input
into a [cond; uncond] batch (``RVQEARTTSModel.forward``) whose second half
replaces the whole conditioning sum with ``null_emb``, then blending the
two hidden rows inside ``generate_step`` (lm_head EOS blend and the
post-MLP blend inside ``MoGHead.infer``).
On the native path the conditional stream's KV lives inside vLLM's paged
cache, where a second same-length sequence cannot ride along without
scheduler-level pair locking. This class instead mirrors the stream
position-for-position on the vendored HF backbone with its own
``DynamicCache`` — one extra [1, 1, H] decode step per frame. Both streams
are fed the SAME sampled code row each frame (prev-codes conditioning is
shared in NeMo's layout; only the text conditioning differs).
"""
def __init__(self, model: Any) -> None:
self.model = model
self.cache: Any = None
self.pos = 0
@torch.inference_mode()
def prefill(self, embeds: torch.Tensor) -> None:
"""Consume the speaker-prompt prefill embeddings (``[1, T, H]``)."""
from transformers import DynamicCache
self.cache = DynamicCache()
length = int(embeds.shape[1])
position_ids = torch.arange(length, device=embeds.device).unsqueeze(0)
outputs = self.model.backbone(
inputs_embeds=embeds,
attention_mask=None,
position_ids=position_ids,
past_key_values=self.cache,
use_cache=True,
)
self.cache = outputs.past_key_values
self.pos = length
@torch.inference_mode()
def step(self, embed: torch.Tensor) -> torch.Tensor:
"""One decode step (``[1, 1, H]`` in) -> uncond hidden (``[1, 1, H]``)."""
if self.cache is None:
raise RuntimeError("UncondStream.step called before prefill")
position_ids = torch.tensor([[self.pos]], device=embed.device)
outputs = self.model.backbone(
inputs_embeds=embed,
attention_mask=None,
position_ids=position_ids,
past_key_values=self.cache,
use_cache=True,
)
self.cache = outputs.past_key_values
self.pos += 1
return outputs.last_hidden_state
class UncondStepGraph:
"""Captured per-frame decode step for the unconditional CFG stream.
Backbone-only StaticCache stepping with the same capture hygiene as
``talker_graph.TalkerStepGraph``: a prepared 4D boolean attention mask
(triggers the HF early-exit so no python-int cache length is baked into
the graph), explicit ``position_ids``/``cache_position`` tensors, and the
in-graph slot-open ``index_fill_``. One session owns the graph at a time
(StaticCache addresses are capture-bound); concurrent sessions fall back
to the eager ``UncondStream``.
"""
_BUCKETS = (512, 1024, 2048, 4096)
def __init__(
self,
model: Any,
*,
hidden_size: int,
dtype: torch.dtype,
device: torch.device,
max_cache_len: int = 4096,
) -> None:
self.model = model
self.dtype = dtype
self.device = device
self.max_cache_len = int(max_cache_len)
self._buckets: dict[int, dict[str, Any]] = {}
self._broken = False
self.owner: str | None = None
self._embed_buf = torch.zeros(1, 1, hidden_size, dtype=dtype, device=device)
self._hidden_buf = torch.zeros(1, 1, hidden_size, dtype=dtype, device=device)
self._cache_pos = torch.zeros(1, dtype=torch.long, device=device)
self._attn_mask = torch.zeros(1, 1, 1, self.max_cache_len, dtype=torch.bool, device=device)
self._cache: Any = None
self._active_bucket = 0
def bucket_for(self, total_positions: int | None) -> int | None:
"""Smallest bucket holding ``total_positions``; None if it cannot fit."""
for bucket in self._BUCKETS:
if bucket > self.max_cache_len:
break
if total_positions is not None and total_positions <= bucket:
return bucket
largest = max(b for b in self._BUCKETS if b <= self.max_cache_len)
return largest if total_positions is None or total_positions <= largest else None
def _get_bucket(self, bucket_len: int) -> dict[str, Any]:
state = self._buckets.get(bucket_len)
if state is None:
from transformers import StaticCache
state = {
"cache": StaticCache(
config=self.model.backbone.config,
max_batch_size=1,
max_cache_len=bucket_len,
device=self.device,
dtype=self.dtype,
),
"graph": None,
}
self._buckets[bucket_len] = state
return state
def _one_step(self) -> None:
self._attn_mask.index_fill_(3, self._cache_pos, True)
outputs = self.model.backbone(
inputs_embeds=self._embed_buf,
attention_mask=self._attn_mask[..., : self._active_bucket],
position_ids=self._cache_pos.reshape(1, 1),
past_key_values=self._cache,
use_cache=True,
cache_position=self._cache_pos,
)
self._hidden_buf.copy_(outputs.last_hidden_state)
self._cache_pos.add_(1)
def start_session(self, request_id: str, prefill_embeds: torch.Tensor, total_positions: int | None) -> bool:
"""Claim the graph for one session and prefill its cache. False -> use eager."""
if self._broken or self.owner is not None:
return False
bucket_len = self.bucket_for(total_positions)
if bucket_len is None:
return False
try:
with torch.inference_mode():
state = self._get_bucket(bucket_len)
self._cache = state["cache"]
self._active_bucket = bucket_len
self._cache.reset()
init_len = int(prefill_embeds.shape[1])
self.model.backbone(
inputs_embeds=prefill_embeds.to(self.dtype),
position_ids=torch.arange(init_len, device=self.device).unsqueeze(0),
past_key_values=self._cache,
use_cache=True,
cache_position=torch.arange(init_len, device=self.device),
)
self._attn_mask.zero_()
self._attn_mask[..., :init_len] = True
self._cache_pos.fill_(init_len)
if state["graph"] is None:
side = torch.cuda.Stream()
side.wait_stream(torch.cuda.current_stream())
saved_pos = int(self._cache_pos.item())
with torch.cuda.stream(side):
for _ in range(3):
self._one_step()
torch.cuda.current_stream().wait_stream(side)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
self._one_step()
state["graph"] = graph
# Rewind the warmup/capture side effects: re-prefill so the
# cache holds only the speaker prompt again.
self._cache.reset()
self.model.backbone(
inputs_embeds=prefill_embeds.to(self.dtype),
position_ids=torch.arange(init_len, device=self.device).unsqueeze(0),
past_key_values=self._cache,
use_cache=True,
cache_position=torch.arange(init_len, device=self.device),
)
self._attn_mask.zero_()
self._attn_mask[..., :init_len] = True
self._cache_pos.fill_(saved_pos)
self.owner = request_id
return True
except Exception:
self._broken = True
import logging
logging.getLogger(__name__).exception(
"NemotronVoiceChat native talker: uncond step graph capture failed; stepping eagerly."
)
return False
def step(self, embed: torch.Tensor) -> torch.Tensor:
"""One captured decode step: embed [1,1,H] -> uncond hidden [1,1,H]."""
if int(self._cache_pos.item()) >= self._active_bucket:
raise RuntimeError(
f"NemotronVoiceChat uncond CFG stream exceeded its StaticCache bucket "
f"({self._active_bucket} positions)."
)
with torch.inference_mode():
self._embed_buf.copy_(embed.to(self.dtype))
self._buckets[self._active_bucket]["graph"].replay()
return self._hidden_buf.clone()
def release(self, request_id: str) -> None:
if self.owner == request_id:
self.owner = None