[Perf][NemotronVoiceChat] Fast defaults for all three deploy profiles: 2.8x offline, realtime full-duplex audio (#6354)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e43c787544
commit
4179b0cae1
22 files changed
+2606
-195
No files matched your search
@@ -705,6 +705,10 @@ class DuplexEARTTS(nn.Module):
|
||||
generation_config=None,
|
||||
ignore_eos_flag_stop=True,
|
||||
request_id=None,
|
||||
current_subword_embed=None,
|
||||
cache_position=None,
|
||||
attention_mask=None,
|
||||
position_ids=None,
|
||||
):
|
||||
"""
|
||||
Runs a single autoregressive prediction step to infer audio codec codes.
|
||||
@@ -756,6 +760,20 @@ class DuplexEARTTS(nn.Module):
|
||||
# request_id is only used by vLLM backend — skip for native PyTorch inference
|
||||
if request_id is not None:
|
||||
inputs["request_id"] = request_id
|
||||
# Fast-path extras (see RVQEARTTSModel.forward): a precomputed subword
|
||||
# embedding bypasses the CAS encoder (host-sync free); cache_position
|
||||
# enables StaticCache decoding; an explicit (4D) attention_mask and
|
||||
# position_ids keep the mask and RoPE fully tensor-driven, which CUDA
|
||||
# graph capture requires (the HF-derived paths read python-int cache
|
||||
# lengths that would be baked into the captured graph).
|
||||
if current_subword_embed is not None:
|
||||
inputs["subword_embeds"] = current_subword_embed
|
||||
if cache_position is not None:
|
||||
inputs["cache_position"] = cache_position
|
||||
if attention_mask is not None:
|
||||
inputs["attention_mask"] = attention_mask
|
||||
if position_ids is not None:
|
||||
inputs["position_ids"] = position_ids
|
||||
|
||||
outputs = self.tts_model(**inputs)
|
||||
|
||||
|
||||
@@ -1141,14 +1141,24 @@ class RVQEARTTSModel(nn.Module):
|
||||
subword_ids: Tensor | None,
|
||||
subword_mask: Tensor | None,
|
||||
uncond_dec_flag: Tensor,
|
||||
subword_embeds: Tensor | None = None,
|
||||
) -> Tensor:
|
||||
"""Computes the final conditioning tensor by combining all sources."""
|
||||
"""Computes the final conditioning tensor by combining all sources.
|
||||
|
||||
``subword_embeds`` short-circuits the CharAwareSubwordEncoder with a
|
||||
precomputed embedding (same shape as its output). Besides skipping the
|
||||
char-level backbone, this path is free of host syncs
|
||||
(``subword_mask.any()`` / the encoder's ``.cpu().tolist()``), which
|
||||
makes it CUDA-graph capturable.
|
||||
"""
|
||||
cond = torch.zeros((1, 1, self.hidden_size), device=uncond_dec_flag.device)
|
||||
|
||||
if self.embed_context is not None and context_hidden_state is not None:
|
||||
cond = cond + self.embed_context(context_hidden_state)
|
||||
|
||||
if self.embed_subword is not None and subword_ids is not None:
|
||||
if subword_embeds is not None:
|
||||
cond = cond + subword_embeds
|
||||
elif self.embed_subword is not None and subword_ids is not None:
|
||||
# Infer subword mask from context if not provided
|
||||
if subword_mask is None and context_hidden_state is not None:
|
||||
subword_mask = torch.any(context_hidden_state != 0, dim=-1)
|
||||
@@ -1244,6 +1254,8 @@ class RVQEARTTSModel(nn.Module):
|
||||
ignore_eos_flag_stop: bool = False,
|
||||
asr_speech_tokens_emb: Tensor | None = None,
|
||||
audio_prompt_latent: Tensor | None = None,
|
||||
subword_embeds: Tensor | None = None,
|
||||
cache_position: Tensor | None = None,
|
||||
) -> RVQEARTTSOutput:
|
||||
"""
|
||||
Performs a forward pass handling training, generation, or single-step inference.
|
||||
@@ -1352,13 +1364,17 @@ class RVQEARTTSModel(nn.Module):
|
||||
subword_ids = torch.cat([subword_ids] * 2, 0)
|
||||
if subword_mask is not None:
|
||||
subword_mask = torch.cat([subword_mask] * 2, 0)
|
||||
if subword_embeds is not None:
|
||||
subword_embeds = torch.cat([subword_embeds] * 2, 0)
|
||||
if asr_speech_tokens_emb is not None:
|
||||
asr_speech_tokens_emb = torch.cat([asr_speech_tokens_emb] * 2, 0)
|
||||
|
||||
uncond_dec_flag = torch.cat([uncond_dec_flag, torch.ones_like(uncond_dec_flag)], 0)
|
||||
|
||||
# Prepare conditioning
|
||||
cond = self._prepare_conditioning(context_hidden_state, subword_ids, subword_mask, uncond_dec_flag)
|
||||
cond = self._prepare_conditioning(
|
||||
context_hidden_state, subword_ids, subword_mask, uncond_dec_flag, subword_embeds=subword_embeds
|
||||
)
|
||||
|
||||
if asr_speech_tokens_emb is not None:
|
||||
cond = cond + asr_speech_tokens_emb
|
||||
@@ -1368,13 +1384,20 @@ class RVQEARTTSModel(nn.Module):
|
||||
else:
|
||||
inputs_embeds = code_embeds + cond
|
||||
|
||||
# Main backbone pass
|
||||
# Main backbone pass. cache_position is threaded only when provided so
|
||||
# backbones without the parameter keep working (it is required for
|
||||
# StaticCache-based decoding, where the write index cannot be derived
|
||||
# from the cache contents).
|
||||
backbone_kwargs: dict[str, Any] = {}
|
||||
if cache_position is not None:
|
||||
backbone_kwargs["cache_position"] = cache_position
|
||||
backbone_outputs = self.backbone(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=attention_mask,
|
||||
position_ids=position_ids,
|
||||
past_key_values=past_key_values,
|
||||
use_cache=use_cache,
|
||||
**backbone_kwargs,
|
||||
)
|
||||
hidden_states = backbone_outputs.last_hidden_state
|
||||
|
||||
@@ -1564,7 +1587,9 @@ class RVQEARTTSModel(nn.Module):
|
||||
else:
|
||||
eos_flag = lm_logits.argmax(-1) == 1
|
||||
|
||||
if torch.all(eos_flag) and not ignore_eos_flag_stop:
|
||||
# Check the cheap Python flag first: evaluating torch.all() forces a
|
||||
# host sync per frame even when the result is ignored.
|
||||
if not ignore_eos_flag_stop and torch.all(eos_flag):
|
||||
return None, lm_logits, eos_flag
|
||||
else:
|
||||
lm_logits = None
|
||||
@@ -1573,17 +1598,18 @@ class RVQEARTTSModel(nn.Module):
|
||||
# Initialize the full code tensor
|
||||
code = torch.zeros((b, t, d), dtype=torch.long, device=device) + self.config.codebook_size
|
||||
|
||||
# 3. Set up the iterative denoising schedule for the continuous part
|
||||
rates = torch.linspace(0.0, 1.0, num_iter + 1, device=device)[:-1].unsqueeze(-1)
|
||||
masking_rates = get_masking_rate(rates, exponent=exponent)
|
||||
num_maskings = torch.ceil(masking_rates * self.config.num_quantizers).long()
|
||||
|
||||
ks = num_maskings - F.pad(num_maskings[1:], [0, 0, 0, 1])
|
||||
# 3. Set up the iterative denoising schedule for the continuous part.
|
||||
# The schedule is a pure function of (num_iter, exponent,
|
||||
# num_quantizers) — compute it once as Python ints and cache it, so the
|
||||
# per-frame loop below runs without any GPU->CPU sync (`.item()` /
|
||||
# `torch.all(k == 0)` on a device tensor would stall the dispatch
|
||||
# pipeline several times per 80 ms frame).
|
||||
ks = self._unmask_schedule(num_iter, float(exponent))
|
||||
|
||||
# 4. Iteratively unmask the continuous part of the code
|
||||
cnt = 0
|
||||
for i, k in enumerate(ks):
|
||||
if torch.all(k == 0):
|
||||
if k == 0:
|
||||
continue
|
||||
|
||||
# Prepare input for the MoG head
|
||||
@@ -1593,7 +1619,7 @@ class RVQEARTTSModel(nn.Module):
|
||||
|
||||
mog_input_embeds = self.embed_code(self.depthsum_embedding(code).to(self.embed_code.weight.dtype))
|
||||
if self.config.random_target_masking:
|
||||
mog_input_embeds += self.embed_target_mask(cnt + k - 1)
|
||||
mog_input_embeds += self.embed_target_mask(torch.tensor(cnt + k - 1, device=device))
|
||||
if guidance_scale_i > 0.0:
|
||||
mog_input_embeds = torch.cat(
|
||||
[mog_input_embeds + hidden_states, mog_input_embeds + uncond_hidden_states], 0
|
||||
@@ -1607,10 +1633,26 @@ class RVQEARTTSModel(nn.Module):
|
||||
top_p_or_k=top_p_or_k_i,
|
||||
)
|
||||
z = mog_mu + torch.exp(mog_logs) * torch.randn_like(mog_mu) * noise_scale_i
|
||||
code = depthsum_encoding_step(self.rvq_embs, z, code, cnt, k[0].item())
|
||||
cnt += k[0].item()
|
||||
code = depthsum_encoding_step(self.rvq_embs, z, code, cnt, k)
|
||||
cnt += k
|
||||
return code, lm_logits, eos_flag
|
||||
|
||||
def _unmask_schedule(self, num_iter: int, exponent: float) -> list[int]:
|
||||
"""Per-iteration unmask counts as Python ints (cached; sync-free loop)."""
|
||||
cache = getattr(self, "_unmask_schedule_cache", None)
|
||||
if cache is None:
|
||||
cache = {}
|
||||
self._unmask_schedule_cache = cache
|
||||
key = (int(num_iter), float(exponent), int(self.config.num_quantizers))
|
||||
ks = cache.get(key)
|
||||
if ks is None:
|
||||
rates = torch.linspace(0.0, 1.0, num_iter + 1)[:-1].unsqueeze(-1)
|
||||
masking_rates = get_masking_rate(rates, exponent=exponent)
|
||||
num_maskings = torch.ceil(masking_rates * self.config.num_quantizers).long()
|
||||
ks = (num_maskings - F.pad(num_maskings[1:], [0, 0, 0, 1])).reshape(-1).tolist()
|
||||
cache[key] = ks
|
||||
return ks
|
||||
|
||||
def load_state_dict(self, state_dict, strict: bool = True):
|
||||
try:
|
||||
super().load_state_dict(state_dict, strict=strict)
|
||||
|
||||
@@ -82,9 +82,22 @@ class NemotronVoiceChatCode2Wav(nn.Module):
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
# Incremental streaming decode (hf_overrides.use_incremental_codec_cache):
|
||||
# keep a CausalConv1dCache per request so each async chunk decodes ONLY
|
||||
# its new frames — O(T) codec work — instead of re-decoding the whole
|
||||
# cumulative prefix per chunk (O(T^2)). This is the codec-cache path
|
||||
# NVIDIA's own serving stack runs (S2S_USE_CODEC_CACHE); the default
|
||||
# stays the prefix re-decode, whose concatenation is bit-equal to the
|
||||
# one-shot decode of the final stack.
|
||||
self._use_codec_cache = bool(getattr(self.config, "use_incremental_codec_cache", False))
|
||||
self.requires_request_ids = self._use_codec_cache
|
||||
# Shared per-request causal-conv caches; ALSO used by duplex serving,
|
||||
# whose per-segment payloads mark themselves with meta.codec_streaming
|
||||
# + meta.request_id (independent of the hf_override).
|
||||
self._stream_caches: dict[str, Any] = {}
|
||||
|
||||
# Constructed in load_weights (owns the tts_model.audio_codec.* subtree).
|
||||
self.audio_codec: nn.Module | None = None
|
||||
self._duplex_codec_caches: dict[str, object] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Runner-facing placeholder hooks.
|
||||
@@ -130,6 +143,7 @@ class NemotronVoiceChatCode2Wav(nn.Module):
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
request_ids: list[str] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
sr_tensor = torch.tensor(self._sample_rate, dtype=torch.int32)
|
||||
@@ -189,25 +203,40 @@ class NemotronVoiceChatCode2Wav(nn.Module):
|
||||
left_context_frames = max(int(runtime_info["meta.left_context_size"]), 0)
|
||||
if left_context_frames >= frames:
|
||||
continue # terminal empty chunk: all frames already emitted
|
||||
lens = torch.tensor([frames], device=device, dtype=torch.long)
|
||||
is_stream_chunk = (isinstance(meta, Mapping) and "left_context_size" in meta) or (
|
||||
isinstance(runtime_info, Mapping) and "meta.left_context_size" in runtime_info
|
||||
)
|
||||
request_id = str(request_ids[i]) if request_ids is not None and i < len(request_ids) else None
|
||||
codec_cache = None
|
||||
decode_codes = codes
|
||||
if codec_streaming:
|
||||
# Duplex serving: the per-segment payload carries ONLY the new
|
||||
# frames; continuity lives in the per-request causal-conv cache
|
||||
# keyed by the talker-forwarded meta.request_id.
|
||||
if not isinstance(cache_request_id, str) or not cache_request_id:
|
||||
raise ValueError("Nemotron VoiceChat incremental codec input lacks meta.request_id")
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.nemo_vendored.ear_tts_vae_codec import (
|
||||
CausalConv1dCache,
|
||||
)
|
||||
|
||||
codec_cache = self._duplex_codec_caches.setdefault(cache_request_id, CausalConv1dCache())
|
||||
codec_cache = self._stream_caches.setdefault(cache_request_id, CausalConv1dCache())
|
||||
elif self._use_codec_cache and is_stream_chunk and request_id is not None:
|
||||
# Async-chunk streaming: cumulative payload; decode ONLY the
|
||||
# frames past the already-emitted prefix, with the causal-conv
|
||||
# left context carried in the per-request cache.
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.nemo_vendored.ear_tts_vae_codec import (
|
||||
CausalConv1dCache,
|
||||
)
|
||||
|
||||
codec_cache = self._stream_caches.setdefault(request_id, CausalConv1dCache())
|
||||
decode_codes = codes[left_context_frames:]
|
||||
lens = torch.tensor([decode_codes.shape[0]], device=device, dtype=torch.long)
|
||||
# NeMo decodes the codec strictly in fp32.
|
||||
with torch.autocast(device_type=device.type, enabled=False):
|
||||
wav, wav_len = self.audio_codec.decode(
|
||||
codes.unsqueeze(0),
|
||||
lens,
|
||||
cache=codec_cache,
|
||||
)
|
||||
wav, wav_len = self.audio_codec.decode(decode_codes.unsqueeze(0), lens, cache=codec_cache)
|
||||
wav = wav.reshape(-1)[: int(wav_len.reshape(-1)[0])]
|
||||
if left_context_frames > 0 and not codec_streaming:
|
||||
if left_context_frames > 0 and codec_cache is None:
|
||||
# Prefix re-decode path: slice off the samples already emitted.
|
||||
wav = wav[left_context_frames * self._wav_per_frame :]
|
||||
audios[i] = wav.detach().to(dtype=torch.float32).cpu()
|
||||
logger.debug(
|
||||
@@ -222,15 +251,15 @@ class NemotronVoiceChatCode2Wav(nn.Module):
|
||||
multimodal_outputs={"model_outputs": audios, "sr": srs},
|
||||
)
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
for request_id in finished_req_ids:
|
||||
self._duplex_codec_caches.pop(str(request_id), None)
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
raise TypeError(f"NemotronVoiceChatCode2Wav expected OmniOutput, got {type(model_outputs)}")
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
for request_id in finished_req_ids:
|
||||
self._stream_caches.pop(str(request_id), None)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weights.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -113,13 +113,71 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
self.has_postprocess = False
|
||||
self.requires_full_prefix_cached_hidden_states = False
|
||||
|
||||
# Stateful per-request execution is only validated at batch size 1
|
||||
# (the shipped offline scope); fail fast on silent multi-request use.
|
||||
# The vendored (eager / captured-step) paths share single-session
|
||||
# buffers (StaticCache, capture-time addresses) and stay batch=1; the
|
||||
# NATIVE path keeps all per-request state in _sessions (paged KV,
|
||||
# per-request MoG on each request's own hidden slice) and supports
|
||||
# multi-session batching (validated at max_num_seqs<=4). The native
|
||||
# flag is read below, so peek at it here for the guard.
|
||||
max_num_seqs = int(getattr(vllm_config.scheduler_config, "max_num_seqs", 1))
|
||||
if max_num_seqs != 1:
|
||||
if max_num_seqs != 1 and not bool(getattr(self.config, "use_native_talker", False)):
|
||||
raise NotImplementedError(
|
||||
f"NemotronVoiceChat talker supports max_num_seqs=1 only (got {max_num_seqs}); "
|
||||
"the vendored EAR-TTS session state is not batch-safe."
|
||||
f"NemotronVoiceChat talker supports max_num_seqs=1 only (got {max_num_seqs}) on the "
|
||||
"vendored paths; the EAR-TTS session buffers are not batch-safe. Use "
|
||||
"hf_overrides.use_native_talker for multi-session batching."
|
||||
)
|
||||
|
||||
# CUDA-graph fast path (hf_overrides.use_talker_cuda_graphs). Off by
|
||||
# default: the eager path is the NeMo bit-parity acceptance path, while
|
||||
# the captured step replays the same math ~3x faster with a
|
||||
# graph-managed RNG (audio equivalent but not bit-identical).
|
||||
self._use_cuda_graphs = bool(getattr(self.config, "use_talker_cuda_graphs", False))
|
||||
self._graph_max_cache_len = int(getattr(self.config, "talker_graph_max_cache_len", 4096))
|
||||
self._step_graph: Any | None = None
|
||||
|
||||
# Native-vLLM path (hf_overrides.use_native_talker, experimental): the
|
||||
# Gemma3 backbone runs as a real vLLM model on PagedAttention (KV and
|
||||
# per-step attention cost scale with the ACTUAL session length; vLLM's
|
||||
# own decode CUDA graphs apply), mirroring NVIDIA's deployment. The
|
||||
# per-frame MoG sampling runs on the backbone's hidden state in
|
||||
# make_omni_output. Classifier-free guidance is NOT applied yet on
|
||||
# this path (single conditional stream; the paired-request CFG design
|
||||
# is a documented follow-up), so audio quality trails the vendored
|
||||
# CFG paths — use for development only.
|
||||
self._use_native_backbone = bool(getattr(self.config, "use_native_talker", False))
|
||||
# Classifier-free guidance on the native path
|
||||
# (hf_overrides.native_talker_guidance): the conditional stream stays
|
||||
# on vLLM paged KV; the unconditional stream mirrors it on the vendored
|
||||
# HF backbone (see talker_native.UncondStream) and the pair is blended
|
||||
# per NeMo's generate_step math. Off by default (measured ASR-equivalent
|
||||
# at this checkpoint's guidance_scale=0.2, but kept available for
|
||||
# listening-test parity with the vendored paths).
|
||||
self._native_guidance = self._use_native_backbone and bool(
|
||||
getattr(self.config, "native_talker_guidance", False)
|
||||
)
|
||||
# Captured MoG sampling step (built lazily on the first decode frame;
|
||||
# falls back to eager sampling if capture fails).
|
||||
self._mog_graph: Any | None = None
|
||||
self._uncond_graph: Any | None = None
|
||||
self.backbone: nn.Module | None = None
|
||||
if self._use_native_backbone:
|
||||
if self._use_cuda_graphs:
|
||||
raise ValueError(
|
||||
"NemotronVoiceChat talker: use_native_talker and use_talker_cuda_graphs are mutually exclusive."
|
||||
)
|
||||
from vllm.model_executor.models.utils import init_vllm_registered_model, maybe_prefix
|
||||
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
synthesize_backbone_config,
|
||||
)
|
||||
|
||||
tts_config = dict(getattr(self.config, "tts_cfg", {}) or {}).get("tts_config") or {}
|
||||
backbone_cfg = synthesize_backbone_config(dict(tts_config))
|
||||
self.backbone = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=backbone_cfg,
|
||||
prefix=maybe_prefix(prefix, "backbone"),
|
||||
architectures=["Gemma3ForCausalLM"],
|
||||
)
|
||||
|
||||
# Vendored DuplexEARTTS; constructed in load_weights.
|
||||
@@ -141,6 +199,16 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**_: Any,
|
||||
) -> torch.Tensor:
|
||||
if self._use_native_backbone:
|
||||
# Real Gemma3 forward on PagedAttention; preprocess injected the
|
||||
# fused embeddings, make_omni_output turns the hidden state into
|
||||
# codes (MoG) and the CONTINUE/STOP flag.
|
||||
return self.backbone(
|
||||
input_ids=None,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
# The per-frame compute happens in preprocess; the "hidden state" only
|
||||
# transports the stop flag (column 0) to compute_logits.
|
||||
if inputs_embeds is not None:
|
||||
@@ -161,6 +229,8 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
if self._use_native_backbone:
|
||||
return self._native_make_omni_output(model_outputs, **kwargs)
|
||||
hidden = model_outputs
|
||||
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information") or []
|
||||
codes_list: list[torch.Tensor] = []
|
||||
@@ -275,6 +345,36 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
)
|
||||
return timeline
|
||||
|
||||
@staticmethod
|
||||
def _expected_total_tokens(info: dict[str, Any]) -> int | None:
|
||||
"""Producer-announced FINAL timeline length (meta.expected_total_tokens)."""
|
||||
meta = info.get("meta")
|
||||
value = meta.get("expected_total_tokens") if isinstance(meta, dict) else None
|
||||
if value is None:
|
||||
value = info.get("meta.expected_total_tokens")
|
||||
return int(value) if value is not None else None
|
||||
|
||||
def _graph_session_usable(self, timeline: torch.Tensor, expected_len: int | None) -> bool:
|
||||
"""Whether the CUDA-graph fast path can serve this session.
|
||||
|
||||
``expected_len`` is the known FINAL timeline length (sync mode, or a
|
||||
frame-locked async producer's meta.expected_total_tokens); the graph
|
||||
picks the smallest fitting cache bucket from it, since per-step
|
||||
attention cost scales with the bucket size. ``None`` = unbounded
|
||||
(duplex) -> largest bucket.
|
||||
"""
|
||||
if self._step_graph is None:
|
||||
return False
|
||||
needed = max(int(timeline.numel()), expected_len or 0)
|
||||
if not self._step_graph.fits(needed):
|
||||
logger.warning(
|
||||
"NemotronVoiceChat talker: timeline of %d positions exceeds the CUDA-graph "
|
||||
"capacity; using the eager per-frame step for this request.",
|
||||
needed,
|
||||
)
|
||||
return False
|
||||
return self._step_graph.ensure_captured(expected_len)
|
||||
|
||||
def _init_session(self, request_id: str, info: dict[str, Any], device: torch.device) -> dict[str, Any]:
|
||||
assert self.tts is not None
|
||||
timeline = self._timeline(info).to(device)
|
||||
@@ -283,15 +383,6 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
f"NemotronVoiceChat talker timeline for request {request_id} has "
|
||||
f"{timeline.numel()} positions; need at least 2 (prompt + one frame)."
|
||||
)
|
||||
# Warmup: pre-baked speaker latent + init prompt forward (NeMo
|
||||
# offline_inference lines around get_init_inputs/first forward).
|
||||
self.tts.set_init_inputs(speaker_name=self._speaker_name)
|
||||
init_inputs = self.tts.get_init_inputs(B=1)
|
||||
guidance_enabled = self._guidance_enabled()
|
||||
generation_config = self.tts._get_generation_config(guidance_enabled)
|
||||
init_inputs.update({"use_cache": True, "past_key_values": None, "guidance_enabled": guidance_enabled})
|
||||
with torch.inference_mode():
|
||||
outputs = self.tts.tts_model(**init_inputs)
|
||||
prompt_len = self._logical_prompt_len(info)
|
||||
if prompt_len is None:
|
||||
raise ValueError(
|
||||
@@ -304,17 +395,17 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
codec_streaming = scalar_bool(meta.get("codec_streaming")) if isinstance(meta, dict) else False
|
||||
codec_streaming = codec_streaming or scalar_bool(info.get("meta.codec_streaming"))
|
||||
codec_request_id = meta.get("request_id") if isinstance(meta, dict) else None
|
||||
# Warmup: pre-baked speaker latent (NeMo offline_inference lines around
|
||||
# get_init_inputs); the init prompt forward runs in the eager branch
|
||||
# below, or inside the CUDA-graph session's own StaticCache prefill.
|
||||
self.tts.set_init_inputs(speaker_name=self._speaker_name)
|
||||
guidance_enabled = self._guidance_enabled()
|
||||
session = {
|
||||
"timeline": timeline,
|
||||
"prompt_len": prompt_len,
|
||||
"step": 1, # NeMo's loop starts at t=1
|
||||
"past_key_values": outputs.past_key_values,
|
||||
"code": init_inputs["code"][:, -1:],
|
||||
"first_context_subword_id": init_inputs["subword_ids"][:, -1].unsqueeze(-1),
|
||||
"current_subword_mask": torch.ones(1, 1, device=device, dtype=torch.bool),
|
||||
"tts_initialized": False,
|
||||
"guidance_enabled": guidance_enabled,
|
||||
"generation_config": generation_config,
|
||||
"codes_rows": [],
|
||||
"codec_streaming": codec_streaming,
|
||||
"codec_request_id": codec_request_id,
|
||||
@@ -324,14 +415,434 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
"sync_mode": info.get("nvc_text_timeline") is not None,
|
||||
"upstream_finished": self._upstream_finished(info) or info.get("nvc_text_timeline") is not None,
|
||||
}
|
||||
if session["sync_mode"]:
|
||||
expected_len: int | None = int(timeline.numel())
|
||||
else:
|
||||
expected_len = self._expected_total_tokens(info)
|
||||
if self._graph_session_usable(timeline, expected_len):
|
||||
self._step_graph.start_session(timeline, expected_len=expected_len)
|
||||
session["use_graph"] = True
|
||||
else:
|
||||
init_inputs = self.tts.get_init_inputs(B=1)
|
||||
generation_config = self.tts._get_generation_config(guidance_enabled)
|
||||
init_inputs.update({"use_cache": True, "past_key_values": None, "guidance_enabled": guidance_enabled})
|
||||
with torch.inference_mode():
|
||||
outputs = self.tts.tts_model(**init_inputs)
|
||||
session.update(
|
||||
{
|
||||
"use_graph": False,
|
||||
"past_key_values": outputs.past_key_values,
|
||||
"code": init_inputs["code"][:, -1:],
|
||||
"first_context_subword_id": init_inputs["subword_ids"][:, -1].unsqueeze(-1),
|
||||
"tts_initialized": False,
|
||||
"generation_config": generation_config,
|
||||
}
|
||||
)
|
||||
self._sessions[request_id] = session
|
||||
return session
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Native-vLLM path (experimental; see __init__).
|
||||
# ------------------------------------------------------------------
|
||||
def _encode_cas_rows(self, ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Batched CharAwareSubwordEncoder rows for timeline tokens ([n, H])."""
|
||||
model = self.tts.tts_model
|
||||
mask = torch.ones(1, ids.numel(), dtype=torch.bool, device=ids.device)
|
||||
with torch.inference_mode():
|
||||
return model.embed_subword(ids.unsqueeze(0), mask)[0]
|
||||
|
||||
def _init_native_session(self, request_id: str, info: dict[str, Any], device: torch.device) -> dict[str, Any]:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
build_prefill_embeds,
|
||||
)
|
||||
|
||||
timeline = self._timeline(info).to(device)
|
||||
prompt_len = self._logical_prompt_len(info)
|
||||
if prompt_len is None:
|
||||
raise ValueError(
|
||||
"NemotronVoiceChat talker request is missing its logical prompt length "
|
||||
"('nvc_logical_prompt_len' or async meta.num_processed_tokens); the code2wav "
|
||||
"producer cannot trim the prompt-region codes without it."
|
||||
)
|
||||
meta = info.get("meta")
|
||||
codec_streaming = scalar_bool(meta.get("codec_streaming")) if isinstance(meta, dict) else False
|
||||
codec_streaming = codec_streaming or scalar_bool(info.get("meta.codec_streaming"))
|
||||
codec_request_id = meta.get("request_id") if isinstance(meta, dict) else None
|
||||
sync_mode = info.get("nvc_text_timeline") is not None
|
||||
if not sync_mode and not codec_streaming:
|
||||
raise NotImplementedError(
|
||||
"NemotronVoiceChat native talker supports the sync pipeline and duplex "
|
||||
"(resumable) serving; the offline async-chunk pipeline would free-run past "
|
||||
"the received timeline (non-resumable requests cannot park per segment) — "
|
||||
"use the captured-step fast_streaming yaml for offline streaming."
|
||||
)
|
||||
self.tts.set_init_inputs(speaker_name=self._speaker_name)
|
||||
init_inputs = self.tts.get_init_inputs(B=1)
|
||||
init_len = int(init_inputs["code"].shape[1])
|
||||
model = self.tts.tts_model
|
||||
with torch.inference_mode():
|
||||
prefill = build_prefill_embeds(
|
||||
model,
|
||||
code=init_inputs["code"],
|
||||
subword_ids=init_inputs["subword_ids"],
|
||||
subword_mask=init_inputs["subword_mask"],
|
||||
audio_mask=init_inputs["audio_mask"],
|
||||
audio_prompt_latent=init_inputs.get("audio_prompt_latent"),
|
||||
uncond=False,
|
||||
)[0] # [init_len, H]
|
||||
uncond_stream = None
|
||||
if self._native_guidance:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
UncondStepGraph,
|
||||
UncondStream,
|
||||
)
|
||||
|
||||
with torch.inference_mode():
|
||||
uncond_prefill = build_prefill_embeds(
|
||||
model,
|
||||
code=init_inputs["code"],
|
||||
subword_ids=init_inputs["subword_ids"],
|
||||
subword_mask=init_inputs["subword_mask"],
|
||||
audio_mask=init_inputs["audio_mask"],
|
||||
audio_prompt_latent=init_inputs.get("audio_prompt_latent"),
|
||||
uncond=True,
|
||||
).to(self._dtype)
|
||||
# Captured StaticCache step when the graph is free (one session at
|
||||
# a time; sync sessions pick the smallest fitting bucket, duplex
|
||||
# takes the largest). Concurrent sessions step eagerly.
|
||||
if self._uncond_graph is None:
|
||||
self._uncond_graph = UncondStepGraph(
|
||||
model,
|
||||
hidden_size=int(uncond_prefill.shape[-1]),
|
||||
dtype=self._dtype,
|
||||
device=timeline.device,
|
||||
max_cache_len=self._graph_max_cache_len,
|
||||
)
|
||||
total_positions = (init_len + int(timeline.numel()) + 8) if sync_mode else None
|
||||
if self._uncond_graph.start_session(request_id, uncond_prefill, total_positions):
|
||||
uncond_stream = self._uncond_graph
|
||||
else:
|
||||
uncond_stream = UncondStream(model)
|
||||
uncond_stream.prefill(uncond_prefill)
|
||||
session = {
|
||||
"use_native": True,
|
||||
"timeline": timeline,
|
||||
"cas_table": self._encode_cas_rows(timeline),
|
||||
"prompt_len": int(prompt_len),
|
||||
"init_len": init_len,
|
||||
"prefill_embeds": prefill,
|
||||
"step": 1, # NeMo's loop starts at t=1
|
||||
"code": init_inputs["code"][:, -1:],
|
||||
# Kept for KV-recompute replay: step t=1's feedback code, before
|
||||
# session["code"] starts tracking the latest sampled frame.
|
||||
"initial_code": init_inputs["code"][:, -1:],
|
||||
# CFG: native_talker_guidance mirrors the stream on the vendored
|
||||
# HF backbone (UncondStream); otherwise a single conditional
|
||||
# stream (measured ASR-equivalent at guidance_scale=0.2; see yaml).
|
||||
"generation_config": self.tts._get_generation_config(self._native_guidance),
|
||||
"uncond_stream": uncond_stream,
|
||||
"uncond_hidden": None,
|
||||
# NeMo's inference_force_speech_silence_on_eos (default on): EOS
|
||||
# text frames feed the silence codec frame back instead of the
|
||||
# sampled codes.
|
||||
"silence_codes": (
|
||||
self.tts.codec_silence_tokens.reshape(1, 1, -1).to(device=timeline.device, dtype=torch.long)
|
||||
if bool(self.tts.cfg.get("inference_force_speech_silence_on_eos", True))
|
||||
else None
|
||||
),
|
||||
"text_eos_id": int(self.tts.text_eos_id),
|
||||
"codes_rows": [],
|
||||
"pending_step": None,
|
||||
"codec_streaming": codec_streaming,
|
||||
"codec_request_id": codec_request_id,
|
||||
"sync_mode": info.get("nvc_text_timeline") is not None,
|
||||
"upstream_finished": self._upstream_finished(info) or info.get("nvc_text_timeline") is not None,
|
||||
}
|
||||
self._sessions[request_id] = session
|
||||
return session
|
||||
|
||||
def _refresh_native_timeline(self, session: dict[str, Any], info: dict[str, Any], device: torch.device) -> None:
|
||||
"""Adopt a longer timeline from the latest chunk; CAS-encode new rows."""
|
||||
latest = self._try_timeline(info)
|
||||
old_len = int(session["timeline"].numel())
|
||||
if latest is not None and latest.numel() > old_len:
|
||||
latest = latest.to(device)
|
||||
new_rows = self._encode_cas_rows(latest[old_len:].to(torch.long))
|
||||
session["timeline"] = latest
|
||||
session["cas_table"] = torch.cat([session["cas_table"], new_rows.to(session["cas_table"].dtype)], dim=0)
|
||||
if self._upstream_finished(info):
|
||||
session["upstream_finished"] = True
|
||||
|
||||
def _native_preprocess(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
info: dict[str, Any],
|
||||
device: torch.device,
|
||||
span: int,
|
||||
request_id: str,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
build_decode_embeds,
|
||||
)
|
||||
|
||||
# NOTE: resumable (duplex) segments report _omni_is_prefill=True on
|
||||
# EVERY wake (each appended placeholder extends the prompt), so the
|
||||
# branch is decided by the engine POSITION, never by the prefill flag:
|
||||
# positions [0, init_len) carry the speaker-prompt prefill embeddings,
|
||||
# every later position consumes exactly one timeline step.
|
||||
session = self._sessions.get(request_id)
|
||||
if session is None:
|
||||
session = self._init_native_session(request_id, info, device)
|
||||
else:
|
||||
self._refresh_native_timeline(session, info, device)
|
||||
offset = max(0, int(info.get("_omni_num_computed_tokens", 0) or 0))
|
||||
init_len = int(session["init_len"])
|
||||
step = int(session["step"])
|
||||
total = int(session["timeline"].numel())
|
||||
prefill_head: torch.Tensor | None = None
|
||||
if offset < init_len:
|
||||
# Speaker-prompt prefill region (possibly chunked across steps).
|
||||
if offset + span <= init_len:
|
||||
prefill: torch.Tensor = session["prefill_embeds"]
|
||||
return input_ids, prefill[offset : offset + span].to(self._dtype), {}
|
||||
# A span crossing the boundary is only legal while a KV-cache
|
||||
# recomputation replays already-generated positions in bulk; a
|
||||
# LIVE first timeline step sharing a chunk with the prefill means
|
||||
# a timeline chunk outpaced the speaker prefill.
|
||||
if step <= 1:
|
||||
raise ValueError(
|
||||
f"NemotronVoiceChat native talker got a span [{offset}, {offset + span}) crossing "
|
||||
f"the speaker-prompt boundary ({init_len}); the stage prompt must be exactly "
|
||||
f"{init_len} placeholder tokens (set hf_overrides.talker_init_len / "
|
||||
f"async_chunk_prewarm_prompt_len to {init_len} in the deploy yaml). A timeline "
|
||||
"chunk extended this request before its speaker prefill finished — timeline "
|
||||
"positions are strictly sequential and cannot share a step with the prefill."
|
||||
)
|
||||
prefill_head = session["prefill_embeds"][offset:init_len]
|
||||
span -= init_len - offset
|
||||
offset = init_len
|
||||
|
||||
t_first = offset - init_len + 1
|
||||
t_last = t_first + span - 1
|
||||
if t_last > step:
|
||||
raise RuntimeError(
|
||||
f"NemotronVoiceChat native talker was scheduled timeline steps {t_first}..{t_last} but "
|
||||
f"the session is at step {step}; the per-frame code feedback is strictly sequential. "
|
||||
"The duplex producer drips one new timeline token per chunk — a position past the "
|
||||
"session step means chunks outpaced engine steps (raise the frame budget or "
|
||||
"investigate the engine stall)."
|
||||
)
|
||||
if t_last >= total:
|
||||
raise RuntimeError(f"NemotronVoiceChat native talker stepped past the timeline: step {t_last} of {total}.")
|
||||
replay_last = min(t_last, step - 1)
|
||||
if replay_last >= t_first and len(session["codes_rows"]) < replay_last - 1:
|
||||
raise RuntimeError(
|
||||
f"NemotronVoiceChat native talker cannot replay steps {t_first}..{replay_last}: only "
|
||||
f"{len(session['codes_rows'])} stored code rows."
|
||||
)
|
||||
replay_rows: list[torch.Tensor] = []
|
||||
if replay_last >= t_first:
|
||||
# KV-cache recomputation replays already-generated engine
|
||||
# positions. Rebuild their decode embeds deterministically from
|
||||
# the stored code history: no resampling (pending_step is only set
|
||||
# for a live step) and the manual unconditional stream is not
|
||||
# advanced (it was never rewound).
|
||||
if not session.get("replaying"):
|
||||
session["replaying"] = True
|
||||
logger.warning(
|
||||
"NemotronVoiceChat native talker replaying timeline steps %d..%d after KV recompute.",
|
||||
t_first,
|
||||
replay_last,
|
||||
)
|
||||
replay_rows = [self._native_replay_embeds(session, rt) for rt in range(t_first, replay_last + 1)]
|
||||
if t_last < step:
|
||||
# Pure replay chunk: no live position this step.
|
||||
embeds = torch.cat(replay_rows, dim=0)
|
||||
if prefill_head is not None:
|
||||
embeds = torch.cat([prefill_head.to(embeds.dtype), embeds], dim=0)
|
||||
return input_ids, embeds.to(self._dtype), {}
|
||||
session.pop("replaying", None)
|
||||
t = step
|
||||
model = self.tts.tts_model
|
||||
with torch.inference_mode():
|
||||
prev_codes = session["code"]
|
||||
if session["silence_codes"] is not None:
|
||||
# NeMo's inference_force_speech_silence_on_eos: on an EOS text
|
||||
# frame the code feedback becomes the silence frame, so the
|
||||
# turn ends in clean silence instead of a trailing artifact.
|
||||
# Tensor-level select — no host sync on the timeline token.
|
||||
is_eos = session["timeline"][t : t + 1].reshape(1, 1, 1) == session["text_eos_id"]
|
||||
prev_codes = torch.where(is_eos, session["silence_codes"].expand_as(prev_codes), prev_codes)
|
||||
embeds = build_decode_embeds(
|
||||
model,
|
||||
prev_codes=prev_codes,
|
||||
subword_embed=session["cas_table"][t : t + 1].unsqueeze(0),
|
||||
uncond=False,
|
||||
)[0] # [1, H]
|
||||
if session["uncond_stream"] is not None:
|
||||
# Advance the unconditional stream in lockstep: same prev codes,
|
||||
# null conditioning. Its hidden row pairs with this step's
|
||||
# conditional row in make_omni_output.
|
||||
uncond_embed = build_decode_embeds(
|
||||
model,
|
||||
prev_codes=prev_codes,
|
||||
subword_embed=session["cas_table"][t : t + 1].unsqueeze(0),
|
||||
uncond=True,
|
||||
).to(self._dtype)
|
||||
session["uncond_hidden"] = session["uncond_stream"].step(uncond_embed)
|
||||
session["pending_step"] = t
|
||||
embeds = embeds.to(self._dtype)
|
||||
if replay_rows:
|
||||
embeds = torch.cat([*replay_rows, embeds], dim=0)
|
||||
if prefill_head is not None:
|
||||
embeds = torch.cat([prefill_head.to(embeds.dtype), embeds], dim=0)
|
||||
return input_ids, embeds, {}
|
||||
|
||||
def _native_replay_embeds(self, session: dict[str, Any], t: int) -> torch.Tensor:
|
||||
"""Decode embeds for an already-generated step ``t`` (KV recompute).
|
||||
|
||||
Reconstructs the exact feedback the original step saw: the codes
|
||||
sampled at step ``t-1`` (or the speaker-prompt code for ``t == 1``)
|
||||
with the same silence-on-EOS substitution.
|
||||
"""
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
build_decode_embeds,
|
||||
)
|
||||
|
||||
model = self.tts.tts_model
|
||||
with torch.inference_mode():
|
||||
template: torch.Tensor = session["code"]
|
||||
if t == 1:
|
||||
prev_codes = session["initial_code"]
|
||||
else:
|
||||
prev_codes = session["codes_rows"][t - 2].reshape(template.shape).to(dtype=template.dtype)
|
||||
if session["silence_codes"] is not None:
|
||||
is_eos = session["timeline"][t : t + 1].reshape(1, 1, 1) == session["text_eos_id"]
|
||||
prev_codes = torch.where(is_eos, session["silence_codes"].expand_as(prev_codes), prev_codes)
|
||||
embeds = build_decode_embeds(
|
||||
model,
|
||||
prev_codes=prev_codes,
|
||||
subword_embed=session["cas_table"][t : t + 1].unsqueeze(0),
|
||||
uncond=False,
|
||||
)[0]
|
||||
return embeds.to(self._dtype)
|
||||
|
||||
def _native_make_omni_output(self, hidden: torch.Tensor, **kwargs: Any) -> OmniOutput:
|
||||
infos = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information") or []
|
||||
spans = kwargs.get("request_token_spans")
|
||||
flag = torch.zeros(hidden.shape[0], 2, dtype=torch.float32, device=hidden.device)
|
||||
# Batch-aligned per-request lists: the generic mm splitter
|
||||
# (to_payload_element) hands list element ``idx`` to request ``idx``
|
||||
# but clones a shared tensor to EVERY request, so a single shared
|
||||
# payload would leak the last request's codes into the others'
|
||||
# streams. Requests with nothing to ship this step keep placeholder
|
||||
# entries.
|
||||
empty_codes = torch.empty(0, dtype=torch.long)
|
||||
audio_list: list[torch.Tensor] = [empty_codes for _ in infos]
|
||||
prompt_len_list: list[torch.Tensor] = [torch.zeros(1, dtype=torch.long) for _ in infos]
|
||||
streaming_list: list[torch.Tensor] = [torch.zeros(1, dtype=torch.bool) for _ in infos]
|
||||
emitted = False
|
||||
|
||||
def _emit(index: int, session_state: dict[str, Any]) -> None:
|
||||
nonlocal emitted
|
||||
audio_list[index] = torch.cat(session_state["codes_rows"], dim=0)
|
||||
# 1-D tensors on purpose: the wire-payload builder indexes
|
||||
# element.shape[0] (see the eager make_omni_output).
|
||||
prompt_len_list[index] = torch.tensor([session_state["prompt_len"]], dtype=torch.long)
|
||||
streaming_list[index] = torch.tensor([bool(session_state["codec_streaming"])], dtype=torch.bool)
|
||||
emitted = True
|
||||
|
||||
for i, info in enumerate(infos):
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
request_id = info.get("request_id")
|
||||
session = self._sessions.get(str(request_id)) if request_id is not None else None
|
||||
if not session or not session.get("use_native"):
|
||||
continue
|
||||
pending = session.get("pending_step")
|
||||
if pending is None:
|
||||
# Prefill forward: no new codes this step, but keep the
|
||||
# request's cumulative history in its own slot so the
|
||||
# splitter never falls back to a shared tensor.
|
||||
if session.get("codes_rows"):
|
||||
_emit(i, session)
|
||||
continue
|
||||
session["pending_step"] = None
|
||||
if spans is None:
|
||||
raise RuntimeError("NemotronVoiceChat native talker needs request_token_spans from the runner.")
|
||||
start, end = spans[i]
|
||||
model = self.tts.tts_model
|
||||
if session["uncond_stream"] is not None:
|
||||
# CFG: pair the conditional row with this frame's uncond row
|
||||
# ([cond; uncond] order — generate_step chunks in that order)
|
||||
# and let generate_step apply NeMo's hidden-space blends
|
||||
# (lm_head EOS blend + the post-MLP blend inside MoGHead.infer).
|
||||
uncond_hidden = session["uncond_hidden"]
|
||||
if uncond_hidden is None:
|
||||
raise RuntimeError(
|
||||
"NemotronVoiceChat native talker CFG desynchronized: no unconditional "
|
||||
"hidden row for this frame (preprocess/make_omni_output mismatch)."
|
||||
)
|
||||
session["uncond_hidden"] = None
|
||||
pair = torch.cat(
|
||||
[hidden[end - 1 : end].reshape(1, 1, -1), uncond_hidden.reshape(1, 1, -1)],
|
||||
dim=0,
|
||||
).to(self._dtype)
|
||||
if self._mog_graph is None:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
MoGStepGraph,
|
||||
)
|
||||
|
||||
self._mog_graph = MoGStepGraph(model, session["generation_config"], batch=2)
|
||||
codes = self._mog_graph.run(pair)
|
||||
else:
|
||||
if self._mog_graph is None:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
MoGStepGraph,
|
||||
)
|
||||
|
||||
self._mog_graph = MoGStepGraph(model, session["generation_config"], batch=1)
|
||||
codes = self._mog_graph.run(hidden[end - 1 : end].reshape(1, 1, -1))
|
||||
session["code"] = codes
|
||||
session["codes_rows"].append(codes.reshape(1, -1).to(torch.long))
|
||||
session["step"] = int(pending) + 1
|
||||
# Duplex (codec_streaming) sessions stop per segment when the
|
||||
# received timeline is exhausted and resume with KV intact when
|
||||
# the next thinker frame arrives; sync sessions stop at the end.
|
||||
finished = int(session["step"]) >= int(session["timeline"].numel()) and self._segment_stops_at_exhaustion(
|
||||
session
|
||||
)
|
||||
if finished:
|
||||
flag[start:end, 0] = 1.0
|
||||
_emit(i, session)
|
||||
# Belt and braces: also mirror into the live per-request buffer so
|
||||
# the flush-time full-payload producer can read it from
|
||||
# additional_information as well as from the wire payload.
|
||||
info["codes"] = {"audio": audio_list[i]}
|
||||
info["meta"] = {
|
||||
"nvc_tts_step": int(session["step"]),
|
||||
"nvc_logical_prompt_len": int(session["prompt_len"]),
|
||||
"codec_streaming": bool(session["codec_streaming"]),
|
||||
"request_id": session["codec_request_id"],
|
||||
}
|
||||
outputs: dict[str, Any] = {}
|
||||
if emitted:
|
||||
outputs = {
|
||||
"codes": {"audio": audio_list},
|
||||
"meta": {
|
||||
"nvc_logical_prompt_len": prompt_len_list,
|
||||
"codec_streaming": streaming_list,
|
||||
},
|
||||
}
|
||||
return OmniOutput(text_hidden_states=flag, multimodal_outputs=outputs)
|
||||
|
||||
def _refresh_session_timeline(self, session: dict[str, Any], info: dict[str, Any], device: torch.device) -> None:
|
||||
"""Adopt a longer timeline from the latest async-chunk payload."""
|
||||
latest = self._try_timeline(info)
|
||||
if latest is not None and latest.numel() > int(session["timeline"].numel()):
|
||||
session["timeline"] = latest.to(device)
|
||||
if session.get("use_graph"):
|
||||
self._step_graph.extend_timeline(session["timeline"])
|
||||
if self._upstream_finished(info):
|
||||
session["upstream_finished"] = True
|
||||
|
||||
@@ -374,15 +885,23 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
session["code"] = code
|
||||
session["past_key_values"] = past_key_values
|
||||
session["step"] = t + 1
|
||||
# Native codec streaming uses one resumable scheduler segment per
|
||||
# received timeline suffix. The pre-existing offline async-chunk path,
|
||||
# however, must stay alive until the thinker sends its terminal marker;
|
||||
# stopping at an intermediate cumulative snapshot starves Stage 2.
|
||||
finished = (t + 1) >= int(session["timeline"].numel()) and (
|
||||
bool(session["codec_streaming"]) or bool(session["upstream_finished"])
|
||||
)
|
||||
finished = (t + 1) >= int(session["timeline"].numel()) and self._segment_stops_at_exhaustion(session)
|
||||
return code.reshape(1, -1).to(torch.long), finished
|
||||
|
||||
@staticmethod
|
||||
def _segment_stops_at_exhaustion(session: dict[str, Any]) -> bool:
|
||||
"""Whether exhausting the received timeline should emit the stop token.
|
||||
|
||||
Duplex serving submits RESUMABLE talker requests and marks them with
|
||||
``codec_streaming``: exhausting the timeline ends the current scheduler
|
||||
segment, and the parked request resumes (TTS state intact) when the
|
||||
next thinker frame arrives. Offline requests are NOT resumable — a
|
||||
stop token would end them permanently — so mid-stream exhaustion emits
|
||||
CONTINUE instead, and the stop waits for the upstream's final chunk
|
||||
(``meta.finished``; sync mode ships the whole timeline up front, so it
|
||||
counts as finished from the start)."""
|
||||
return bool(session["codec_streaming"]) or bool(session["upstream_finished"])
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -394,6 +913,9 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
span = int(input_ids.shape[0])
|
||||
request_id = require_request_id(info, "talker")
|
||||
|
||||
if self._use_native_backbone:
|
||||
return self._native_preprocess(input_ids, info, device, span, request_id)
|
||||
|
||||
embeds = torch.zeros((span, self._hidden), device=device, dtype=self._dtype)
|
||||
if request_id not in self._sessions:
|
||||
# The request id, not vLLM's per-segment prefill flag, is the
|
||||
@@ -404,47 +926,56 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
|
||||
session = self._sessions[request_id]
|
||||
self._refresh_session_timeline(session, info, device)
|
||||
# Sync mode: one NeMo TTS step per engine step (the verified parity
|
||||
# path, unchanged). Async-chunk mode: DRAIN every received timeline
|
||||
# position on each wake. The engine tokens are placeholders
|
||||
# (CONTINUE/STOP) and the codes ride the cumulative payload, so
|
||||
# multiple TTS steps per engine step are safe — and necessary:
|
||||
# DRAIN every received timeline position on each wake, in both modes.
|
||||
# The engine tokens are placeholders (CONTINUE/STOP) and the codes ride
|
||||
# the cumulative payload, so multiple TTS steps per engine step are
|
||||
# safe — and necessary:
|
||||
# * sync mode ships the WHOLE timeline up front, so the first decode
|
||||
# wake runs every TTS step back-to-back instead of paying a full
|
||||
# engine round trip (scheduler + sampler + IPC) per 80 ms frame;
|
||||
# the codes are computed in the same order as the stepwise loop,
|
||||
# so the output is bit-identical,
|
||||
# * the save_async background thread can coalesce thinker steps into
|
||||
# one chunk (fewer wakes than new positions), and
|
||||
# * the prompt-region PAD steps (logical_prompt_len - 1 of them) all
|
||||
# become available with chunk 0 and must not cost one upstream
|
||||
# chunk each.
|
||||
# Rows stay on the GPU (tiny [1, Q] longs): a per-frame .cpu() would
|
||||
# stall the dispatch pipeline once per frame; the stage producers move
|
||||
# the final payload to CPU when shipping.
|
||||
finished = False
|
||||
steps_run = 0
|
||||
new_code_rows: list[torch.Tensor] = []
|
||||
codec_streaming = bool(session["codec_streaming"])
|
||||
while int(session["step"]) < int(session["timeline"].numel()):
|
||||
codes_row, finished = self._step_session(session)
|
||||
if codec_streaming:
|
||||
# One D2H copy per wake is enough; avoid synchronizing after every drained TTS step.
|
||||
new_code_rows.append(codes_row)
|
||||
else:
|
||||
session["codes_rows"].append(codes_row.cpu())
|
||||
steps_run += 1
|
||||
if session["sync_mode"]:
|
||||
break
|
||||
total = int(session["timeline"].numel())
|
||||
if session.get("use_graph"):
|
||||
steps_run = total - int(session["step"])
|
||||
if steps_run > 0:
|
||||
self._step_graph.run_steps(steps_run)
|
||||
session["step"] = total
|
||||
finished = self._segment_stops_at_exhaustion(session)
|
||||
else:
|
||||
while int(session["step"]) < total:
|
||||
codes_row, finished = self._step_session(session)
|
||||
session["codes_rows"].append(codes_row)
|
||||
steps_run += 1
|
||||
if steps_run == 0:
|
||||
# Zero-progress wake (duplicate/terminal-marker chunk): no TTS step
|
||||
# was run, so never fabricate a frame. Stop the current scheduler
|
||||
# segment only for native codec streaming, or once the ordinary
|
||||
# offline async producer has sent its terminal marker.
|
||||
finished = int(session["step"]) >= int(session["timeline"].numel()) and (
|
||||
bool(session["codec_streaming"]) or bool(session["upstream_finished"])
|
||||
# was run, so never fabricate a frame. Stop only when this session
|
||||
# may end at exhaustion (see _segment_stops_at_exhaustion);
|
||||
# otherwise emit CONTINUE and wait for more upstream input.
|
||||
finished = int(session["step"]) >= int(session["timeline"].numel()) and self._segment_stops_at_exhaustion(
|
||||
session
|
||||
)
|
||||
if finished:
|
||||
embeds[:, 0] = 1.0
|
||||
return input_ids, embeds, {}
|
||||
if codec_streaming:
|
||||
# The code2wav producer owns suffix slicing and therefore requires
|
||||
# cumulative history on every wake. Keep the history on CPU and do
|
||||
# one D2H transfer for all rows produced by this wake.
|
||||
session["codes_rows"].append(torch.cat(new_code_rows, dim=0).cpu())
|
||||
cumulative = torch.cat(session["codes_rows"], dim=0)
|
||||
# The code2wav producers own the suffix slicing and therefore receive
|
||||
# the cumulative history on every wake. Rows stay on the GPU (tiny
|
||||
# [1, Q] longs; a per-step .cpu() would stall dispatch once per frame);
|
||||
# the producers move the shipped payload to CPU.
|
||||
if session.get("use_graph"):
|
||||
cumulative = self._step_graph.codes_rows(1, int(session["step"]))
|
||||
else:
|
||||
cumulative = torch.cat(session["codes_rows"], dim=0)
|
||||
if finished:
|
||||
embeds[:, 0] = 1.0 # stop flag -> compute_logits emits the stop token
|
||||
info_update: dict[str, Any] = {
|
||||
@@ -466,6 +997,8 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
for request_id in finished_req_ids:
|
||||
self._sessions.pop(str(request_id), None)
|
||||
if self._uncond_graph is not None:
|
||||
self._uncond_graph.release(str(request_id))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weights.
|
||||
@@ -497,22 +1030,49 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
tts = DuplexEARTTS({"data": tts_data, "model": tts_model_cfg})
|
||||
|
||||
prefix = "tts_model."
|
||||
backbone_prefix = "tts_model.tts_model.backbone."
|
||||
state: dict[str, torch.Tensor] = {}
|
||||
backbone_weights: list[tuple[str, torch.Tensor]] = []
|
||||
for name, tensor in weights:
|
||||
if not name.startswith(prefix):
|
||||
continue
|
||||
if self._use_native_backbone and name.startswith(backbone_prefix):
|
||||
# The backbone runs as a native vLLM Gemma3 model instead.
|
||||
backbone_weights.append((name, tensor))
|
||||
if not self._native_guidance:
|
||||
continue
|
||||
# CFG keeps a second copy in the vendored module: the
|
||||
# unconditional stream decodes on the HF backbone.
|
||||
target = name.removeprefix(prefix)
|
||||
if target.startswith("audio_codec."):
|
||||
state[target] = tensor.to(dtype=torch.float32)
|
||||
else:
|
||||
state[target] = tensor.to(dtype=self._dtype)
|
||||
if self._use_native_backbone:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
iter_backbone_weights,
|
||||
)
|
||||
|
||||
loaded = self.backbone.load_weights(
|
||||
iter_backbone_weights(backbone_weights, hidden_size=self._hidden, dtype=self._dtype)
|
||||
)
|
||||
logger.info("NemotronVoiceChat native talker backbone loaded %d vLLM tensors", len(loaded))
|
||||
if not state:
|
||||
raise ValueError(
|
||||
"No 'tts_model.*' tensors found in the checkpoint; the NemotronVoiceChat "
|
||||
"unified model.safetensors is required for the talker stage."
|
||||
)
|
||||
if self._use_native_backbone and not self._native_guidance:
|
||||
# The HF backbone inside the vendored model is unused on the native
|
||||
# path (only the embedding/CAS/MoG modules are consulted); drop it
|
||||
# before loading so its ~2.4 GiB of fp32 parameters are freed and
|
||||
# its (rerouted) weights are not reported missing. With CFG the
|
||||
# backbone stays: the unconditional stream decodes on it.
|
||||
tts.tts_model.backbone = nn.Identity()
|
||||
missing, unexpected = tts.load_state_dict(state, strict=False)
|
||||
real_missing = [m for m in missing if not m.startswith("audio_codec.")]
|
||||
if self._use_native_backbone and not self._native_guidance:
|
||||
real_missing = [m for m in real_missing if not m.startswith("tts_model.backbone.")]
|
||||
if real_missing:
|
||||
raise ValueError(f"NemotronVoiceChat talker is missing TTS weights: {sorted(real_missing)[:10]} ...")
|
||||
if unexpected:
|
||||
@@ -533,4 +1093,40 @@ class NemotronVoiceChatTalkerForConditionalGeneration(nn.Module):
|
||||
self._speaker_name,
|
||||
self._guidance_enabled(),
|
||||
)
|
||||
if self._use_cuda_graphs:
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_graph import (
|
||||
TalkerStepGraph,
|
||||
)
|
||||
|
||||
step_graph = TalkerStepGraph(
|
||||
self.tts,
|
||||
dtype=self._dtype,
|
||||
device=torch.device(device) if not isinstance(device, torch.device) else device,
|
||||
max_cache_len=self._graph_max_cache_len,
|
||||
guidance_enabled=self._guidance_enabled(),
|
||||
)
|
||||
supported, reason = step_graph.config_supported()
|
||||
if supported:
|
||||
self._step_graph = step_graph
|
||||
else:
|
||||
logger.warning(
|
||||
"NemotronVoiceChat talker CUDA graphs requested but unsupported (%s); "
|
||||
"using the eager per-frame step.",
|
||||
reason,
|
||||
)
|
||||
if self._use_native_backbone:
|
||||
# Capture the per-frame MoG sampling graph at load time; a lazy
|
||||
# first-frame capture would otherwise cost the first session a
|
||||
# few hundred ms of first-packet latency.
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.talker_native import (
|
||||
MoGStepGraph,
|
||||
)
|
||||
|
||||
mog_batch = 2 if self._native_guidance else 1
|
||||
self._mog_graph = MoGStepGraph(
|
||||
self.tts.tts_model,
|
||||
self.tts._get_generation_config(self._native_guidance),
|
||||
batch=mog_batch,
|
||||
)
|
||||
self._mog_graph.ensure_captured(torch.zeros(mog_batch, 1, self._hidden, dtype=self._dtype, device=device))
|
||||
return {name for name, _ in self.named_parameters()}
|
||||
@@ -61,43 +61,74 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Mel columns kept ahead of the earliest column a perception chunk consumes,
|
||||
# so reflect-pad / preemphasis boundary effects at the trimmed rolling-window
|
||||
# head (which only reach the first ~2 columns) never touch a consumed column.
|
||||
_DUPLEX_MEL_MARGIN_COLS = 8
|
||||
|
||||
_LM_ARCHITECTURE = "NemotronHForCausalLM"
|
||||
|
||||
|
||||
def slice_perception_streaming_mel(
|
||||
processed_signal: torch.Tensor,
|
||||
def _streaming_cfg_at(value: Any, index: int) -> int:
|
||||
return int(value[index] if isinstance(value, list | tuple) else value)
|
||||
|
||||
|
||||
def perception_chunk_geometry(
|
||||
frame_idx: int,
|
||||
stream_total_cols: int,
|
||||
streaming_cfg: Any,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Slice one 80 ms mel chunk using NeMo's cache-manager geometry."""
|
||||
|
||||
def _at(value: Any, index: int) -> int:
|
||||
return int(value[index] if isinstance(value, list | tuple) else value)
|
||||
|
||||
chunk_size_first = _at(streaming_cfg.chunk_size, 0)
|
||||
chunk_size = _at(streaming_cfg.chunk_size, 1)
|
||||
shift_size_first = _at(streaming_cfg.shift_size, 0)
|
||||
shift_size = _at(streaming_cfg.shift_size, 1)
|
||||
pre_encode_first = _at(streaming_cfg.pre_encode_cache_size, 0)
|
||||
pre_encode_size = _at(streaming_cfg.pre_encode_cache_size, 1)
|
||||
|
||||
if frame_idx == 0:
|
||||
mel_chunk = processed_signal[:, :, :chunk_size_first]
|
||||
if pre_encode_first > 0:
|
||||
mel_chunk = torch.nn.functional.pad(mel_chunk, (pre_encode_first, 0))
|
||||
return mel_chunk, 0
|
||||
) -> tuple[int, int, int]:
|
||||
"""Stream-global (cache_start, chunk_start, chunk_end) mel columns for ``frame_idx > 0``."""
|
||||
chunk_size = _streaming_cfg_at(streaming_cfg.chunk_size, 1)
|
||||
shift_size_first = _streaming_cfg_at(streaming_cfg.shift_size, 0)
|
||||
shift_size = _streaming_cfg_at(streaming_cfg.shift_size, 1)
|
||||
pre_encode_size = _streaming_cfg_at(streaming_cfg.pre_encode_cache_size, 1)
|
||||
|
||||
chunk_start = shift_size_first + (frame_idx - 1) * shift_size
|
||||
chunk_end = chunk_start + chunk_size
|
||||
# Match NeMo's tail-boundary rule. With one 80 ms frame per call this
|
||||
# normally leaves the expected trailing STFT columns unused.
|
||||
offset = chunk_size - shift_size_first
|
||||
if chunk_end > processed_signal.shape[-1] - offset:
|
||||
chunk_end = processed_signal.shape[-1] - offset
|
||||
if chunk_end > stream_total_cols - offset:
|
||||
chunk_end = stream_total_cols - offset
|
||||
chunk_start = chunk_end - chunk_size
|
||||
main_chunk = processed_signal[:, :, chunk_start:chunk_end]
|
||||
cache_start = max(0, chunk_start - pre_encode_size)
|
||||
cache_mel = processed_signal[:, :, cache_start:chunk_start]
|
||||
return cache_start, chunk_start, chunk_end
|
||||
|
||||
|
||||
def slice_perception_streaming_mel(
|
||||
processed_signal: torch.Tensor,
|
||||
frame_idx: int,
|
||||
streaming_cfg: Any,
|
||||
*,
|
||||
window_col0: int = 0,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Slice one 80 ms mel chunk using NeMo's cache-manager geometry.
|
||||
|
||||
``processed_signal`` may be the mel of a trailing window of the stream
|
||||
instead of the full history; ``window_col0`` is the stream-global column
|
||||
index of its first column (the window must end at the stream tail).
|
||||
"""
|
||||
chunk_size_first = _streaming_cfg_at(streaming_cfg.chunk_size, 0)
|
||||
pre_encode_first = _streaming_cfg_at(streaming_cfg.pre_encode_cache_size, 0)
|
||||
pre_encode_size = _streaming_cfg_at(streaming_cfg.pre_encode_cache_size, 1)
|
||||
|
||||
if frame_idx == 0:
|
||||
if window_col0 != 0:
|
||||
raise ValueError("perception stream start requires an untrimmed mel window")
|
||||
mel_chunk = processed_signal[:, :, :chunk_size_first]
|
||||
if pre_encode_first > 0:
|
||||
mel_chunk = torch.nn.functional.pad(mel_chunk, (pre_encode_first, 0))
|
||||
return mel_chunk, 0
|
||||
|
||||
stream_total_cols = window_col0 + int(processed_signal.shape[-1])
|
||||
cache_start, chunk_start, chunk_end = perception_chunk_geometry(frame_idx, stream_total_cols, streaming_cfg)
|
||||
if cache_start < window_col0:
|
||||
raise ValueError(
|
||||
f"perception mel window starts at column {window_col0} but frame {frame_idx} needs column {cache_start}"
|
||||
)
|
||||
main_chunk = processed_signal[:, :, chunk_start - window_col0 : chunk_end - window_col0]
|
||||
cache_mel = processed_signal[:, :, cache_start - window_col0 : chunk_start - window_col0]
|
||||
if cache_mel.shape[-1] < pre_encode_size:
|
||||
cache_mel = torch.nn.functional.pad(
|
||||
cache_mel,
|
||||
@@ -300,12 +331,14 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
|
||||
|
||||
# Stateful per-request execution is only validated at batch size 1
|
||||
# (the shipped offline scope); fail fast on silent multi-request use.
|
||||
# Multi-session batching: all per-request state (audio frames, fused
|
||||
# prefill embeds, function-token feedback) lives in _sessions keyed by
|
||||
# request id; the runner drives preprocess per request and hands
|
||||
# postprocess each request's own hidden-state slice, and the NemotronH
|
||||
# backbone batches natively on paged KV. Validated at max_num_seqs<=4.
|
||||
max_num_seqs = int(getattr(vllm_config.scheduler_config, "max_num_seqs", 1))
|
||||
if max_num_seqs != 1:
|
||||
raise NotImplementedError(
|
||||
f"NemotronVoiceChat thinker supports max_num_seqs=1 only (got {max_num_seqs}); "
|
||||
"multi-request batching of the per-request fusion state is not implemented."
|
||||
)
|
||||
if max_num_seqs > 1:
|
||||
logger.info("NemotronVoiceChat thinker: multi-session batching enabled (max_num_seqs=%d)", max_num_seqs)
|
||||
|
||||
# request_id -> per-request state (audio frames, prefill embeds, counters).
|
||||
self._sessions: dict[str, dict[str, Any]] = {}
|
||||
@@ -367,11 +400,16 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
multimodal_outputs: dict[str, torch.Tensor] = {}
|
||||
if self._use_function_head and model_outputs.numel():
|
||||
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information")
|
||||
single_request = isinstance(info_dicts, list) and len(info_dicts) == 1
|
||||
if self._use_function_head and model_outputs.numel() and single_request:
|
||||
# Only a single-request batch can attribute the batch's last row
|
||||
# to a request. With several concurrent requests this key is
|
||||
# omitted and postprocess computes each request's token from its
|
||||
# own hidden-state slice instead (and applies any forced token).
|
||||
with torch.inference_mode():
|
||||
function_token = self.function_head(model_outputs[-1:, :].to(self._dtype)).argmax(dim=-1)
|
||||
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information")
|
||||
info = info_dicts[0] if isinstance(info_dicts, list) and len(info_dicts) == 1 else None
|
||||
info = info_dicts[0]
|
||||
request_id = info.get("request_id") if isinstance(info, dict) else None
|
||||
session = self._sessions.get(request_id) if isinstance(request_id, str) else None
|
||||
forced_token = session.get("forced_function_token") if isinstance(session, dict) else None
|
||||
@@ -474,25 +512,51 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
|
||||
)
|
||||
audio = torch.cat([audio, frame_audio])
|
||||
else:
|
||||
# Stream start. A session normally starts at input sequence 1;
|
||||
# seeing a later sequence means model-side session state was
|
||||
# dropped mid-stream (request abort racing an already-queued step,
|
||||
# or a stage-request re-open). Restart perception from this frame
|
||||
# instead of failing the engine step.
|
||||
if source_input_seq != 1:
|
||||
raise ValueError(
|
||||
f"Nemotron VoiceChat cache-aware perception must start at input sequence 1, got {source_input_seq}"
|
||||
logger.warning(
|
||||
"Nemotron VoiceChat perception session reset mid-stream; restarting at input sequence %d.",
|
||||
source_input_seq,
|
||||
)
|
||||
session["duplex_seq_base"] = source_input_seq - 1
|
||||
session["duplex_window_col0"] = 0
|
||||
audio = frame_audio
|
||||
|
||||
frame_idx = source_input_seq - 1 - int(session.get("duplex_seq_base", 0))
|
||||
|
||||
encoder = self.perception.encoder
|
||||
streaming_cfg = encoder.streaming_cfg
|
||||
window_col0 = int(session.get("duplex_window_col0", 0))
|
||||
if frame_idx > 0:
|
||||
# Bound the rolling audio window: each mel column is local (STFT
|
||||
# centered on hop-aligned samples, no cross-stream normalization),
|
||||
# so only the columns the chunk slice consumes need to exist.
|
||||
# Keep a margin ahead of cache_start so reflect-pad/preemphasis
|
||||
# boundary effects at the trimmed window head never reach a
|
||||
# consumed column.
|
||||
hop = int(getattr(self._streaming_preprocessor.featurizer, "hop_length", 160))
|
||||
total_samples = window_col0 * hop + int(audio.numel())
|
||||
stream_total_cols = total_samples // hop + 1
|
||||
cache_start, _, _ = perception_chunk_geometry(frame_idx, stream_total_cols, streaming_cfg)
|
||||
desired_col0 = max(0, cache_start - _DUPLEX_MEL_MARGIN_COLS)
|
||||
if desired_col0 > window_col0:
|
||||
audio = audio[(desired_col0 - window_col0) * hop :]
|
||||
window_col0 = desired_col0
|
||||
with torch.inference_mode():
|
||||
processed_signal, _ = self._streaming_preprocessor(
|
||||
input_signal=audio.unsqueeze(0),
|
||||
length=torch.tensor([audio.numel()], device=device, dtype=torch.long),
|
||||
)
|
||||
|
||||
frame_idx = source_input_seq - 1
|
||||
mel_chunk, drop_extra_pre_encoded = slice_perception_streaming_mel(
|
||||
processed_signal,
|
||||
frame_idx,
|
||||
streaming_cfg,
|
||||
window_col0=window_col0,
|
||||
)
|
||||
if frame_idx == 0:
|
||||
caches = encoder.get_initial_cache_state(
|
||||
@@ -538,6 +602,7 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
|
||||
)
|
||||
stable = stable[0].to(self._dtype)
|
||||
session["duplex_audio"] = audio
|
||||
session["duplex_window_col0"] = window_col0
|
||||
session["perception_cache_last_channel"] = cache_channel
|
||||
session["perception_cache_last_time"] = cache_time
|
||||
session["perception_cache_last_channel_len"] = cache_channel_len
|
||||
@@ -599,12 +664,14 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
|
||||
|
||||
self._sync_forced_function_response(session, runtime_config)
|
||||
|
||||
if source_input_seq <= 1:
|
||||
offset = max(0, int(info.get("_omni_num_computed_tokens", 0) or 0))
|
||||
prefill = session["prefill_embeds"]
|
||||
span = int(input_ids.shape[0])
|
||||
if offset + span > int(prefill.shape[0]):
|
||||
raise ValueError("Nemotron VoiceChat duplex first append exceeds its fused prompt")
|
||||
offset = max(0, int(info.get("_omni_num_computed_tokens", 0) or 0))
|
||||
span = int(input_ids.shape[0])
|
||||
prefill = session["prefill_embeds"]
|
||||
if offset + span <= int(prefill.shape[0]):
|
||||
# Engine positions inside the fused prompt prefill (possibly
|
||||
# chunked). The position decides, not the duplex input sequence:
|
||||
# after a mid-stream session reset the re-opened request prefills
|
||||
# its prompt again at a sequence far beyond 1.
|
||||
return (
|
||||
input_ids,
|
||||
prefill[offset : offset + span].to(self._dtype),
|
||||
@@ -887,14 +954,18 @@ class NemotronVoiceChatThinkerForConditionalGeneration(nn.Module, HasInnerState,
|
||||
if isinstance(request_id, str):
|
||||
session = self._sessions.get(request_id)
|
||||
if session is not None:
|
||||
# StreamingUpdate replaces the runner payload at each append;
|
||||
# retain the function channel in model-owned session state.
|
||||
session["func_token"] = int(token.item())
|
||||
forced = session.pop("forced_function_token", None)
|
||||
if isinstance(forced, int):
|
||||
queue = session.get("forced_function_tokens")
|
||||
if not isinstance(queue, list) or not queue or queue.pop(0) != forced:
|
||||
raise RuntimeError("Nemotron VoiceChat forced function-token queue lost alignment")
|
||||
# Apply the override here too: with several concurrent
|
||||
# requests make_omni_output omits the shared token, so the
|
||||
# token above is this request's own computed one.
|
||||
token = torch.full_like(token, forced)
|
||||
# StreamingUpdate replaces the runner payload at each append;
|
||||
# retain the function channel in model-owned session state.
|
||||
session["func_token"] = int(token.item())
|
||||
update: dict[str, Any] = {"nvc_prev_function_token": token}
|
||||
if os.environ.get("NEMOTRON_VOICECHAT_DEBUG_FUNCTION_TIMELINE", "0") == "1":
|
||||
existing = kwargs.get("nvc_function_tokens")
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CUDA-graph fast path for the NemotronVoiceChat talker per-frame step.
|
||||
|
||||
The eager talker step is launch-bound: one 80 ms frame costs ~3200 kernel
|
||||
launches (28-layer Gemma3 backbone + 5 MoG unmasking passes + a 31-level
|
||||
depthsum loop) for ~9 ms of actual GPU work, so the eager wall time is
|
||||
~25-34 ms/frame. This module captures the WHOLE step into one CUDA graph so
|
||||
the drain loop degenerates to ``graph.replay()`` per frame (~12 ms/frame,
|
||||
GPU-bound). This mirrors NVIDIA's own deployment of this model, which serves
|
||||
EAR-TTS through vLLM decode CUDA graphs with CFG baked into the capture.
|
||||
|
||||
What makes the step capturable:
|
||||
|
||||
* ``StaticCache`` (preallocated KV, write index via ``cache_position``)
|
||||
instead of a growing HF ``DynamicCache``;
|
||||
* the CharAwareSubwordEncoder — dynamic char-length shapes plus per-call
|
||||
``.cpu().tolist()`` / ``subword_mask.any()`` host syncs — is hoisted OUT of
|
||||
the step: the frame-locked text timeline is known ahead of the audio, so its
|
||||
subword embeddings are precomputed in one batched call per session and the
|
||||
in-graph step just gathers a row (``subword_embeds`` fast path in the
|
||||
vendored model);
|
||||
* the code feedback, output row write, step counter, and cache position all
|
||||
advance INSIDE the graph, so a replay is self-contained.
|
||||
|
||||
The captured state machine (all device tensors, no host round trips):
|
||||
|
||||
cur_id = timeline_ids[step_ctr] # for EOS-silence gating
|
||||
cur_emb = cas_table[step_ctr] # precomputed CAS row
|
||||
code = one EAR-TTS step (backbone + MoG, CFG batch=2, StaticCache)
|
||||
code_buf, codes_out[step_ctr] = code # feedback + output
|
||||
step_ctr += 1; cache_pos += 1
|
||||
|
||||
MoG sampling noise draws from the CUDA graph-safe Philox generator, which
|
||||
advances per replay — audio is NOT bit-identical to the eager path (nor is the
|
||||
eager path bit-stable across process RNG states); the frame-locked text channel
|
||||
is unaffected. Sessions longer than ``max_cache_len`` fall back to the eager
|
||||
path (the talker decides per session).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TalkerStepGraph:
|
||||
"""Owns the StaticCache, the per-session buffers, and the captured graph.
|
||||
|
||||
Sessions are strictly serial (the talker enforces ``max_num_seqs=1``); the
|
||||
same buffers and capture are reused across requests via ``start_session``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tts: Any,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
max_cache_len: int,
|
||||
guidance_enabled: bool,
|
||||
) -> None:
|
||||
self.tts = tts
|
||||
self.model = tts.tts_model
|
||||
self.device = device
|
||||
self.dtype = dtype
|
||||
self.max_cache_len = int(max_cache_len)
|
||||
self.guidance_enabled = bool(guidance_enabled)
|
||||
self.generation_config = tts._get_generation_config(self.guidance_enabled)
|
||||
|
||||
# Attention in StaticCache mode runs over the FULL padded cache length,
|
||||
# so per-step cost scales with the bucket size (~11 ms/step at 512
|
||||
# positions vs ~20 ms/step at 4096 in fp32 on H100). Sessions with a
|
||||
# known timeline (sync mode) pick the smallest fitting bucket; growing
|
||||
# (async) sessions use the largest, since codes already shipped cannot
|
||||
# be regenerated under a different cache length mid-request.
|
||||
self._bucket_lens = [b for b in (512, 1024, 2048) if b < self.max_cache_len] + [self.max_cache_len]
|
||||
self._buckets: dict[int, dict[str, Any]] = {}
|
||||
self._active_bucket: int | None = None
|
||||
self._broken = False
|
||||
self._init_len: int | None = None
|
||||
self._timeline_len = 0
|
||||
|
||||
hidden = int(self.model.hidden_size)
|
||||
quantizers = int(self.model.config.num_quantizers)
|
||||
with torch.inference_mode():
|
||||
self._timeline_ids = torch.zeros(self.max_cache_len, dtype=torch.long, device=device)
|
||||
self._cas_table = torch.zeros(self.max_cache_len, hidden, dtype=dtype, device=device)
|
||||
self._codes_out = torch.zeros(self.max_cache_len, quantizers, dtype=torch.long, device=device)
|
||||
self._code_buf = torch.zeros(1, 1, quantizers, dtype=torch.long, device=device)
|
||||
self._step_ctr = torch.ones(1, dtype=torch.long, device=device)
|
||||
self._cache_pos = torch.zeros(1, dtype=torch.long, device=device)
|
||||
self._ones_mask = torch.ones(1, 1, dtype=torch.bool, device=device)
|
||||
# Prepared 4D attention mask over the cache slots (True = attend).
|
||||
# Passing a ready 4D mask makes transformers' mask pipeline return
|
||||
# it as-is; the HF-derived mask (and anything else that reads the
|
||||
# cache's python-int lengths) would otherwise bake the capture-time
|
||||
# position into the graph. Per-bucket graphs slice a leading view.
|
||||
self._attn_mask = torch.zeros(1, 1, 1, self.max_cache_len, dtype=torch.bool, device=device)
|
||||
self._cache: Any = None # the ACTIVE bucket's StaticCache
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Support / capacity checks.
|
||||
# ------------------------------------------------------------------
|
||||
def config_supported(self) -> tuple[bool, str]:
|
||||
"""Static-config gate for the graph path (independent of the request)."""
|
||||
if self.device.type != "cuda":
|
||||
return False, f"device {self.device} is not CUDA"
|
||||
if not self.guidance_enabled:
|
||||
return False, "classifier-free guidance disabled (capture assumes the CFG batch layout)"
|
||||
if self.tts.cfg.tts_config.context_hidden_size is not None:
|
||||
return False, "context-LM conditioning enabled (prev-subword context is not captured)"
|
||||
if self.model.embed_subword is None:
|
||||
return False, "no CharAwareSubwordEncoder to precompute"
|
||||
sliding_window = getattr(self.model.backbone.config, "sliding_window", None)
|
||||
if sliding_window is not None and self.max_cache_len >= int(sliding_window):
|
||||
# The single prepared mask stands in for both the full and the
|
||||
# sliding mask, which is only equivalent while no position can
|
||||
# fall outside the window.
|
||||
return False, (
|
||||
f"max_cache_len {self.max_cache_len} >= backbone sliding_window {sliding_window}; "
|
||||
"the prepared causal mask would diverge from the sliding mask"
|
||||
)
|
||||
return True, ""
|
||||
|
||||
def fits(self, timeline_len: int) -> bool:
|
||||
init_len = self._init_len if self._init_len is not None else 64
|
||||
return timeline_len <= self.max_cache_len - init_len - 1
|
||||
|
||||
def _bucket_len_for(self, timeline_len: int | None) -> int:
|
||||
"""Smallest bucket that fits ``init_len + timeline_len + 1`` positions.
|
||||
|
||||
``None`` (growing async timeline) selects the largest bucket: codes
|
||||
already shipped cannot be regenerated under a different cache length.
|
||||
"""
|
||||
if timeline_len is None:
|
||||
return self._bucket_lens[-1]
|
||||
init_len = self._init_len if self._init_len is not None else 64
|
||||
needed = init_len + timeline_len + 1
|
||||
for bucket in self._bucket_lens:
|
||||
if needed <= bucket:
|
||||
return bucket
|
||||
return self._bucket_lens[-1]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Capture.
|
||||
# ------------------------------------------------------------------
|
||||
def _get_bucket(self, bucket_len: int) -> dict[str, Any]:
|
||||
bucket = self._buckets.get(bucket_len)
|
||||
if bucket is None:
|
||||
from transformers import StaticCache
|
||||
|
||||
bucket = {
|
||||
"cache": StaticCache(
|
||||
config=self.model.backbone.config,
|
||||
max_batch_size=2 if self.guidance_enabled else 1,
|
||||
max_cache_len=bucket_len,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
),
|
||||
"graph": None,
|
||||
}
|
||||
self._buckets[bucket_len] = bucket
|
||||
return bucket
|
||||
|
||||
def _init_forward(self) -> None:
|
||||
"""Eager speaker-prompt prefill into the StaticCache (per session)."""
|
||||
init_inputs = self.tts.get_init_inputs(B=1)
|
||||
init_len = int(init_inputs["code"].shape[1])
|
||||
if self._init_len is None:
|
||||
self._init_len = init_len
|
||||
elif init_len != self._init_len:
|
||||
raise RuntimeError(
|
||||
f"NemotronVoiceChat talker graph: init prompt length changed ({self._init_len} -> {init_len}); "
|
||||
"the captured graph assumes a fixed speaker prompt."
|
||||
)
|
||||
init_inputs.update(
|
||||
{
|
||||
"use_cache": True,
|
||||
"past_key_values": self._cache,
|
||||
"guidance_enabled": self.guidance_enabled,
|
||||
"cache_position": torch.arange(init_len, device=self.device),
|
||||
}
|
||||
)
|
||||
self.model(**init_inputs)
|
||||
self._cache_pos.fill_(init_len)
|
||||
self._code_buf.copy_(init_inputs["code"][:, -1:])
|
||||
# Seed the prepared mask: the speaker-prompt slots are attendable, the
|
||||
# per-step slot bits are set inside the captured step.
|
||||
self._attn_mask.zero_()
|
||||
self._attn_mask[..., :init_len] = True
|
||||
|
||||
def _one_step(self) -> None:
|
||||
"""The captured body: one EAR-TTS frame step + in-graph state advance."""
|
||||
cur_id = self._timeline_ids.index_select(0, self._step_ctr).reshape(1, 1)
|
||||
cur_emb = self._cas_table.index_select(0, self._step_ctr).reshape(1, 1, -1)
|
||||
# Open this step's own cache slot before attending (a token attends to
|
||||
# itself), then run the step with fully tensor-driven mask/positions.
|
||||
self._attn_mask.index_fill_(3, self._cache_pos, True)
|
||||
code, _ = self.tts.infer_codes_one_step(
|
||||
current_subword_id=cur_id,
|
||||
# prev_subword_id only feeds the context-LM channel, which
|
||||
# config_supported() guarantees is disabled.
|
||||
prev_subword_id=cur_id,
|
||||
current_subword_mask=self._ones_mask,
|
||||
prev_audio_tokens=self._code_buf,
|
||||
past_key_values=self._cache,
|
||||
guidance_enabled=self.guidance_enabled,
|
||||
generation_config=self.generation_config,
|
||||
ignore_eos_flag_stop=True,
|
||||
current_subword_embed=cur_emb,
|
||||
cache_position=self._cache_pos,
|
||||
attention_mask=self._attn_mask[..., : self._active_bucket],
|
||||
position_ids=self._cache_pos.reshape(1, 1),
|
||||
)
|
||||
self._code_buf.copy_(code)
|
||||
self._codes_out.index_copy_(0, self._step_ctr, code.reshape(1, -1))
|
||||
self._step_ctr.add_(1)
|
||||
self._cache_pos.add_(1)
|
||||
|
||||
def ensure_captured(self, timeline_len: int | None = None) -> bool:
|
||||
"""Warm up and capture the bucket for this timeline (throwaway session)."""
|
||||
if self._broken:
|
||||
return False
|
||||
bucket_len = self._bucket_len_for(timeline_len)
|
||||
bucket = self._get_bucket(bucket_len)
|
||||
if bucket["graph"] is not None:
|
||||
return True
|
||||
try:
|
||||
with torch.inference_mode():
|
||||
# Throwaway session: buffer VALUES are irrelevant for capture,
|
||||
# only shapes/addresses matter. PAD-filled timeline rows and a
|
||||
# zero CAS table are fine.
|
||||
self._cache = bucket["cache"]
|
||||
self._active_bucket = bucket_len
|
||||
self._cache.reset()
|
||||
self._step_ctr.fill_(1)
|
||||
self._cache_pos.zero_()
|
||||
self._init_forward()
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(3):
|
||||
self._one_step()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
self._one_step()
|
||||
bucket["graph"] = graph
|
||||
logger.info(
|
||||
"NemotronVoiceChat talker CUDA graph captured (bucket=%d, init_len=%d)",
|
||||
bucket_len,
|
||||
self._init_len,
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
self._broken = True
|
||||
logger.exception(
|
||||
"NemotronVoiceChat talker CUDA graph capture failed; falling back to the eager per-frame step."
|
||||
)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session lifecycle.
|
||||
# ------------------------------------------------------------------
|
||||
def start_session(self, timeline: torch.Tensor, *, expected_len: int | None) -> None:
|
||||
"""Reset all captured state and prefill for a new request.
|
||||
|
||||
``expected_len`` is the FINAL timeline length when known up front
|
||||
(sync mode ships the whole timeline; frame-locked async producers ship
|
||||
meta.expected_total_tokens) and selects the smallest fitting bucket;
|
||||
``None`` (unbounded growth, e.g. duplex) selects the largest bucket.
|
||||
"""
|
||||
bucket_len = self._bucket_len_for(expected_len)
|
||||
bucket = self._buckets[bucket_len]
|
||||
assert bucket["graph"] is not None, "ensure_captured() must succeed before start_session()"
|
||||
with torch.inference_mode():
|
||||
self._cache = bucket["cache"]
|
||||
self._active_bucket = bucket_len
|
||||
self._cache.reset()
|
||||
self._step_ctr.fill_(1)
|
||||
self._cache_pos.zero_()
|
||||
self._codes_out.zero_()
|
||||
self._timeline_len = 0
|
||||
self._init_forward()
|
||||
self.extend_timeline(timeline)
|
||||
|
||||
def extend_timeline(self, timeline: torch.Tensor) -> None:
|
||||
"""Adopt a (possibly longer) timeline; CAS-encode only the new rows."""
|
||||
new_len = int(timeline.numel())
|
||||
if new_len <= self._timeline_len:
|
||||
return
|
||||
if not self.fits(new_len):
|
||||
raise RuntimeError(
|
||||
f"NemotronVoiceChat talker graph: timeline of {new_len} positions exceeds the "
|
||||
f"captured capacity ({self.max_cache_len - (self._init_len or 0) - 1} frames). Raise "
|
||||
"hf_overrides.talker_graph_max_cache_len or disable hf_overrides.use_talker_cuda_graphs."
|
||||
)
|
||||
active = self._active_bucket
|
||||
if active is not None and (self._init_len or 0) + new_len + 1 > active:
|
||||
raise RuntimeError(
|
||||
f"NemotronVoiceChat talker graph: timeline of {new_len} positions exceeds the active "
|
||||
f"cache bucket ({active} positions). Raise hf_overrides.talker_graph_max_cache_len or "
|
||||
"disable hf_overrides.use_talker_cuda_graphs."
|
||||
)
|
||||
start = self._timeline_len
|
||||
with torch.inference_mode():
|
||||
new_ids = timeline[start:new_len].to(device=self.device, dtype=torch.long).reshape(-1)
|
||||
self._timeline_ids[start:new_len] = new_ids
|
||||
mask = torch.ones(1, new_ids.numel(), dtype=torch.bool, device=self.device)
|
||||
# One batched CAS call per chunk of new tokens (the encoder's host
|
||||
# syncs and dynamic char shapes stay OUTSIDE the captured step).
|
||||
embeds = self.model.embed_subword(new_ids.unsqueeze(0), mask)[0]
|
||||
self._cas_table[start:new_len] = embeds.to(self.dtype)
|
||||
self._timeline_len = new_len
|
||||
|
||||
def run_steps(self, num_steps: int) -> None:
|
||||
graph = self._buckets[self._active_bucket]["graph"]
|
||||
for _ in range(num_steps):
|
||||
graph.replay()
|
||||
|
||||
def codes_rows(self, first_step: int, last_step: int) -> torch.Tensor:
|
||||
"""Codes for NeMo steps ``first_step..last_step-1`` (row t = step t)."""
|
||||
with torch.inference_mode():
|
||||
return self._codes_out[first_step:last_step].clone()
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Native-vLLM building blocks for the NemotronVoiceChat talker.
|
||||
|
||||
The eager/CUDA-graph talker executes the vendored HF Gemma3 backbone inside the
|
||||
per-request ``preprocess`` hook, next to (not inside) the vLLM engine. The
|
||||
native path (``hf_overrides.use_native_talker``) instead runs the backbone as a
|
||||
real vLLM model — PagedAttention KV cache (capacity and per-step attention cost
|
||||
scale with the ACTUAL session length, no StaticCache buckets), vLLM's own decode
|
||||
CUDA graphs, and, later, multi-session batching. This mirrors NVIDIA's own
|
||||
deployment, whose EAR-TTS is served by a vLLM engine.
|
||||
|
||||
This module hosts the pieces that are independent of the talker class:
|
||||
|
||||
* ``synthesize_backbone_config``: the checkpoint stores the backbone as a NeMo
|
||||
``tts_config.backbone_config`` dict; vLLM needs a real ``Gemma3TextConfig``.
|
||||
The embedding table is never used (the fused embeddings are built from
|
||||
codes + text conditioning), so a tiny dummy vocab keeps it negligible.
|
||||
* ``iter_backbone_weights``: remaps the checkpoint's
|
||||
``tts_model.tts_model.backbone.*`` HF-Gemma3 tensors onto the names vLLM's
|
||||
``Gemma3ForCausalLM`` loader expects, and fabricates the dummy
|
||||
``model.embed_tokens.weight``.
|
||||
* ``build_prefill_embeds`` / ``build_decode_embeds``: the per-token fused input
|
||||
embeddings, numerically verified against the vendored
|
||||
``RVQEARTTSModel.forward`` embedding section for both CFG roles (the
|
||||
speaker-prompt prefill must use the checkpoint's pre-baked
|
||||
``audio_prompt_latent``, not the frozen projection).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
_DUMMY_VOCAB_SIZE = 8
|
||||
|
||||
|
||||
def synthesize_backbone_config(tts_config: dict[str, Any]) -> Any:
|
||||
"""Build a ``Gemma3TextConfig`` from the checkpoint's NeMo backbone dict."""
|
||||
from transformers import AutoConfig
|
||||
|
||||
backbone_type = tts_config.get("backbone_type")
|
||||
if backbone_type != "gemma3_text":
|
||||
raise NotImplementedError(
|
||||
f"NemotronVoiceChat native talker supports a gemma3_text backbone only (got {backbone_type!r})."
|
||||
)
|
||||
backbone_cfg = dict(tts_config.get("backbone_config") or {})
|
||||
backbone_cfg.pop("use_cache", None)
|
||||
config = AutoConfig.for_model(
|
||||
backbone_type,
|
||||
**backbone_cfg,
|
||||
vocab_size=_DUMMY_VOCAB_SIZE,
|
||||
tie_word_embeddings=True,
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def iter_backbone_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
*,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Yield vLLM-Gemma3 weights from the unified checkpoint stream.
|
||||
|
||||
Checkpoint names: ``tts_model.tts_model.backbone.<hf gemma3 names>``.
|
||||
vLLM ``Gemma3ForCausalLM`` expects ``model.<hf gemma3 names>`` plus an
|
||||
embedding table (tied lm_head); the embedding is never consulted (fused
|
||||
embeddings are injected via ``inputs_embeds``), so a dummy is fabricated.
|
||||
"""
|
||||
prefix = "tts_model.tts_model.backbone."
|
||||
for name, tensor in weights:
|
||||
if name.startswith(prefix):
|
||||
yield "model." + name.removeprefix(prefix), tensor.to(dtype=dtype)
|
||||
yield (
|
||||
"model.embed_tokens.weight",
|
||||
torch.zeros(_DUMMY_VOCAB_SIZE, hidden_size, dtype=dtype),
|
||||
)
|
||||
|
||||
|
||||
def build_prefill_embeds(
|
||||
model: Any,
|
||||
*,
|
||||
code: torch.Tensor,
|
||||
subword_ids: torch.Tensor,
|
||||
subword_mask: torch.Tensor,
|
||||
audio_mask: torch.Tensor,
|
||||
audio_prompt_latent: torch.Tensor | None,
|
||||
uncond: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Speaker-prompt prefill embeddings for one CFG role.
|
||||
|
||||
Mirrors the ``audio_mask`` inference branch of the vendored
|
||||
``RVQEARTTSModel.forward`` (verified to ~4e-6 max abs diff against the
|
||||
embeddings the vendored forward feeds its backbone). ``model`` is the
|
||||
vendored ``RVQEARTTSModel``. Returns ``[1, T, H]``.
|
||||
"""
|
||||
shifted_code = F.pad(code[:, :-1], [0, 0, 1, 0])
|
||||
code_embed = model.depthsum_embedding(shifted_code)
|
||||
bos_mask = (audio_mask & (~F.pad(audio_mask[:, :-1], [1, 0]))).unsqueeze(-1)
|
||||
pre_bos_mask = bos_mask.cumsum(dim=1) == 0
|
||||
code_embed = model.embed_code(code_embed)
|
||||
if model.config.get("use_audio_prompt_frozen_projection", False):
|
||||
if audio_prompt_latent is None:
|
||||
weight = model.audio_prompt_projection_W.to(code_embed.device, code_embed.dtype)
|
||||
audio_prompt_latent = torch.nn.functional.linear(code_embed, weight.T)
|
||||
code_embed = torch.where(pre_bos_mask, audio_prompt_latent, code_embed)
|
||||
code_embeds = code_embed + bos_mask * model.bos_emb
|
||||
flag = torch.full((code.size(0), 1, 1), bool(uncond), dtype=torch.bool, device=code.device)
|
||||
cond = model._prepare_conditioning(None, subword_ids, subword_mask, flag)
|
||||
if model.config.use_gated_fusion_for_text_audio:
|
||||
return model.gated_fusion_audio_text(code_embeds, cond)
|
||||
return code_embeds + cond
|
||||
|
||||
|
||||
class MoGStepGraph:
|
||||
"""One captured CUDA graph for the per-frame MoG sampling step.
|
||||
|
||||
On the native path the backbone runs under vLLM's own decode CUDA graphs,
|
||||
but the MoG iterative unmasking (``RVQEARTTSModel.generate_step``: 5
|
||||
effective MoG-head passes + a 31-level depthsum quantization loop) would
|
||||
otherwise run eagerly in ``make_omni_output`` — ~1000 kernel launches for
|
||||
~2 ms of GPU work per 80 ms frame. All shapes are static and the unmask
|
||||
schedule is Python ints, so the whole step captures into one graph whose
|
||||
replay is a single launch. MoG noise draws from the CUDA-graph-safe Philox
|
||||
stream (fresh randomness per replay, not bit-identical to eager).
|
||||
|
||||
``batch`` is the leading hidden dim (1 for the single conditional stream;
|
||||
2 once paired-CFG lands). Capture failures fall back to eager permanently.
|
||||
"""
|
||||
|
||||
def __init__(self, model: Any, generation_config: dict[str, Any], *, batch: int = 1) -> None:
|
||||
self.model = model
|
||||
self.generation_config = dict(generation_config)
|
||||
self.batch = int(batch)
|
||||
self._graph: torch.cuda.CUDAGraph | None = None
|
||||
self._broken = False
|
||||
self._hidden_buf: torch.Tensor | None = None
|
||||
self._code_buf: torch.Tensor | None = None
|
||||
|
||||
def _step(self) -> None:
|
||||
codes, _, _ = self.model.generate_step(self._hidden_buf, ignore_eos_flag_stop=True, **self.generation_config)
|
||||
self._code_buf.copy_(codes)
|
||||
|
||||
def ensure_captured(self, hidden: torch.Tensor) -> bool:
|
||||
if self._graph is not None:
|
||||
return True
|
||||
if self._broken:
|
||||
return False
|
||||
try:
|
||||
with torch.inference_mode():
|
||||
self._hidden_buf = hidden.clone()
|
||||
# Under CFG (guidance_scale set) generate_step chunks the
|
||||
# [cond; uncond] pair and returns codes for the effective
|
||||
# batch only.
|
||||
out_batch = self.batch
|
||||
if self.generation_config.get("guidance_scale") is not None:
|
||||
out_batch = max(self.batch // 2, 1)
|
||||
self._code_buf = torch.zeros(
|
||||
out_batch, 1, int(self.model.config.num_quantizers), dtype=torch.long, device=hidden.device
|
||||
)
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(3):
|
||||
self._step()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
self._step()
|
||||
self._graph = graph
|
||||
return True
|
||||
except Exception:
|
||||
self._broken = True
|
||||
self._graph = None
|
||||
import logging
|
||||
|
||||
logging.getLogger(__name__).exception(
|
||||
"NemotronVoiceChat native talker: MoG step graph capture failed; sampling eagerly."
|
||||
)
|
||||
return False
|
||||
|
||||
def run(self, hidden: torch.Tensor) -> torch.Tensor:
|
||||
"""MoG-sample one frame from ``hidden`` ([batch, 1, H]) -> codes [batch, 1, Q]."""
|
||||
if self.ensure_captured(hidden):
|
||||
with torch.inference_mode():
|
||||
self._hidden_buf.copy_(hidden)
|
||||
self._graph.replay()
|
||||
return self._code_buf.clone()
|
||||
with torch.inference_mode():
|
||||
codes, _, _ = self.model.generate_step(hidden, ignore_eos_flag_stop=True, **self.generation_config)
|
||||
return codes
|
||||
|
||||
|
||||
def build_decode_embeds(
|
||||
model: Any,
|
||||
*,
|
||||
prev_codes: torch.Tensor,
|
||||
subword_embed: torch.Tensor,
|
||||
uncond: bool,
|
||||
) -> torch.Tensor:
|
||||
"""One decode-step fused embedding for one CFG role (``[1, 1, H]``).
|
||||
|
||||
``subword_embed`` is the precomputed CharAwareSubwordEncoder row for the
|
||||
current timeline token (host-sync free; see the CAS table precompute).
|
||||
"""
|
||||
code_embeds = model.embed_code(model.depthsum_embedding(prev_codes))
|
||||
flag = torch.full((prev_codes.size(0), 1, 1), bool(uncond), dtype=torch.bool, device=prev_codes.device)
|
||||
cond = model._prepare_conditioning(None, None, None, flag, subword_embeds=subword_embed)
|
||||
if model.config.use_gated_fusion_for_text_audio:
|
||||
return model.gated_fusion_audio_text(code_embeds, cond)
|
||||
return code_embeds + cond
|
||||
|
||||
|
||||
class UncondStream:
|
||||
"""The unconditional CFG stream, run on the vendored HF Gemma3 backbone.
|
||||
|
||||
NeMo applies classifier-free guidance by duplicating every backbone input
|
||||
into a [cond; uncond] batch (``RVQEARTTSModel.forward``) whose second half
|
||||
replaces the whole conditioning sum with ``null_emb``, then blending the
|
||||
two hidden rows inside ``generate_step`` (lm_head EOS blend and the
|
||||
post-MLP blend inside ``MoGHead.infer``).
|
||||
|
||||
On the native path the conditional stream's KV lives inside vLLM's paged
|
||||
cache, where a second same-length sequence cannot ride along without
|
||||
scheduler-level pair locking. This class instead mirrors the stream
|
||||
position-for-position on the vendored HF backbone with its own
|
||||
``DynamicCache`` — one extra [1, 1, H] decode step per frame. Both streams
|
||||
are fed the SAME sampled code row each frame (prev-codes conditioning is
|
||||
shared in NeMo's layout; only the text conditioning differs).
|
||||
"""
|
||||
|
||||
def __init__(self, model: Any) -> None:
|
||||
self.model = model
|
||||
self.cache: Any = None
|
||||
self.pos = 0
|
||||
|
||||
@torch.inference_mode()
|
||||
def prefill(self, embeds: torch.Tensor) -> None:
|
||||
"""Consume the speaker-prompt prefill embeddings (``[1, T, H]``)."""
|
||||
from transformers import DynamicCache
|
||||
|
||||
self.cache = DynamicCache()
|
||||
length = int(embeds.shape[1])
|
||||
position_ids = torch.arange(length, device=embeds.device).unsqueeze(0)
|
||||
outputs = self.model.backbone(
|
||||
inputs_embeds=embeds,
|
||||
attention_mask=None,
|
||||
position_ids=position_ids,
|
||||
past_key_values=self.cache,
|
||||
use_cache=True,
|
||||
)
|
||||
self.cache = outputs.past_key_values
|
||||
self.pos = length
|
||||
|
||||
@torch.inference_mode()
|
||||
def step(self, embed: torch.Tensor) -> torch.Tensor:
|
||||
"""One decode step (``[1, 1, H]`` in) -> uncond hidden (``[1, 1, H]``)."""
|
||||
if self.cache is None:
|
||||
raise RuntimeError("UncondStream.step called before prefill")
|
||||
position_ids = torch.tensor([[self.pos]], device=embed.device)
|
||||
outputs = self.model.backbone(
|
||||
inputs_embeds=embed,
|
||||
attention_mask=None,
|
||||
position_ids=position_ids,
|
||||
past_key_values=self.cache,
|
||||
use_cache=True,
|
||||
)
|
||||
self.cache = outputs.past_key_values
|
||||
self.pos += 1
|
||||
return outputs.last_hidden_state
|
||||
|
||||
|
||||
class UncondStepGraph:
|
||||
"""Captured per-frame decode step for the unconditional CFG stream.
|
||||
|
||||
Backbone-only StaticCache stepping with the same capture hygiene as
|
||||
``talker_graph.TalkerStepGraph``: a prepared 4D boolean attention mask
|
||||
(triggers the HF early-exit so no python-int cache length is baked into
|
||||
the graph), explicit ``position_ids``/``cache_position`` tensors, and the
|
||||
in-graph slot-open ``index_fill_``. One session owns the graph at a time
|
||||
(StaticCache addresses are capture-bound); concurrent sessions fall back
|
||||
to the eager ``UncondStream``.
|
||||
"""
|
||||
|
||||
_BUCKETS = (512, 1024, 2048, 4096)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: Any,
|
||||
*,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
max_cache_len: int = 4096,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.max_cache_len = int(max_cache_len)
|
||||
self._buckets: dict[int, dict[str, Any]] = {}
|
||||
self._broken = False
|
||||
self.owner: str | None = None
|
||||
self._embed_buf = torch.zeros(1, 1, hidden_size, dtype=dtype, device=device)
|
||||
self._hidden_buf = torch.zeros(1, 1, hidden_size, dtype=dtype, device=device)
|
||||
self._cache_pos = torch.zeros(1, dtype=torch.long, device=device)
|
||||
self._attn_mask = torch.zeros(1, 1, 1, self.max_cache_len, dtype=torch.bool, device=device)
|
||||
self._cache: Any = None
|
||||
self._active_bucket = 0
|
||||
|
||||
def bucket_for(self, total_positions: int | None) -> int | None:
|
||||
"""Smallest bucket holding ``total_positions``; None if it cannot fit."""
|
||||
for bucket in self._BUCKETS:
|
||||
if bucket > self.max_cache_len:
|
||||
break
|
||||
if total_positions is not None and total_positions <= bucket:
|
||||
return bucket
|
||||
largest = max(b for b in self._BUCKETS if b <= self.max_cache_len)
|
||||
return largest if total_positions is None or total_positions <= largest else None
|
||||
|
||||
def _get_bucket(self, bucket_len: int) -> dict[str, Any]:
|
||||
state = self._buckets.get(bucket_len)
|
||||
if state is None:
|
||||
from transformers import StaticCache
|
||||
|
||||
state = {
|
||||
"cache": StaticCache(
|
||||
config=self.model.backbone.config,
|
||||
max_batch_size=1,
|
||||
max_cache_len=bucket_len,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
),
|
||||
"graph": None,
|
||||
}
|
||||
self._buckets[bucket_len] = state
|
||||
return state
|
||||
|
||||
def _one_step(self) -> None:
|
||||
self._attn_mask.index_fill_(3, self._cache_pos, True)
|
||||
outputs = self.model.backbone(
|
||||
inputs_embeds=self._embed_buf,
|
||||
attention_mask=self._attn_mask[..., : self._active_bucket],
|
||||
position_ids=self._cache_pos.reshape(1, 1),
|
||||
past_key_values=self._cache,
|
||||
use_cache=True,
|
||||
cache_position=self._cache_pos,
|
||||
)
|
||||
self._hidden_buf.copy_(outputs.last_hidden_state)
|
||||
self._cache_pos.add_(1)
|
||||
|
||||
def start_session(self, request_id: str, prefill_embeds: torch.Tensor, total_positions: int | None) -> bool:
|
||||
"""Claim the graph for one session and prefill its cache. False -> use eager."""
|
||||
if self._broken or self.owner is not None:
|
||||
return False
|
||||
bucket_len = self.bucket_for(total_positions)
|
||||
if bucket_len is None:
|
||||
return False
|
||||
try:
|
||||
with torch.inference_mode():
|
||||
state = self._get_bucket(bucket_len)
|
||||
self._cache = state["cache"]
|
||||
self._active_bucket = bucket_len
|
||||
self._cache.reset()
|
||||
init_len = int(prefill_embeds.shape[1])
|
||||
self.model.backbone(
|
||||
inputs_embeds=prefill_embeds.to(self.dtype),
|
||||
position_ids=torch.arange(init_len, device=self.device).unsqueeze(0),
|
||||
past_key_values=self._cache,
|
||||
use_cache=True,
|
||||
cache_position=torch.arange(init_len, device=self.device),
|
||||
)
|
||||
self._attn_mask.zero_()
|
||||
self._attn_mask[..., :init_len] = True
|
||||
self._cache_pos.fill_(init_len)
|
||||
if state["graph"] is None:
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
saved_pos = int(self._cache_pos.item())
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(3):
|
||||
self._one_step()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
self._one_step()
|
||||
state["graph"] = graph
|
||||
# Rewind the warmup/capture side effects: re-prefill so the
|
||||
# cache holds only the speaker prompt again.
|
||||
self._cache.reset()
|
||||
self.model.backbone(
|
||||
inputs_embeds=prefill_embeds.to(self.dtype),
|
||||
position_ids=torch.arange(init_len, device=self.device).unsqueeze(0),
|
||||
past_key_values=self._cache,
|
||||
use_cache=True,
|
||||
cache_position=torch.arange(init_len, device=self.device),
|
||||
)
|
||||
self._attn_mask.zero_()
|
||||
self._attn_mask[..., :init_len] = True
|
||||
self._cache_pos.fill_(saved_pos)
|
||||
self.owner = request_id
|
||||
return True
|
||||
except Exception:
|
||||
self._broken = True
|
||||
import logging
|
||||
|
||||
logging.getLogger(__name__).exception(
|
||||
"NemotronVoiceChat native talker: uncond step graph capture failed; stepping eagerly."
|
||||
)
|
||||
return False
|
||||
|
||||
def step(self, embed: torch.Tensor) -> torch.Tensor:
|
||||
"""One captured decode step: embed [1,1,H] -> uncond hidden [1,1,H]."""
|
||||
if int(self._cache_pos.item()) >= self._active_bucket:
|
||||
raise RuntimeError(
|
||||
f"NemotronVoiceChat uncond CFG stream exceeded its StaticCache bucket "
|
||||
f"({self._active_bucket} positions)."
|
||||
)
|
||||
with torch.inference_mode():
|
||||
self._embed_buf.copy_(embed.to(self.dtype))
|
||||
self._buckets[self._active_bucket]["graph"].replay()
|
||||
return self._hidden_buf.clone()
|
||||
|
||||
def release(self, request_id: str) -> None:
|
||||
if self.owner == request_id:
|
||||
self.owner = None
|
||||
Reference in new issue
Block a user