Nick CaoandCodex
b81f2c5d4b
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Codex <noreply@openai.com>
112 lines
4.7 KiB
YAML
112 lines
4.7 KiB
YAML
# NemotronVoiceChat-11B offline speech-to-speech deployment.
|
|
# 3 stages on one GPU: thinker (NemotronH AR) -> talker (EAR-TTS AR) -> code2wav
|
|
# (RVQ-VAE decode). thinker->talker is token-path only; talker->code2wav ships
|
|
# the [frames, 31] code stack via the shared-memory connector.
|
|
#
|
|
# The default is the FAST profile (2.8x on the acceptance fixture, 10.3 s ->
|
|
# 3.65 s / RTF 0.23 on 1x H100): a bf16 + CUDA-graph thinker and the talker's
|
|
# Gemma3 backbone running as a native vLLM model (PagedAttention KV + vLLM
|
|
# decode CUDA graphs, MoG sampling in one captured graph) — the way NVIDIA's
|
|
# own deployment serves EAR-TTS. Every fast setting is marked with a
|
|
# "PARITY:" comment; apply those edits to bit-match the eager fp32 NeMo
|
|
# reference (196/196 token parity, md5-anchored WAV on the acceptance
|
|
# fixture).
|
|
#
|
|
# Notes on the native talker:
|
|
# * classifier-free guidance is OFF by default (a paired ASR study over 4
|
|
# fixtures at this checkpoint's guidance_scale=0.2 found no systematic
|
|
# difference). Opt in via hf_overrides.native_talker_guidance below.
|
|
# * the talker's engine prompt covers the speaker-prompt prefill
|
|
# (hf_overrides.talker_init_len; 37 for the shipped "Aria" speaker). The
|
|
# talker validates the value and reports the right one on mismatch.
|
|
pipeline: nemotron_voicechat
|
|
async_chunk: false
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
|
|
stages:
|
|
- stage_id: 0 # thinker: speech -> frame-locked text (final text output)
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.62
|
|
# PARITY: set enforce_eager: true and dtype: float32. bf16 + CUDA graphs
|
|
# is ~4x faster per frame and halves the weights (~43 GB -> ~21 GB); its
|
|
# greedy text can legitimately diverge from the fp32 reference (measured
|
|
# within one word on the acceptance fixture, ASR-clean).
|
|
enforce_eager: false
|
|
dtype: bfloat16
|
|
# NemotronH mamba SSM state in fp32 (matches NVIDIA's own vLLM deployment
|
|
# of this model; bf16 SSM state accumulates drift over long sequences).
|
|
mamba_ssm_cache_dtype: float32
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
max_num_batched_tokens: 8192
|
|
max_model_len: 8192
|
|
devices: "0"
|
|
default_sampling_params:
|
|
# Greedy, frame-locked: the example sets max_tokens to the
|
|
# exact acoustic frame count per request; text EOS never stops generation.
|
|
temperature: 0.0
|
|
ignore_eos: true
|
|
seed: 0
|
|
|
|
- stage_id: 1 # talker: text timeline -> 31-quantizer code stacks
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.12
|
|
# PARITY: set enforce_eager: true and delete the hf_overrides block. The
|
|
# vendored eager fp32 EAR-TTS step is the NeMo-bit-exact path; the native
|
|
# backbone replays the same math on PagedAttention with fresh
|
|
# CUDA-graph-safe RNG draws (audio equivalent, not bit-identical).
|
|
enforce_eager: false
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
skip_tokenizer_init: true
|
|
max_num_batched_tokens: 4096
|
|
max_model_len: 16384
|
|
# fp32 like the NeMo reference: the EAR-TTS MoG/codec-embedding path mixes
|
|
# fp32 buffers into the forward, and 1B params in fp32 are cheap.
|
|
dtype: float32
|
|
devices: "0"
|
|
hf_overrides:
|
|
use_native_talker: true
|
|
talker_init_len: 37
|
|
# Classifier-free guidance (checkpoint guidance_scale=0.2): mirrors the
|
|
# stream on the vendored HF backbone (captured StaticCache step) and
|
|
# blends per NeMo's generate_step math. 196-frame fixture: 3.65 s ->
|
|
# 5.3 s (RTF 0.23 -> 0.34). Requires gpu_memory_utilization ~0.18 on
|
|
# this stage (the vendored backbone copy stays loaded). Off by default:
|
|
# measured ASR-equivalent at this checkpoint's scale.
|
|
# native_talker_guidance: true
|
|
output_connectors:
|
|
to_stage_2: connector_of_shared_memory
|
|
default_sampling_params:
|
|
# Placeholder-token loop; the stage emits stop token 1 after the last
|
|
# timeline position. 16347 = max_model_len (16384) minus the native
|
|
# talker's 37-token speaker-prompt prefill: the request carries the
|
|
# placeholder prompt tokens, and prompt + max_tokens must fit
|
|
# max_model_len.
|
|
temperature: 0.0
|
|
max_tokens: 16347
|
|
stop_token_ids: [1]
|
|
detokenize: false
|
|
seed: 0
|
|
|
|
- stage_id: 2 # code2wav: codes -> 22.05 kHz PCM (fp32 codec)
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.06
|
|
enforce_eager: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
skip_tokenizer_init: true
|
|
max_num_batched_tokens: 65536
|
|
max_model_len: 65536
|
|
devices: "0"
|
|
input_connectors:
|
|
from_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
max_tokens: 1
|
|
detokenize: false
|
|
seed: 0
|