# NemotronVoiceChat-11B offline speech-to-speech deployment. # 3 stages on one GPU: thinker (NemotronH AR) -> talker (EAR-TTS AR) -> code2wav # (RVQ-VAE decode). thinker->talker is token-path only; talker->code2wav ships # the [frames, 31] code stack via the shared-memory connector. # # The default is the FAST profile (2.8x on the acceptance fixture, 10.3 s -> # 3.65 s / RTF 0.23 on 1x H100): a bf16 + CUDA-graph thinker and the talker's # Gemma3 backbone running as a native vLLM model (PagedAttention KV + vLLM # decode CUDA graphs, MoG sampling in one captured graph) — the way NVIDIA's # own deployment serves EAR-TTS. Every fast setting is marked with a # "PARITY:" comment; apply those edits to bit-match the eager fp32 NeMo # reference (196/196 token parity, md5-anchored WAV on the acceptance # fixture). # # Notes on the native talker: # * classifier-free guidance is OFF by default (a paired ASR study over 4 # fixtures at this checkpoint's guidance_scale=0.2 found no systematic # difference). Opt in via hf_overrides.native_talker_guidance below. # * the talker's engine prompt covers the speaker-prompt prefill # (hf_overrides.talker_init_len; 37 for the shipped "Aria" speaker). The # talker validates the value and reports the right one on mismatch. pipeline: nemotron_voicechat async_chunk: false connectors: connector_of_shared_memory: name: SharedMemoryConnector extra: shm_threshold_bytes: 65536 stages: - stage_id: 0 # thinker: speech -> frame-locked text (final text output) max_num_seqs: 1 gpu_memory_utilization: 0.62 # PARITY: set enforce_eager: true and dtype: float32. bf16 + CUDA graphs # is ~4x faster per frame and halves the weights (~43 GB -> ~21 GB); its # greedy text can legitimately diverge from the fp32 reference (measured # within one word on the acceptance fixture, ASR-clean). enforce_eager: false dtype: bfloat16 # NemotronH mamba SSM state in fp32 (matches NVIDIA's own vLLM deployment # of this model; bf16 SSM state accumulates drift over long sequences). mamba_ssm_cache_dtype: float32 enable_prefix_caching: false async_scheduling: false max_num_batched_tokens: 8192 max_model_len: 8192 devices: "0" default_sampling_params: # Greedy, frame-locked: the example sets max_tokens to the # exact acoustic frame count per request; text EOS never stops generation. temperature: 0.0 ignore_eos: true seed: 0 - stage_id: 1 # talker: text timeline -> 31-quantizer code stacks max_num_seqs: 1 gpu_memory_utilization: 0.12 # PARITY: set enforce_eager: true and delete the hf_overrides block. The # vendored eager fp32 EAR-TTS step is the NeMo-bit-exact path; the native # backbone replays the same math on PagedAttention with fresh # CUDA-graph-safe RNG draws (audio equivalent, not bit-identical). enforce_eager: false enable_prefix_caching: false async_scheduling: false skip_tokenizer_init: true max_num_batched_tokens: 4096 max_model_len: 16384 # fp32 like the NeMo reference: the EAR-TTS MoG/codec-embedding path mixes # fp32 buffers into the forward, and 1B params in fp32 are cheap. dtype: float32 devices: "0" hf_overrides: use_native_talker: true talker_init_len: 37 # Classifier-free guidance (checkpoint guidance_scale=0.2): mirrors the # stream on the vendored HF backbone (captured StaticCache step) and # blends per NeMo's generate_step math. 196-frame fixture: 3.65 s -> # 5.3 s (RTF 0.23 -> 0.34). Requires gpu_memory_utilization ~0.18 on # this stage (the vendored backbone copy stays loaded). Off by default: # measured ASR-equivalent at this checkpoint's scale. # native_talker_guidance: true output_connectors: to_stage_2: connector_of_shared_memory default_sampling_params: # Placeholder-token loop; the stage emits stop token 1 after the last # timeline position. 16347 = max_model_len (16384) minus the native # talker's 37-token speaker-prompt prefill: the request carries the # placeholder prompt tokens, and prompt + max_tokens must fit # max_model_len. temperature: 0.0 max_tokens: 16347 stop_token_ids: [1] detokenize: false seed: 0 - stage_id: 2 # code2wav: codes -> 22.05 kHz PCM (fp32 codec) max_num_seqs: 1 gpu_memory_utilization: 0.06 enforce_eager: true enable_prefix_caching: false async_scheduling: false skip_tokenizer_init: true max_num_batched_tokens: 65536 max_model_len: 65536 devices: "0" input_connectors: from_stage_1: connector_of_shared_memory default_sampling_params: temperature: 0.0 max_tokens: 1 detokenize: false seed: 0