Nick CaoandCodex
b81f2c5d4b
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Codex <noreply@openai.com>
105 lines
4.0 KiB
YAML
105 lines
4.0 KiB
YAML
# NemotronVoiceChat-11B STREAMING speech-to-speech deployment (async-chunk).
|
|
#
|
|
# Same 3 stages as nemotron_labs_voicechat.yaml, but the stages overlap:
|
|
# the thinker streams its frame-locked text timeline to the talker per decode
|
|
# step (cumulative ids.all payloads), and the talker streams prompt-trimmed
|
|
# cumulative code stacks to code2wav every `codec_chunk_frames` frames. The
|
|
# codec re-decodes the full prefix per chunk and emits only the new samples
|
|
# (NeMo decode_one_audio_step semantics), so the concatenated stream is
|
|
# seam-free. First audio arrives after ~codec_chunk_frames thinker steps
|
|
# instead of after the whole pipeline completes.
|
|
#
|
|
# The default is the FAST profile (TTFP 1.47 s / RTF 0.33 on the 196-frame
|
|
# fixture, 1x H100): bf16 + CUDA-graph thinker, CUDA-graph EAR-TTS talker
|
|
# step, and incremental codec decode. Every fast setting is marked with a
|
|
# "PARITY:" comment; apply those edits for the eager fp32 reference
|
|
# execution. Streamed audio is validated by ASR/WER, not bit parity (prefix
|
|
# decodes are not bit-identical to the one-shot decode of the final stack).
|
|
pipeline: nemotron_voicechat
|
|
async_chunk: true
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
name: SharedMemoryConnector
|
|
extra:
|
|
# Streaming cadence for talker -> code2wav: emit a chunk once this many
|
|
# NEW frames (80 ms each) have accumulated (5 frames = 0.4 s of audio).
|
|
# Small chunks are cheap because the incremental codec cache decodes
|
|
# only each chunk's new frames; on the 196-frame fixture 5-frame chunks
|
|
# cut TTFP by ~0.4 s vs 13-frame chunks at unchanged total time.
|
|
# PARITY: 13 (~1 s per chunk) was the reference cadence.
|
|
codec_chunk_frames: 5
|
|
|
|
stages:
|
|
- stage_id: 0 # thinker: speech -> frame-locked text (final text output)
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.62
|
|
# PARITY: set enforce_eager: true and dtype: float32 for the fp32
|
|
# reference thinker (greedy text can legitimately diverge under bf16).
|
|
enforce_eager: false
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
max_num_batched_tokens: 8192
|
|
max_model_len: 8192
|
|
dtype: bfloat16
|
|
mamba_ssm_cache_dtype: float32
|
|
devices: "0"
|
|
output_connectors:
|
|
to_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
ignore_eos: true
|
|
seed: 0
|
|
|
|
- stage_id: 1 # talker: text timeline -> 31-quantizer code stacks
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.12
|
|
enforce_eager: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
skip_tokenizer_init: true
|
|
max_num_batched_tokens: 4096
|
|
max_model_len: 16384
|
|
dtype: float32
|
|
devices: "0"
|
|
hf_overrides:
|
|
# Capture the vendored per-frame EAR-TTS step (backbone + MoG, CFG
|
|
# batch layout) into CUDA graphs (~3x per-frame). PARITY: delete this
|
|
# block for the eager reference step (bit-exact vs NeMo).
|
|
use_talker_cuda_graphs: true
|
|
talker_graph_max_cache_len: 4096
|
|
input_connectors:
|
|
from_stage_0: connector_of_shared_memory
|
|
output_connectors:
|
|
to_stage_2: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
max_tokens: 16383
|
|
stop_token_ids: [1]
|
|
detokenize: false
|
|
seed: 0
|
|
|
|
- stage_id: 2 # code2wav: codes -> 22.05 kHz PCM (fp32 codec)
|
|
max_num_seqs: 1
|
|
gpu_memory_utilization: 0.06
|
|
enforce_eager: true
|
|
enable_prefix_caching: false
|
|
async_scheduling: false
|
|
skip_tokenizer_init: true
|
|
max_num_batched_tokens: 65536
|
|
max_model_len: 65536
|
|
devices: "0"
|
|
hf_overrides:
|
|
# Decode only each chunk's NEW frames with a per-request causal-conv
|
|
# cache (NVIDIA's S2S_USE_CODEC_CACHE path) instead of re-decoding the
|
|
# cumulative prefix per chunk (O(T^2) in the reply length).
|
|
# PARITY: delete this block for the prefix re-decode reference.
|
|
use_incremental_codec_cache: true
|
|
input_connectors:
|
|
from_stage_1: connector_of_shared_memory
|
|
default_sampling_params:
|
|
temperature: 0.0
|
|
max_tokens: 1
|
|
detokenize: false
|
|
seed: 0
|