58 lines
2.1 KiB
YAML
58 lines
2.1 KiB
YAML
# Opt-in CUDA Model Runner V2 profile for MiniCPM-o 4.5 turn serving: the
|
|
# Thinker (stage 0), Talker (stage 1) and Code2Wav (stage 2) run on MRv2.
|
|
# The Thinker uses native multimodal encoding and tensor-only FULL graphs;
|
|
# downstream stages use the native data plane. These stages support
|
|
# turn-based sessions only (duplex needs the V1 chunk adapter).
|
|
#
|
|
# On MRv2 the Talker keeps its sampled codec id, 16-frame penalty history and
|
|
# EOS state on the device (no per-step D2H wait), and Code2Wav schedules
|
|
# individual chunks instead of holding a slot per utterance, so more than
|
|
# max_num_seqs streams can be live under load.
|
|
base_config: minicpmo_4_5_turn.yaml
|
|
|
|
connectors:
|
|
connector_of_shared_memory:
|
|
extra:
|
|
hift_graph_capture_batch_sizes: [1, 2, 4, 8]
|
|
# TF32 tensor cores for the vocoder's fp32 flow GEMMs: ~25% less GPU
|
|
# time per chunk (batch 1-8). A numerics change: the HiFT input mel moves
|
|
# by ~3e-4 relative L2; dev-short WER/UTMOS unchanged (64 prompts).
|
|
token2wav_allow_tf32: true
|
|
|
|
stages:
|
|
- stage_id: 0
|
|
model_runner: v2
|
|
# llm2tts consumes the completed Thinker payload via the orchestrator.
|
|
# Only Talker -> Code2Wav is a streaming connector edge. Keep the latent
|
|
# rows on the full-payload output channel instead of partitioning them
|
|
# into an async connector that this stage does not have.
|
|
async_chunk: false
|
|
- stage_id: 1
|
|
model_runner: v2
|
|
max_num_seqs: 8
|
|
- stage_id: 2
|
|
model_runner: v2
|
|
max_num_seqs: 8
|
|
async_scheduling: false
|
|
env:
|
|
# Decode waiting first chunks in their own step, but only while every
|
|
# stream with a later chunk ready still holds 0.5 s of unplayed audio,
|
|
# so first packets do not come at the cost of playback underruns.
|
|
VLLM_OMNI_CODEC_FIRST_CHUNK_EXPRESS: "1"
|
|
VLLM_OMNI_CODEC_FIRST_CHUNK_EXPRESS_SLACK_S: "0.5"
|
|
|
|
platforms:
|
|
cuda:
|
|
stages:
|
|
- stage_id: 1
|
|
# Reserve enough KV for eight concurrent 4096-token Talker requests.
|
|
kv_cache_memory_bytes: 2147483648
|
|
npu:
|
|
model_runner: v1
|
|
xpu:
|
|
model_runner: v1
|
|
rocm:
|
|
model_runner: v1
|
|
musa:
|
|
model_runner: v1
|