Files
vllm-omni/vllm_omni/deploy/minicpmo_4_5_turn_mrv2.yaml
T

58 lines
2.1 KiB
YAML

# Opt-in CUDA Model Runner V2 profile for MiniCPM-o 4.5 turn serving: the
# Thinker (stage 0), Talker (stage 1) and Code2Wav (stage 2) run on MRv2.
# The Thinker uses native multimodal encoding and tensor-only FULL graphs;
# downstream stages use the native data plane. These stages support
# turn-based sessions only (duplex needs the V1 chunk adapter).
#
# On MRv2 the Talker keeps its sampled codec id, 16-frame penalty history and
# EOS state on the device (no per-step D2H wait), and Code2Wav schedules
# individual chunks instead of holding a slot per utterance, so more than
# max_num_seqs streams can be live under load.
base_config: minicpmo_4_5_turn.yaml
connectors:
connector_of_shared_memory:
extra:
hift_graph_capture_batch_sizes: [1, 2, 4, 8]
# TF32 tensor cores for the vocoder's fp32 flow GEMMs: ~25% less GPU
# time per chunk (batch 1-8). A numerics change: the HiFT input mel moves
# by ~3e-4 relative L2; dev-short WER/UTMOS unchanged (64 prompts).
token2wav_allow_tf32: true
stages:
- stage_id: 0
model_runner: v2
# llm2tts consumes the completed Thinker payload via the orchestrator.
# Only Talker -> Code2Wav is a streaming connector edge. Keep the latent
# rows on the full-payload output channel instead of partitioning them
# into an async connector that this stage does not have.
async_chunk: false
- stage_id: 1
model_runner: v2
max_num_seqs: 8
- stage_id: 2
model_runner: v2
max_num_seqs: 8
async_scheduling: false
env:
# Decode waiting first chunks in their own step, but only while every
# stream with a later chunk ready still holds 0.5 s of unplayed audio,
# so first packets do not come at the cost of playback underruns.
VLLM_OMNI_CODEC_FIRST_CHUNK_EXPRESS: "1"
VLLM_OMNI_CODEC_FIRST_CHUNK_EXPRESS_SLACK_S: "0.5"
platforms:
cuda:
stages:
- stage_id: 1
# Reserve enough KV for eight concurrent 4096-token Talker requests.
kv_cache_memory_bytes: 2147483648
npu:
model_runner: v1
xpu:
model_runner: v1
rocm:
model_runner: v1
musa:
model_runner: v1