[Performance][MiniCPM-o] Whole-Euler CUDA graphs for Code2Wav with a shared attention arena (#8007)

This commit is contained in:
BeatSeat authored and GitHub committed 2026-09-29 09:14:12 +10:00
1 parent 0173cb3748
commit 3e62bba3cb
15 files changed
+3425 -210

No files matched your search

+15 -9
View File
@@ -7,7 +7,7 @@ session_mode: duplex
# A smaller window serializes first-packet generation for concurrent chat turns.
duplex_session:
# Matches Stage 0/1 max_num_seqs below.
max_sessions: 4
max_sessions: 16
trust_remote_code: true
enable_prefix_caching: false
@@ -27,12 +27,17 @@ connectors:
# HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it.
enable_hift_graph: true
enable_cfm_graph: true
# Whole-Euler CUDA graphs exist for batch 1, 2, 4, ... up to
# micro_batch_size, which defaults to Stage 2 max_num_seqs: all graphs
# share one attention-cache arena of that many requests (~0.6 GiB each
# in fp32, 10 GiB at 16). Larger batches replay micro-batches;
# B > max_graph_batch runs eager.
max_graph_batch: 16
cfm_max_graphs: 32
cfm_graph_bucket_frames: 16
stages:
- stage_id: 0
max_num_seqs: 4
max_num_seqs: 16
# Leave headroom on the shared GPU for Talker KV + concurrent Code2Wav.
gpu_memory_utilization: 0.55
enforce_eager: false
@@ -62,7 +67,7 @@ stages:
repetition_penalty: 1.0
- stage_id: 1
max_num_seqs: 4
max_num_seqs: 16
# Preserve the checkpoint's full-attention Talker prompt while it fits;
# native duplex rolls over to a one-chunk window at context capacity.
# Leave device-level headroom for the colocated HiFi-GAN cuDNN workspace.
@@ -92,7 +97,7 @@ stages:
detokenize: false
- stage_id: 2
max_num_seqs: 4
max_num_seqs: 16
gpu_memory_utilization: 0.18
enforce_eager: true
enable_chunked_prefill: false
@@ -113,10 +118,11 @@ platforms:
stages:
- stage_id: 1
# Full-attention occupancy is bounded by max_num_seqs * max_model_len
# (4 * 4096 = 16384 tokens). Keep the reported CUDA KV token capacity
# above that bound because physical-preemption replay is unsupported.
# The explicit cap leaves device headroom for concurrent Stage 2.
kv_cache_memory_bytes: 2147483648 # 2 GiB
# (16 * 4096 = 65536 tokens). Keep the reported CUDA KV token capacity
# above that bound because physical-preemption replay is unsupported
# (2 GiB holds ~35k tokens). The explicit cap leaves device headroom
# for concurrent Stage 2.
kv_cache_memory_bytes: 4294967296 # 4 GiB
npu:
stages:
-1
View File
@@ -32,7 +32,6 @@ connectors:
enable_hift_graph: true
enable_cfm_graph: true
cfm_max_graphs: 32
cfm_graph_bucket_frames: 16
stages:
- stage_id: 0
-1
View File
@@ -32,7 +32,6 @@ connectors:
enable_hift_graph: true
enable_cfm_graph: true
cfm_max_graphs: 32
cfm_graph_bucket_frames: 16
stages:
- stage_id: 0
@@ -39,7 +39,6 @@ connectors:
enable_hift_graph: true
enable_cfm_graph: true
cfm_max_graphs: 32
cfm_graph_bucket_frames: 16
stages:
- stage_id: 0