From 149b9f179a5f85082136e15ba065756022debc69 Mon Sep 17 00:00:00 2001 From: Nick Cao Date: Wed, 8 Apr 2026 20:40:17 -0400 Subject: [PATCH] [Perf][Qwen3-TTS][Voxtral-TTS] Share CUDA graph memory pool across decoder capture sizes (#2386) Signed-off-by: Nick Cao Co-authored-by: Claude --- .../models/qwen3_tts/cuda_graph_decoder_wrapper.py | 3 ++- .../voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py index 96f8c799c..8f7eeb542 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py +++ b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py @@ -10,6 +10,7 @@ reducing kernel launch overhead during inference. import torch from torch.cuda import CUDAGraph from vllm.logger import init_logger +from vllm.platforms import current_platform logger = init_logger(__name__) @@ -129,7 +130,7 @@ class CUDAGraphDecoderWrapper: graph = CUDAGraph() with torch.no_grad(): - with torch.cuda.graph(graph): + with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()): static_output = self.decoder(static_input) self.graphs[size] = graph diff --git a/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py b/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py index 395c0d113..a4d58df5b 100644 --- a/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py +++ b/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py @@ -11,6 +11,7 @@ eliminating kernel launch overhead on every decode step. import torch from torch.cuda import CUDAGraph from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import ( AudioSpecialTokens, @@ -196,7 +197,7 @@ class CUDAGraphAcousticTransformerWrapper: graph = CUDAGraph() with torch.no_grad(): - with torch.cuda.graph(graph): + with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()): static_fake_eos, static_audio_codes = self._forward_cudagraph_compatible( static_input, noise=static_noise )