diff --git a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py index 96f8c799c..8f7eeb542 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py +++ b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py @@ -10,6 +10,7 @@ reducing kernel launch overhead during inference. import torch from torch.cuda import CUDAGraph from vllm.logger import init_logger +from vllm.platforms import current_platform logger = init_logger(__name__) @@ -129,7 +130,7 @@ class CUDAGraphDecoderWrapper: graph = CUDAGraph() with torch.no_grad(): - with torch.cuda.graph(graph): + with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()): static_output = self.decoder(static_input) self.graphs[size] = graph diff --git a/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py b/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py index 395c0d113..a4d58df5b 100644 --- a/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py +++ b/vllm_omni/model_executor/models/voxtral_tts/cuda_graph_acoustic_transformer_wrapper.py @@ -11,6 +11,7 @@ eliminating kernel launch overhead on every decode step. import torch from torch.cuda import CUDAGraph from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import ( AudioSpecialTokens, @@ -196,7 +197,7 @@ class CUDAGraphAcousticTransformerWrapper: graph = CUDAGraph() with torch.no_grad(): - with torch.cuda.graph(graph): + with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()): static_fake_eos, static_audio_codes = self._forward_cudagraph_compatible( static_input, noise=static_noise )