[Perf][Qwen3-TTS][Voxtral-TTS] Share CUDA graph memory pool across decoder capture sizes (#2386)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Nick CaoandClaude authored and GitHub committed 2026-04-08 20:40:17 -04:00
1 parent 2c6c07c4f6
commit 149b9f179a
2 files changed
+4 -2

No files matched your search

@@ -10,6 +10,7 @@ reducing kernel launch overhead during inference.
import torch
from torch.cuda import CUDAGraph
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -129,7 +130,7 @@ class CUDAGraphDecoderWrapper:
graph = CUDAGraph()
with torch.no_grad():
with torch.cuda.graph(graph):
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
static_output = self.decoder(static_input)
self.graphs[size] = graph
@@ -11,6 +11,7 @@ eliminating kernel launch overhead on every decode step.
import torch
from torch.cuda import CUDAGraph
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import (
AudioSpecialTokens,
@@ -196,7 +197,7 @@ class CUDAGraphAcousticTransformerWrapper:
graph = CUDAGraph()
with torch.no_grad():
with torch.cuda.graph(graph):
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
static_fake_eos, static_audio_codes = self._forward_cudagraph_compatible(
static_input, noise=static_noise
)