[Perf][Qwen3-TTS][Voxtral-TTS] Share CUDA graph memory pool across decoder capture sizes (#2386)
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
2c6c07c4f6
commit
149b9f179a
2 files changed
+4
-2
No files matched your search
@@ -10,6 +10,7 @@ reducing kernel launch overhead during inference.
|
||||
import torch
|
||||
from torch.cuda import CUDAGraph
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -129,7 +130,7 @@ class CUDAGraphDecoderWrapper:
|
||||
|
||||
graph = CUDAGraph()
|
||||
with torch.no_grad():
|
||||
with torch.cuda.graph(graph):
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
static_output = self.decoder(static_input)
|
||||
|
||||
self.graphs[size] = graph
|
||||
|
||||
+2
-1
@@ -11,6 +11,7 @@ eliminating kernel launch overhead on every decode step.
|
||||
import torch
|
||||
from torch.cuda import CUDAGraph
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from vllm_omni.model_executor.models.voxtral_tts.voxtral_tts_audio_generation import (
|
||||
AudioSpecialTokens,
|
||||
@@ -196,7 +197,7 @@ class CUDAGraphAcousticTransformerWrapper:
|
||||
|
||||
graph = CUDAGraph()
|
||||
with torch.no_grad():
|
||||
with torch.cuda.graph(graph):
|
||||
with torch.cuda.graph(graph, pool=current_platform.get_global_graph_pool()):
|
||||
static_fake_eos, static_audio_codes = self._forward_cudagraph_compatible(
|
||||
static_input, noise=static_noise
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user