[Performance] Capture MOSS-TTS codec streaming decode with NPUGraph (#7280)

Signed-off-by: Wallbreazzz <110282866+Wallbreazzz@users.noreply.github.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-21 06:38:23 +08:00
1 parent e36babd48f
commit ae3880f0e3
4 files changed
+460 -10

No files matched your search

@@ -63,7 +63,7 @@ class _MossCodecStreamSession:
self._cudagraph_wrapper: CUDAGraphStreamingDecoderWrapper | None = None
batch_sizes = sorted({int(size) for size in (graph_batch_sizes or []) if 0 < int(size) <= self._state_capacity})
frame_sizes = sorted({int(size) for size in (graph_frame_sizes or []) if int(size) > 0})
scratch_capacity = max(batch_sizes, default=0) if self._device.type == "cuda" else 0
scratch_capacity = max(batch_sizes, default=0) if self._device.type in ("cuda", "npu") else 0
self._total_state_capacity = self._state_capacity + scratch_capacity
self._state_slot_ids = torch.arange(
self._total_state_capacity,
@@ -76,15 +76,29 @@ class _MossCodecStreamSession:
raise RuntimeError("The streaming codec does not implement a decoder state pool.")
with torch.no_grad():
initialize_state_pool(self._state_capacity, scratch_capacity)
if batch_sizes and frame_sizes and self._device.type == "cuda":
self._cudagraph_wrapper = CUDAGraphStreamingDecoderWrapper(
codec,
state_capacity=self._state_capacity,
batch_sizes=batch_sizes,
frame_sizes=frame_sizes,
num_quantizers=self._n_vq,
vllm_config=vllm_config,
)
if batch_sizes and frame_sizes and self._device.type in ("cuda", "npu"):
if self._device.type == "npu":
from vllm_omni.platforms.npu.models.moss_tts_streaming_decode_wrapper import (
NPUGraphStreamingDecoderWrapper,
)
self._cudagraph_wrapper = NPUGraphStreamingDecoderWrapper(
codec,
state_capacity=self._state_capacity,
batch_sizes=batch_sizes,
frame_sizes=frame_sizes,
num_quantizers=self._n_vq,
vllm_config=vllm_config,
)
else:
self._cudagraph_wrapper = CUDAGraphStreamingDecoderWrapper(
codec,
state_capacity=self._state_capacity,
batch_sizes=batch_sizes,
frame_sizes=frame_sizes,
num_quantizers=self._n_vq,
vllm_config=vllm_config,
)
self._cudagraph_wrapper.warmup(self._device)
self.reset_slots(list(range(self._state_capacity + scratch_capacity)))
if not self._cudagraph_wrapper.is_ready: