[Model] Optimize CosyVoice3 packed streaming with bounded HiFT graphs
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 parent
0e6791e9c4
commit
b3c25a608d
9 files changed
+205
-4
No files matched your search
@@ -966,7 +966,20 @@ class CausalHiFTGenerator(HiFTGenerator):
|
||||
self.conv_pre_look_right = conv_pre_look_right
|
||||
self.f0_predictor = f0_predictor
|
||||
|
||||
def enable_decode_graphs(self) -> None:
|
||||
"""Opt into exact-shape CUDA replay after frozen weights are loaded."""
|
||||
from .hift_graph import HiFTDecodeGraphs
|
||||
|
||||
self.enable_cached_istft()
|
||||
self._decode_graphs = HiFTDecodeGraphs(self._decode_eager)
|
||||
|
||||
def decode(self, x: torch.Tensor, s: torch.Tensor = torch.zeros(1, 1, 0), finalize: bool = True) -> torch.Tensor:
|
||||
runner = getattr(self, "_decode_graphs", None)
|
||||
if runner is not None:
|
||||
return runner.run(x, s, finalize)
|
||||
return self._decode_eager(x, s, finalize=finalize)
|
||||
|
||||
def _decode_eager(self, x: torch.Tensor, s: torch.Tensor, finalize: bool = True) -> torch.Tensor:
|
||||
s_stft_real, s_stft_imag = self._stft(s.squeeze(1))
|
||||
if finalize is True:
|
||||
x = self.conv_pre(x)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Bounded CUDA graph replay for HiFT's deterministic waveform decoder.
|
||||
|
||||
F0, sine/noise generation, phase carry, and per-request streaming state remain
|
||||
outside the graph. Each replay consumes current mel and source tensors and
|
||||
returns an owned waveform, because multiple requests can reuse one shape.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass
|
||||
class _DecodeGraph:
|
||||
graph: torch.cuda.CUDAGraph
|
||||
mel: torch.Tensor
|
||||
source: torch.Tensor
|
||||
output: torch.Tensor
|
||||
|
||||
|
||||
class HiFTDecodeGraphs:
|
||||
"""Capture recurrent exact shapes, with at most 32 resident graphs.
|
||||
|
||||
``decode`` must have warmed its ISTFT envelope before capture and remain
|
||||
deterministic for its mel, source and finalize arguments. Uncommon shapes,
|
||||
CPU execution and nested CUDA capture use that same eager implementation.
|
||||
"""
|
||||
|
||||
def __init__(self, decode: Callable[..., torch.Tensor], max_graphs: int = 32) -> None:
|
||||
self.decode = decode
|
||||
self.max_graphs = max_graphs
|
||||
self.graphs: dict[tuple, _DecodeGraph] = {}
|
||||
self.seen: OrderedDict[tuple, int] = OrderedDict()
|
||||
self.pool = None
|
||||
|
||||
@torch.inference_mode()
|
||||
def run(self, mel: torch.Tensor, source: torch.Tensor, finalize: bool) -> torch.Tensor:
|
||||
"""Replay a recurring layout without aliasing another request's output."""
|
||||
if not mel.is_cuda or not source.is_cuda or torch.cuda.is_current_stream_capturing():
|
||||
return self.decode(mel, source, finalize=finalize)
|
||||
autocast = torch.is_autocast_enabled("cuda")
|
||||
key = (
|
||||
mel.shape,
|
||||
source.shape,
|
||||
mel.device,
|
||||
mel.dtype,
|
||||
source.dtype,
|
||||
finalize,
|
||||
torch.backends.cudnn.allow_tf32,
|
||||
torch.backends.cuda.matmul.allow_tf32,
|
||||
autocast,
|
||||
torch.get_autocast_dtype("cuda") if autocast else None,
|
||||
)
|
||||
state = self.graphs.get(key)
|
||||
if state is None:
|
||||
if len(self.graphs) >= self.max_graphs:
|
||||
return self.decode(mel, source, finalize=finalize)
|
||||
count = self.seen.pop(key, 0) + 1
|
||||
self.seen[key] = count
|
||||
if len(self.seen) > 128:
|
||||
self.seen.popitem(last=False)
|
||||
if count < 3:
|
||||
return self.decode(mel, source, finalize=finalize)
|
||||
static_mel, static_source = mel.clone(), source.clone()
|
||||
stream = torch.cuda.Stream(device=mel.device)
|
||||
stream.wait_stream(torch.cuda.current_stream(mel.device))
|
||||
with torch.cuda.stream(stream):
|
||||
self.decode(static_mel, static_source, finalize=finalize)
|
||||
torch.cuda.current_stream(mel.device).wait_stream(stream)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, pool=self.pool):
|
||||
output = self.decode(static_mel, static_source, finalize=finalize)
|
||||
if self.pool is None:
|
||||
self.pool = graph.pool()
|
||||
state = _DecodeGraph(graph, static_mel, static_source, output)
|
||||
self.graphs[key] = state
|
||||
state.mel.copy_(mel)
|
||||
state.source.copy_(source)
|
||||
state.graph.replay()
|
||||
# A later request may replay this shape before the caller consumes it.
|
||||
return state.output.clone()
|
||||
@@ -170,7 +170,7 @@ class PackedDiT:
|
||||
t,
|
||||
rows,
|
||||
attention,
|
||||
max_seqlen_q=attention.page_table.shape[1],
|
||||
max_seqlen_q=attention.max_seqlen_q,
|
||||
block_modulation=block_modulation,
|
||||
final_modulation=final_modulation,
|
||||
rope=rope,
|
||||
@@ -771,7 +771,9 @@ class FlowGraphRunner:
|
||||
attention.heads, attention.head_dim = self.heads, self.head_dim
|
||||
attention.cache_seqlens, attention.cu_seqlens_q = state.cache_seqlens, state.cu_seqlens_q
|
||||
attention.page_table = state.page_table
|
||||
attention.max_seqlen_q = state.page_table.shape[1]
|
||||
# Streaming queries cover one training chunk, even when their KV
|
||||
# prefix and the packed graph allocation contain thousands of frames.
|
||||
attention.max_seqlen_q = estimator.dit.static_chunk_size if key[0] else state.page_table.shape[1]
|
||||
mark_packed_compile_metadata(rows, attention)
|
||||
mu_cfg = torch.cat((state.mu, torch.zeros_like(state.mu)), dim=1)
|
||||
cond_cfg = torch.cat((state.cond, torch.zeros_like(state.cond)), dim=1)
|
||||
|
||||
@@ -769,4 +769,11 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
if f0_folded == 0:
|
||||
logger.warning("HiFT F0 predictor had no weight-norm layers to fold; check config drift")
|
||||
self.hift.eval()
|
||||
if device.type == "cuda" and cosyvoice3_packed_streaming_enabled():
|
||||
if os.getenv("COSYVOICE3_CACHED_ISTFT", "0") == "1":
|
||||
self.hift.enable_cached_istft()
|
||||
logger.info("CosyVoice3 streaming HiFT: cached ISTFT enabled")
|
||||
if os.getenv("COSYVOICE3_HIFT_GRAPH", "0") == "1":
|
||||
self.hift.enable_decode_graphs()
|
||||
logger.info("CosyVoice3 streaming HiFT: decode CUDA graphs enabled")
|
||||
logger.info(f"Loaded hift weights from {hift_path}")
|
||||
Reference in new issue
Block a user