[Perf][TTS] Optimize cosyvoice TTFP and throughput using TensorRT (#4168)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuekai ZhangandClaude Opus 4.8 authored and GitHub committed 2026-06-10 16:42:34 +08:00
1 parent 57c8f3abfc
commit f1bf544208
11 files changed
+786 -87

No files matched your search

@@ -84,6 +84,11 @@ def _make_code2wav_model(
token_mel_ratio=2 if with_stride_cfg else 0,
)
model.code2wav = _DummyCode2Wav(vocab_size=4, num_samples=num_samples, outputs=outputs)
# Short-circuit the lazy TensorRT estimator swap: these tests exercise the
# forward audio logic, not the TRT path. On a GPU CI runner the swap would
# otherwise run and dereference ``self.model_dir`` (only set in __init__,
# which this fixture bypasses via object.__new__).
model._code2wav_trt_done = True
model.source_cache_len = 4
model.speech_window = torch.hamming_window(8, periodic=False)
model._stream_audio_cache_by_req = {}
+1
View File
@@ -128,6 +128,7 @@ class EmbeddingsStruct(_StructBase):
voice: torch.Tensor | None = None
speech_feat: torch.Tensor | None = None
speech_token: torch.Tensor | None = None
speech_token_len: torch.Tensor | None = None
embedding: torch.Tensor | None = None
thinker_reply: torch.Tensor | None = None
+20 -7
View File
@@ -7,9 +7,22 @@
# references stay in the yaml unconditionally; the runtime only activates
# them when ``async_chunk: true``.
#
# enforce_eager=true everywhere:
# * Stage 0 (talker) — cudagraph not verified on this checkpoint.
# * Stage 1 (code2wav) — CUDA graphs don't work with dynamic conv shapes.
# enforce_eager:
# * Stage 0 (talker) — enforce_eager: false (CUDA graphs enabled).
# * Stage 1 (code2wav) — enforce_eager: true (CUDA graphs don't work with
# dynamic conv shapes).
#
# Streaming continuity is GPU-class sensitive — TUNE PER GPU.
# The defaults below (max_num_seqs: 8, codec_chunk_frames: 15) are tuned for
# H100, where streaming continuity at concurrency 4 stays ~100%. They push
# more audio per scheduler step through the SharedMemoryConnector than a
# slower GPU can keep up with: on H20-3e the same config regressed continuity
# to ~78% at c=4 (vs ~86% with smaller batch/chunk), because code2wav can't
# produce mel chunks fast enough to feed the streaming output → buffer
# underruns → audible gaps.
# On a slower GPU class, lower max_num_seqs (e.g. 4) and/or codec_chunk_frames
# first. Raising connector_get_sleep_s also relaxes the consumer's polling
# cadence, but trades against TTFP — keep it as a last resort.
async_chunk: true
dtype: float32
@@ -21,16 +34,16 @@ connectors:
connector_get_sleep_s: 0.01
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
codec_chunk_frames: 25
codec_chunk_frames: 15
codec_left_context_frames: 25
codec_vocab_size: 6561
stages:
- stage_id: 0
max_num_batched_tokens: 32768
max_num_seqs: 1
max_num_seqs: 8
gpu_memory_utilization: 0.4
enforce_eager: true
enforce_eager: false
trust_remote_code: true
enable_prefix_caching: false
devices: "0"
@@ -49,7 +62,7 @@ stages:
- stage_id: 1
max_num_batched_tokens: 32768
max_num_seqs: 1
max_num_seqs: 8
gpu_memory_utilization: 0.2
enforce_eager: true
trust_remote_code: true
+19 -9
View File
@@ -410,6 +410,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
)
self._fish_speech_tokenizer = None
self._covo_audio_tokenizer = None
# Cached per process: the CosyVoice3 Qwen tokenizer + resolved model
# path used for dynamic-token sizing. Without this, every request
# re-ran snapshot_download + reloaded the tokenizer (~100 ms on the
# TTFP critical path) in _apply_cosyvoice3_dynamic_tokens.
self._cosyvoice3_tokenizer = None
self._is_cosyvoice3 = (
self._tts_stage is not None
@@ -2658,16 +2663,21 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
sampling_params_list = copy.deepcopy(sampling_params_list)
hf_cfg = self.model_config.hf_config
model_path = self.engine_client.model_config.model
if not os.path.isdir(model_path):
from huggingface_hub import snapshot_download
# Build the Qwen tokenizer once per process (resolving the model dir via
# snapshot_download at most once) and reuse it across requests.
tokenizer = self._cosyvoice3_tokenizer
if tokenizer is None:
model_path = self.engine_client.model_config.model
if not os.path.isdir(model_path):
from huggingface_hub import snapshot_download
model_path = snapshot_download(model_path)
tokenizer = get_qwen_tokenizer(
token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path),
skip_special_tokens=hf_cfg.skip_special_tokens,
version=hf_cfg.version,
)
model_path = snapshot_download(model_path)
tokenizer = get_qwen_tokenizer(
token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path),
skip_special_tokens=hf_cfg.skip_special_tokens,
version=hf_cfg.version,
)
self._cosyvoice3_tokenizer = tokenizer
_, text_token_len = extract_text_token(
request.input,
tokenizer,
@@ -141,24 +141,37 @@ class ConditionalCFM(BASECFM):
if isinstance(self.estimator, torch.nn.Module):
return self.estimator(x, mask, mu, t, spks, cond)
else:
# TensorRT estimator: bind raw device pointers. The flow runs in
# fp32 but the engine may have fp16 I/O (strongly-typed fp16 engine),
# so cast inputs/output to the engine's dtype at the boundary. Keep
# references to the cast buffers alive until execute completes (a bare
# ``.contiguous().data_ptr()`` could free the temp -> dangling ptr).
io_dtype = getattr(self.estimator, "io_dtype", x.dtype)
[estimator, stream], trt_engine = self.estimator.acquire_estimator()
# NOTE need to synchronize when switching stream
torch.cuda.current_stream().synchronize()
with stream:
estimator.set_input_shape("x", (2, 80, x.size(2)))
estimator.set_input_shape("mask", (2, 1, x.size(2)))
estimator.set_input_shape("mu", (2, 80, x.size(2)))
x_e = x.to(io_dtype).contiguous()
mask_e = mask.to(io_dtype).contiguous()
mu_e = mu.to(io_dtype).contiguous()
t_e = t.to(io_dtype).contiguous()
spks_e = spks.to(io_dtype).contiguous()
cond_e = cond.to(io_dtype).contiguous()
out_e = torch.empty_like(x_e)
estimator.set_input_shape("x", (2, 80, x_e.size(2)))
estimator.set_input_shape("mask", (2, 1, x_e.size(2)))
estimator.set_input_shape("mu", (2, 80, x_e.size(2)))
estimator.set_input_shape("t", (2,))
estimator.set_input_shape("spks", (2, 80))
estimator.set_input_shape("cond", (2, 80, x.size(2)))
estimator.set_input_shape("cond", (2, 80, x_e.size(2)))
data_ptrs = [
x.contiguous().data_ptr(),
mask.contiguous().data_ptr(),
mu.contiguous().data_ptr(),
t.contiguous().data_ptr(),
spks.contiguous().data_ptr(),
cond.contiguous().data_ptr(),
x.data_ptr(),
x_e.data_ptr(),
mask_e.data_ptr(),
mu_e.data_ptr(),
t_e.data_ptr(),
spks_e.data_ptr(),
cond_e.data_ptr(),
out_e.data_ptr(),
]
for i, j in enumerate(data_ptrs):
estimator.set_tensor_address(trt_engine.get_tensor_name(i), j)
@@ -166,7 +179,7 @@ class ConditionalCFM(BASECFM):
assert estimator.execute_async_v3(torch.cuda.current_stream().cuda_stream) is True
torch.cuda.current_stream().synchronize()
self.estimator.release_estimator(estimator, stream)
return x
return out_e.to(x.dtype)
class CausalConditionalCFM(ConditionalCFM):
@@ -3,10 +3,16 @@
import os
from collections.abc import Iterable, Mapping, Sequence
from functools import partial
from math import gcd
from threading import Lock
import numpy as np
import onnxruntime
import torch
import torch.nn as nn
from huggingface_hub import snapshot_download
from scipy.signal import resample_poly
from transformers import Qwen2Config
from transformers.feature_extraction_utils import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
@@ -32,12 +38,15 @@ from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.data_entry_keys import EmbeddingsStruct, OmniPayloadStruct, to_dict, to_struct
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
from vllm_omni.model_executor.models.cosyvoice3.utils import (
concat_text_with_prompt_ids,
extract_speech_feat,
extract_speech_token,
extract_spk_embedding,
extract_spk_embedding_trt,
extract_text_token,
mel_spectrogram,
unpad_prompt_conditioning,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
@@ -45,6 +54,23 @@ from vllm_omni.utils.speaker_cache import get_speaker_cache
logger = init_logger(__name__)
# Process-wide cache of per-model mm-processor runtime components (tokenizer,
# feat_extractor, campplus session/engine). The mm processor is re-created per
# request (mm_processor_cache_gb: 0), so this avoids rebuilding them every time.
_RUNTIME_COMPONENTS_CACHE: dict[str, dict] = {}
def _cosyvoice3_trt_enabled() -> bool:
"""COSYVOICE3_TRT env toggle (default on) for the optional TensorRT paths.
Gates both the talker speaker-embedding (campplus) engine and the code2wav
flow-decoder estimator engine. Env-var toggle (cf. mimo_audio's
``MIMO_AUDIO_TOKENIZER_CUDA_GRAPH``) because these run outside the stage
worker, so deploy-yaml ``hf_overrides`` / per-stage ``env`` do not reach
them — export ``COSYVOICE3_TRT=0`` in the launching shell to disable.
"""
return os.environ.get("COSYVOICE3_TRT", "1") not in ("0", "false", "False", "")
class CosyVoice3MultiModalProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
@@ -73,39 +99,136 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
if cached_model_dir == model_dir:
return
# If model_dir is an HF repo ID (not a local path), resolve to cache
if not os.path.isdir(model_dir):
from huggingface_hub import snapshot_download
# The mm processor is re-created per request (deploy config sets
# ``mm_processor_cache_gb: 0``), so the instance-level guard above never
# hits across requests. Without a process-wide cache, every request
# would re-run ``snapshot_download``, rebuild the Qwen tokenizer and
# create a fresh ONNX campplus session — ~hundreds of ms of pure
# overhead on the TTFP critical path. Build the heavy components once
# per process, keyed by model_dir, and reuse them.
comps = _RUNTIME_COMPONENTS_CACHE.get(model_dir)
if comps is None:
comps = self._build_runtime_components(model_dir, config)
_RUNTIME_COMPONENTS_CACHE[model_dir] = comps
self.tokenizer = comps["tokenizer"]
self.feat_extractor = comps["feat_extractor"]
self.campplus_session = comps["campplus_session"]
self.campplus_trt = comps["campplus_trt"]
self._cached_model_dir = model_dir
self._speaker_cache = get_speaker_cache()
def _build_runtime_components(self, model_dir: str, config: CosyVoice3Config) -> dict:
"""Build the per-model runtime components once (cached process-wide)."""
# If model_dir is an HF repo ID (not a local path), resolve to cache.
if not os.path.isdir(model_dir):
model_dir = snapshot_download(model_dir)
import onnxruntime
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
from vllm_omni.model_executor.models.cosyvoice3.utils import mel_spectrogram
option = onnxruntime.SessionOptions()
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
option.intra_op_num_threads = 1
self.tokenizer = get_qwen_tokenizer(
tokenizer = get_qwen_tokenizer(
token_path=os.path.join(model_dir, config.qwen_pretrain_path),
skip_special_tokens=config.skip_special_tokens,
version=config.version,
)
self.speech_tokenizer = onnxruntime.InferenceSession(
os.path.join(model_dir, config.speech_tokenizer_path),
sess_options=option,
providers=["CUDAExecutionProvider" if torch.cuda.is_available() else "CPUExecutionProvider"],
)
self.feat_extractor = partial(mel_spectrogram, **getattr(config, "feat_extractor", {}))
self.campplus_session = onnxruntime.InferenceSession(
os.path.join(model_dir, config.campplus_onxx_path),
sess_options=option,
providers=["CPUExecutionProvider"],
)
self._cached_model_dir = model_dir
self._speaker_cache = get_speaker_cache()
feat_extractor = partial(mel_spectrogram, **getattr(config, "feat_extractor", {}))
campplus_onnx_path = os.path.join(model_dir, config.campplus_onxx_path)
# TensorRT speaker-embedding path (default on); a prebuilt TRT engine
# runs campplus on GPU. The engine itself is cached process-wide by
# ``get_campplus_trt``.
campplus_trt = None
if self._speaker_embedding_trt_enabled() and torch.cuda.is_available():
try:
from vllm_omni.model_executor.models.cosyvoice3.speaker_embedding_trt import (
get_campplus_trt,
)
campplus_trt = get_campplus_trt(campplus_onnx_path, device="cuda")
logger.info("CosyVoice3: using TensorRT campplus speaker embedding")
except Exception as exc: # pragma: no cover - defensive fallback
logger.warning(
"CosyVoice3: TensorRT campplus build failed (%s); falling back to ONNX-Runtime speaker embedding",
exc,
)
campplus_trt = None
# Only build the CPU ONNX campplus session when TRT is unavailable —
# otherwise it is never used and creating it (ORT_ENABLE_ALL graph
# optimization) costs ~hundreds of ms.
campplus_session = None
if campplus_trt is None:
option = onnxruntime.SessionOptions()
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
option.intra_op_num_threads = 1
campplus_session = onnxruntime.InferenceSession(
campplus_onnx_path,
sess_options=option,
providers=["CPUExecutionProvider"],
)
return {
"tokenizer": tokenizer,
"feat_extractor": feat_extractor,
"campplus_session": campplus_session,
"campplus_trt": campplus_trt,
}
@staticmethod
def _speaker_embedding_trt_enabled() -> bool:
"""Whether to use the TensorRT campplus speaker-embedding path (default on)."""
return _cosyvoice3_trt_enabled()
# Class-level cached s3tokenizer model — loaded once per process on first
# call to ``_extract_speech_token_via_s3`` and shared across all
# processor instances.
_s3_model = None
@classmethod
def _ensure_s3_model(cls):
if cls._s3_model is not None:
return cls._s3_model
# s3tokenizer is imported lazily (kept off the module top level) so
# callers that don't use the CosyVoice3 talker need not install it.
try:
import s3tokenizer as _s3
except ImportError as e:
raise ImportError(
"CosyVoice3 speech-token extraction requires the 's3tokenizer' "
"package; install it with `pip install s3tokenizer`."
) from e
model = _s3.load_model("speech_tokenizer_v3_25hz")
device = "cuda" if torch.cuda.is_available() else "cpu"
model = model.to(device).eval()
cls._s3_model = (model, _s3, device)
return cls._s3_model
def _extract_speech_token_via_s3(self, audio, return_device):
"""Drop-in replacement for ``extract_speech_token`` that uses the
S3Tokenizer PyTorch model on GPU. Returns the same
``(speech_token[1, T], speech_token_len[1])`` int32 tensors as the
ONNX path so the rest of ``_call_hf_processor`` is unchanged.
"""
model, _s3, dev = self._ensure_s3_model()
# audio is a (waveform_ndarray, sr) tuple; resample to 16 kHz mono float32.
wav, sr = audio
wav = np.asarray(wav, dtype=np.float32)
if wav.ndim == 2:
wav = wav.mean(axis=1)
if int(sr) != 16000:
g = gcd(int(sr), 16000)
wav = resample_poly(wav, 16000 // g, int(sr) // g).astype(np.float32)
audio_t = torch.from_numpy(wav)
mel = _s3.log_mel_spectrogram(audio_t)
mels_p, mels_lens = _s3.padding([mel])
with torch.inference_mode():
codes, codes_lens = model.quantize(mels_p.to(dev), mels_lens.to(dev))
n = int(codes_lens[0].item())
speech_token = codes[:1, :n].to(dtype=torch.int32, device=return_device)
speech_token_len = torch.tensor([n], dtype=torch.int32, device=return_device)
return speech_token, speech_token_len
def _call_hf_processor(
self,
@@ -185,7 +308,11 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
)
return ft
speech_token, speech_token_len = extract_speech_token(audio, self.speech_tokenizer, device)
# Speech-token extraction via the S3Tokenizer PyTorch model on GPU
# (~30x faster than the bundled ``speech_tokenizer_v3.onnx`` CPU ONNX
# path in this venv).
speech_token, speech_token_len = self._extract_speech_token_via_s3(audio, device)
speech_feat, speech_feat_len = extract_speech_feat(audio, self.feat_extractor, device)
if config.sample_rate == 24000:
@@ -193,7 +320,10 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
speech_feat, speech_feat_len[:] = speech_feat[:, : 2 * token_len], 2 * token_len
speech_token, speech_token_len[:] = speech_token[:, :token_len], token_len
embedding = extract_spk_embedding(audio, self.campplus_session, device)
if self.campplus_trt is not None:
embedding = extract_spk_embedding_trt(audio, self.campplus_trt, device)
else:
embedding = extract_spk_embedding(audio, self.campplus_session, device)
# Cache the extracted artifacts for named speakers
if cache_key is not None:
@@ -318,8 +448,6 @@ class CosyVoice3Model(
self.model_stage = vllm_config.model_config.model_stage
model_dir = vllm_config.model_config.model
if not os.path.isdir(model_dir):
from huggingface_hub import snapshot_download
model_dir = snapshot_download(model_dir)
self.model_dir = model_dir
self.model = None
@@ -377,8 +505,6 @@ class CosyVoice3Model(
the pretrained model directory. The cache config is inherited from the parent
to enable PagedAttention with the same memory configuration.
"""
from transformers import Qwen2Config
qwen_config_path = os.path.join(self.model_dir, self.config.llm["llm"]["pretrain_path"])
qwen_hf_config = Qwen2Config.from_pretrained(qwen_config_path)
@@ -469,26 +595,32 @@ class CosyVoice3Model(
top_k: int,
generator: torch.Generator | None,
) -> int:
"""Vectorized nucleus + top-k sampling.
Distribution-equivalent to the reference iterative implementation: the
keep-set is identical (token i is kept iff
``cumsum(sorted_probs)[i] - sorted_probs[i] < top_p`` AND ``i < top_k``)
and the renormalized sampling distribution matches, but the exact token
drawn for a given seed is NOT guaranteed to match. The reference draws
via ``multinomial`` over the stacked kept subset while this draws over
the full sorted vector (zeroed outside the keep-set), so the generator
advances over different-sized inputs and may yield a different sample.
The win: no per-token ``.item()`` D2H syncs from the Python loop —
those dominated the sampler CPU time in profiling.
"""
probs = weighted_scores.softmax(dim=0)
sorted_prob, sorted_idx = probs.sort(descending=True, stable=True)
kept_probs: list[torch.Tensor] = []
kept_indices: list[torch.Tensor] = []
cum_prob = 0.0
max_keep = len(sorted_idx) if top_k <= 0 else min(int(top_k), len(sorted_idx))
for i in range(len(sorted_idx)):
if cum_prob < top_p and len(kept_probs) < max_keep:
cum_prob += float(sorted_prob[i].item())
kept_probs.append(sorted_prob[i])
kept_indices.append(sorted_idx[i])
else:
break
if not kept_probs:
return int(sorted_idx[0].item())
sample_probs = torch.stack(kept_probs)
sample_idx = cls._multinomial_sample(sample_probs, generator=generator)
return int(torch.stack(kept_indices)[int(sample_idx.item())].item())
cum_before = sorted_prob.cumsum(dim=0) - sorted_prob
mask = cum_before < top_p
if top_k > 0:
n = sorted_prob.shape[0]
mask = mask & (torch.arange(n, device=mask.device) < min(int(top_k), n))
weights = sorted_prob * mask.to(sorted_prob.dtype)
# First token always passes (cum_before[0] = 0 < top_p for any top_p > 0),
# so ``weights`` is guaranteed to have at least one nonzero entry. The
# final ``.item()`` is the ONLY D2H sync per call.
sample_idx = torch.multinomial(weights, 1, replacement=True, generator=generator)
return int(sorted_idx[sample_idx].item())
@classmethod
def _ras_sample_one(
@@ -627,11 +759,21 @@ class CosyVoice3Model(
else:
raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.")
def embed_multimodal(self, **kwargs: object) -> torch.Tensor:
def embed_multimodal(self, **kwargs: object):
if self.model_stage == "cosyvoice3_talker":
speech_token = kwargs["speech_token"]
speech_token_emb = self.model.speech_embedding(speech_token)
return speech_token_emb
# vLLM's _execute_mm_encoder batches ALL multimodal items scheduled
# in one engine step into a single call, expecting one embedding
# tensor per item back. When >=2 requests prefill in the same step
# (likely once mm preprocessing is fast, e.g. TensorRT speaker
# embedding), speech_token arrives as a list of per-item tensors
# (variable length), so embed it per item and return a list of
# [T_i, emb] tensors. The single-item path keeps returning a
# [1, T, emb] tensor (the caller's extend() iterates dim 0).
if isinstance(speech_token, (list, tuple)):
emb_dim = self.model.speech_embedding.weight.shape[1]
return [self.model.speech_embedding(t).reshape(-1, emb_dim) for t in speech_token]
return self.model.speech_embedding(speech_token)
else:
raise RuntimeError(f"embed_multimodal is only valid for {self.model_stage}.")
@@ -720,6 +862,121 @@ class CosyVoice3Model(
else:
raise RuntimeError(f"embed_input_ids is not valid for {self.model_stage}.")
@staticmethod
def _split_prompt_conditioning(speech_token, speech_feat, embedding, speech_token_len):
"""Split collated prompt conditioning into per-request lists.
Returns ``(speech_token_list, speech_feat_list, embedding_list,
speech_token_len_list)``, one rank-correct tensor per request:
``speech_token`` ``[1, T]`` (possibly right-padded), ``speech_feat``
``[1, 2T, F]``, ``embedding`` ``[1, D]``, ``speech_token_len`` ``[1, 1]``.
This runs inside the talker ``forward`` which may be CUDA-graph
captured, so it must NOT trigger any host<->device sync (no ``.item()``
/ ``.tolist()`` / Python-int slicing by tensor value). Splitting is pure
on-device indexing; right-padding is dropped later in the (eager)
code2wav stage using the per-request ``speech_token_len``. Robust to
inputs arriving as padded ``[B, ...]`` batch tensors or per-request
lists.
"""
def _rows(x, want_dim):
if isinstance(x, (list, tuple)):
items = list(x)
elif isinstance(x, torch.Tensor):
# ``x.shape[0]`` is a static Python int — no device sync.
items = [x[i] for i in range(x.shape[0])]
else:
return None
out = []
for t in items:
if isinstance(t, torch.Tensor):
while t.dim() < want_dim:
t = t.unsqueeze(0)
if t.shape[0] != 1:
t = t[:1]
t = t.contiguous()
out.append(t)
return out
st_out = _rows(speech_token, 2)
sf_out = _rows(speech_feat, 3)
emb_out = _rows(embedding, 2)
stl_out = _rows(speech_token_len, 2)
return st_out, sf_out, emb_out, stl_out
def _resolve_flow_estimator_onnx(self) -> str | None:
"""Locate the flow-decoder estimator ONNX for the TensorRT engine.
Prefers the fp16 (strongly-typed) ONNX → fp16 engine. Order: env
``COSYVOICE3_ESTIMATOR_ONNX`` → ``<model_dir>/<fp16 name>`` → fetched
from ``flow_estimator_onnx_repo`` → bundled fp32 ONNX (fp32+TF32). Each
is checked for existence; returns ``None`` if nothing is found.
"""
env_path = os.environ.get("COSYVOICE3_ESTIMATOR_ONNX")
if env_path and os.path.exists(env_path):
return env_path
fp16_name = getattr(self.config, "flow_estimator_onnx_path", "flow.decoder.estimator.autocast_fp16.onnx")
local_fp16 = os.path.join(self.model_dir, fp16_name)
if os.path.exists(local_fp16):
return local_fp16
repo = getattr(self.config, "flow_estimator_onnx_repo", None)
if repo:
try:
fetched_dir = snapshot_download(repo, allow_patterns=[fp16_name])
fetched = os.path.join(fetched_dir, fp16_name)
if os.path.exists(fetched):
return fetched
except Exception as exc: # pragma: no cover - network/repo issues
logger.warning("CosyVoice3 code2wav: could not fetch fp16 estimator ONNX from %s (%s)", repo, exc)
fp32_name = getattr(self.config, "flow_estimator_onnx_path_fp32", "flow.decoder.estimator.fp32.onnx")
local_fp32 = os.path.join(self.model_dir, fp32_name)
if os.path.exists(local_fp32):
logger.info("CosyVoice3 code2wav: fp16 estimator ONNX unavailable, falling back to fp32 (%s)", local_fp32)
return local_fp32
return None
def _maybe_enable_code2wav_trt(self) -> None:
"""Swap the flow-decoder estimator to a TensorRT engine once (lazy).
Runs on the first code2wav step — after weights are loaded — so the
torch estimator is fully built first and then dropped. No-op unless
``COSYVOICE3_TRT`` is on (default), CUDA is available, and the estimator
ONNX ships with the model. Falls back to the torch estimator on any
failure. The upstream ``CausalConditionalCFM.forward_estimator`` detects
the non-``nn.Module`` estimator and drives the TRT engine.
"""
if getattr(self, "_code2wav_trt_done", False):
return
self._code2wav_trt_done = True
if not (_cosyvoice3_trt_enabled() and torch.cuda.is_available()):
return
onnx_path = self._resolve_flow_estimator_onnx()
if onnx_path is None:
logger.warning("CosyVoice3 code2wav: no flow-estimator ONNX available; keeping torch estimator")
return
try:
from vllm_omni.model_executor.models.cosyvoice3.flow_estimator_trt import (
build_flow_estimator_trt,
)
wrapper = build_flow_estimator_trt(onnx_path, device="cuda")
# ``estimator`` is a registered nn.Module submodule; delete it first
# (frees the torch estimator weights) so the TRT wrapper can be set
# as a plain attribute — nn.Module.__setattr__ rejects non-Modules.
decoder = self.code2wav.flow_model.decoder
del decoder.estimator
decoder.estimator = wrapper
logger.info("CosyVoice3: using TensorRT flow-decoder estimator (code2wav)")
except Exception as exc: # pragma: no cover - defensive fallback
logger.warning(
"CosyVoice3 code2wav: TensorRT estimator build failed (%s); keeping torch estimator",
exc,
)
def forward(
self,
input_ids: torch.Tensor,
@@ -741,18 +998,43 @@ class CosyVoice3Model(
if "speech_token" in kwargs:
# Prompt conditioning tensors for code2wav: live under
# ``embed.*`` per OmniPayloadStruct schema.
#
# vLLM hands these mm fields to forward() as collated, padded
# batch tensors (speech_token [B, maxT], speech_feat
# [B, 2*maxT, F], embedding [B, D]). Emitting them raw makes the
# downstream per-request payload split fragile at batch>1: it
# intermittently de-batches speech_token to 1-D and leaks the
# whole [B, D] embedding to every request, which corrupts voice
# conditioning and crashes code2wav (`prompt_token.shape[1]`).
# Instead, split into an explicit per-request list of unpadded,
# correctly-ranked tensors so ``to_payload_element`` splits them
# deterministically by request index (list[idx]).
speech_token_list, speech_feat_list, embedding_list, speech_token_len_list = (
self._split_prompt_conditioning(
kwargs.get("speech_token"),
kwargs.get("speech_feat"),
kwargs.get("embedding"),
kwargs.get("speech_token_len"),
)
)
multimodal_outputs = to_dict(
OmniPayloadStruct(
embed=EmbeddingsStruct(
speech_token=kwargs.get("speech_token"),
speech_feat=kwargs.get("speech_feat"),
embedding=kwargs.get("embedding"),
speech_token=speech_token_list,
speech_feat=speech_feat_list,
speech_token_len=speech_token_len_list,
embedding=embedding_list,
),
)
)
return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs)
elif self.model_stage == "cosyvoice3_code2wav":
# Lazily swap the flow-decoder estimator to a TensorRT engine on the
# first code2wav step (after weights are loaded), gated by the same
# COSYVOICE3_TRT env toggle as the talker speaker embedding.
self._maybe_enable_code2wav_trt()
runtime_info = kwargs.get("model_intermediate_buffer")
if runtime_info is None:
runtime_info = kwargs.get("runtime_additional_information", [])
@@ -784,6 +1066,11 @@ class CosyVoice3Model(
speech_token = embed.speech_token if embed else None
speech_feat = embed.speech_feat if embed else None
embedding = embed.embedding if embed else None
# Drop any right-padding carried from batched talker emission.
if speech_token is not None and speech_feat is not None:
speech_token, speech_feat = unpad_prompt_conditioning(
speech_token, speech_feat, embed.speech_token_len if embed else None
)
if speech_token is None or speech_feat is None or embedding is None:
if stream_finished and req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
with self._stream_audio_cache_lock:
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TensorRT engine for the CosyVoice3 flow-decoder (CFM) DiT estimator.
The estimator is the per-step network the conditional flow-matching ODE solver
calls during code2wav (token -> mel). It dominates code2wav latency; the
upstream ``CausalConditionalCFM.forward_estimator`` already supports running it
through a TensorRT engine (it switches on ``self.estimator`` not being an
``nn.Module`` and drives it via ``acquire_estimator`` / ``execute_async_v3``).
This module builds that engine from the bundled ``flow.decoder.estimator*.onnx``
and wraps it so it can be dropped in for the torch estimator.
The estimator engine has 6 inputs ``x, mask, mu, t, spks, cond`` and one output.
``x/mask/mu/cond`` carry a dynamic time dim; ``t``/``spks`` are fixed, so only
the former need an optimization profile (TRT infers the latter from the ONNX).
Shapes mirror the CosyVoice runtime.
Precision: TensorRT >= 11 dropped the weakly-typed FP16/INT8 builder flags, so
fp16 only comes from a STRONGLY_TYPED network built from an fp16 ONNX
(``*autocast_fp16*``, fp16 I/O). An fp32 ONNX is built fp32 + the TF32 matmul
flag. ``EXPLICIT_BATCH`` is implicit (no flag) and ``ITensor.dtype`` is
read-only, so neither is set here.
"""
from __future__ import annotations
import os
import queue
import torch
from vllm.logger import init_logger
from vllm_omni.model_executor.models.cosyvoice3.speaker_embedding_trt import (
_resolve_plan_path,
_trt_logger,
)
logger = init_logger(__name__)
# Optimization-profile shapes for the dynamic-length inputs (CFG batch dim = 2,
# 80 mel channels, time dim min/opt/max). Matches CosyVoice's token2wav.
_DYNAMIC_INPUTS = ("x", "mask", "mu", "cond")
_MIN_SHAPES = ((2, 80, 4), (2, 1, 4), (2, 80, 4), (2, 80, 4))
_OPT_SHAPES = ((2, 80, 500), (2, 1, 500), (2, 80, 500), (2, 80, 500))
_MAX_SHAPES = ((2, 80, 3000), (2, 1, 3000), (2, 80, 3000), (2, 80, 3000))
def _is_fp16_onnx(onnx_path: str) -> bool:
"""Heuristic: the project's fp16 estimator ONNX is exported strongly-typed
and named ``*autocast_fp16*`` / ``*fp16*`` (vs ``*fp32*``)."""
name = os.path.basename(onnx_path).lower()
return "fp16" in name or "autocast" in name
def _convert_onnx_to_trt(onnx_path: str, plan_path: str, strongly_typed: bool) -> None:
import tensorrt as trt
logger.info(
"Building flow-estimator TensorRT engine from %s (%s) ...",
onnx_path,
"strongly-typed/fp16" if strongly_typed else "fp32+TF32",
)
trt_logger = _trt_logger()
builder = trt.Builder(trt_logger)
# STRONGLY_TYPED takes precision from the ONNX graph (fp16 engine from an
# fp16 ONNX) — this is the only way to get fp16 in TRT>=11, which dropped
# the weakly-typed FP16 BuilderFlag. Otherwise EXPLICIT_BATCH is implicit,
# so create the network with no flags.
if strongly_typed:
network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED))
else:
network = builder.create_network(0)
parser = trt.OnnxParser(network, trt_logger)
with open(onnx_path, "rb") as f:
if not parser.parse(f.read()):
errs = "; ".join(str(parser.get_error(i)) for i in range(parser.num_errors))
raise ValueError(f"Failed to parse {onnx_path}: {errs}")
config = builder.create_builder_config()
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 32) # 4 GiB
if not strongly_typed:
# fp32 ONNX: enable the best available reduced-precision matmul flag
# (FP16 on older TRT, else TF32 on Ampere+/Hopper). For a strongly-typed
# network the precision is fixed by the graph, so no flag is set.
for _flag_name in ("FP16", "TF32"):
_flag = getattr(trt.BuilderFlag, _flag_name, None)
if _flag is not None:
config.set_flag(_flag)
break
profile = builder.create_optimization_profile()
for name, mn, op, mx in zip(_DYNAMIC_INPUTS, _MIN_SHAPES, _OPT_SHAPES, _MAX_SHAPES):
profile.set_shape(name, mn, op, mx)
config.add_optimization_profile(profile)
engine_bytes = builder.build_serialized_network(network, config)
if engine_bytes is None:
raise RuntimeError(f"TensorRT failed to build flow-estimator engine from {onnx_path}")
tmp = plan_path + ".tmp"
with open(tmp, "wb") as f:
f.write(engine_bytes)
os.replace(tmp, plan_path)
logger.info("Wrote flow-estimator TensorRT engine to %s", plan_path)
class TrtContextWrapper:
"""Pool of TensorRT execution contexts for the flow estimator.
Exposes the ``acquire_estimator`` / ``release_estimator`` contract that
``CausalConditionalCFM.forward_estimator`` expects.
"""
def __init__(
self, engine, device: str | torch.device, io_dtype: torch.dtype = torch.float32, trt_concurrent: int = 1
):
self.trt_engine = engine
# Engine I/O dtype (fp16 for a strongly-typed fp16 engine). The flow runs
# in fp32, so forward_estimator casts to/from this at the boundary.
self.io_dtype = io_dtype
self._pool: queue.Queue = queue.Queue(maxsize=trt_concurrent)
for _ in range(trt_concurrent):
ctx = engine.create_execution_context()
assert ctx is not None, "failed to create TRT execution context (out of memory?)"
stream = torch.cuda.stream(torch.cuda.Stream(torch.device(device)))
self._pool.put([ctx, stream])
def acquire_estimator(self):
return self._pool.get(), self.trt_engine
def release_estimator(self, context, stream):
self._pool.put([context, stream])
def build_flow_estimator_trt(onnx_path: str, device: str | torch.device) -> TrtContextWrapper:
"""Build/load the flow-estimator TRT engine and return a context-pool wrapper.
An fp16 ONNX (``*autocast_fp16*``) is built as a strongly-typed network (the
only way to get fp16 in TRT>=11); an fp32 ONNX is built fp32 + TF32.
"""
import tensorrt as trt
strongly_typed = _is_fp16_onnx(onnx_path)
plan_path = _resolve_plan_path(onnx_path, prefix="flow_estimator")
if not os.path.exists(plan_path) or os.path.getsize(plan_path) == 0:
_convert_onnx_to_trt(onnx_path, plan_path, strongly_typed=strongly_typed)
runtime = trt.Runtime(_trt_logger())
with open(plan_path, "rb") as f:
engine = runtime.deserialize_cuda_engine(f.read())
if engine is None:
raise RuntimeError(f"Failed to deserialize flow-estimator TensorRT engine {plan_path}")
logger.info("Loaded flow-estimator TensorRT engine (%s)", plan_path)
io_dtype = torch.float16 if strongly_typed else torch.float32
return TrtContextWrapper(engine, device=device, io_dtype=io_dtype)
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TensorRT speaker-embedding (campplus) extractor for CosyVoice3.
Drop-in accelerator for the ONNX-Runtime campplus session used in the
multimodal processor. ``campplus.onnx`` maps an fbank feature
``[B, T, 80]`` to a 192-d speaker embedding ``[B, 192]``; on this stack the
ONNX path only has a CPU execution provider, so it dominates per-request
prompt-processing latency. Building a TensorRT engine once and running it on
GPU removes that bottleneck.
The engine is built lazily from ``campplus.onnx`` on first use and cached as
a ``.plan`` file (engines are device/TRT-version specific, so the cache key
includes both). Set ``COSYVOICE3_TRT_CACHE`` to override the cache dir.
Written against TensorRT >= 10/11: ``EXPLICIT_BATCH`` is the implicit default
(create the network with no flags) and ``ITensor.dtype`` is read-only (I/O
dtypes come from the ONNX graph), so neither is set here.
"""
from __future__ import annotations
import hashlib
import os
import threading
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
# campplus fbank sequence length bounds for the optimization profile.
_MIN_T = 4
_OPT_T = 500
_MAX_T = 3000
_EMBED_DIM = 192
def _trt_logger():
import tensorrt as trt
return trt.Logger(trt.Logger.WARNING)
def _convert_onnx_to_trt(onnx_path: str, plan_path: str) -> None:
import tensorrt as trt
logger.info("Building campplus TensorRT engine from %s ...", onnx_path)
trt_logger = _trt_logger()
builder = trt.Builder(trt_logger)
# EXPLICIT_BATCH is implicit in TRT>=10; create the network with no flags.
network = builder.create_network(0)
parser = trt.OnnxParser(network, trt_logger)
with open(onnx_path, "rb") as f:
if not parser.parse(f.read()):
errs = "; ".join(str(parser.get_error(i)) for i in range(parser.num_errors))
raise ValueError(f"Failed to parse {onnx_path}: {errs}")
config = builder.create_builder_config()
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 31) # 2 GiB
input_name = network.get_input(0).name
profile = builder.create_optimization_profile()
profile.set_shape(
input_name,
(1, _MIN_T, 80),
(1, _OPT_T, 80),
(1, _MAX_T, 80),
)
config.add_optimization_profile(profile)
engine_bytes = builder.build_serialized_network(network, config)
if engine_bytes is None:
raise RuntimeError(f"TensorRT failed to build campplus engine from {onnx_path}")
tmp = plan_path + ".tmp"
with open(tmp, "wb") as f:
f.write(engine_bytes)
os.replace(tmp, plan_path)
logger.info("Wrote campplus TensorRT engine to %s", plan_path)
def _resolve_plan_path(onnx_path: str, prefix: str = "campplus") -> str:
cache_dir = os.environ.get("COSYVOICE3_TRT_CACHE") or os.path.join(
os.path.expanduser("~"), ".cache", "vllm_omni", "cosyvoice3_trt"
)
os.makedirs(cache_dir, exist_ok=True)
try:
# No-arg form targets the current device (avoids the banned
# torch.cuda.current_device per ruff TID251).
dev_name = torch.cuda.get_device_name()
except Exception:
dev_name = "unknown"
import tensorrt as trt
st = os.stat(onnx_path)
key = f"{os.path.abspath(onnx_path)}|{st.st_size}|{int(st.st_mtime)}|{dev_name}|trt{trt.__version__}"
digest = hashlib.sha1(key.encode()).hexdigest()[:16]
return os.path.join(cache_dir, f"{prefix}_{digest}.plan")
class CampplusTRT:
"""Lazily-built TensorRT campplus speaker-embedding extractor."""
def __init__(self, onnx_path: str, device: str | torch.device):
import tensorrt as trt
self.device = torch.device(device)
plan_path = _resolve_plan_path(onnx_path)
if not os.path.exists(plan_path) or os.path.getsize(plan_path) == 0:
_convert_onnx_to_trt(onnx_path, plan_path)
runtime = trt.Runtime(_trt_logger())
with open(plan_path, "rb") as f:
self.engine = runtime.deserialize_cuda_engine(f.read())
if self.engine is None:
raise RuntimeError(f"Failed to deserialize campplus TensorRT engine {plan_path}")
self.context = self.engine.create_execution_context()
self.input_name = self.engine.get_tensor_name(0)
self.output_name = self.engine.get_tensor_name(1)
# TRT execution contexts are not safe for concurrent enqueue; serialize.
self._lock = threading.Lock()
logger.info("Loaded campplus TensorRT engine (%s)", plan_path)
@torch.inference_mode()
def __call__(self, feat: torch.Tensor) -> torch.Tensor:
"""feat: ``[T, 80]`` fbank features. Returns ``[1, 192]`` embedding."""
x = feat.unsqueeze(0).to(self.device, dtype=torch.float32).contiguous()
out = torch.empty((x.shape[0], _EMBED_DIM), device=self.device, dtype=torch.float32)
stream = torch.cuda.current_stream(self.device)
with self._lock:
self.context.set_input_shape(self.input_name, tuple(x.shape))
self.context.set_tensor_address(self.input_name, x.data_ptr())
self.context.set_tensor_address(self.output_name, out.data_ptr())
ok = self.context.execute_async_v3(stream.cuda_stream)
if not ok:
raise RuntimeError("campplus TensorRT execute_async_v3 failed")
stream.synchronize()
return out
# Process-wide cache: the mm processor is re-created per request (the deploy
# config disables the mm-processor object cache), so building a fresh
# CampplusTRT each time would re-deserialize the ~33 MB engine and allocate a
# new execution context on every request — inflating TTFP. Cache by
# (engine path, device) and reuse; the context serializes concurrent enqueues
# via its own lock, so a single shared instance is safe across requests.
_CAMPPLUS_CACHE: dict[tuple[str, str], CampplusTRT] = {}
def get_campplus_trt(onnx_path: str, device: str | torch.device) -> CampplusTRT:
"""Return a process-wide cached :class:`CampplusTRT` (engine + context)."""
key = (os.path.abspath(onnx_path), str(device))
inst = _CAMPPLUS_CACHE.get(key)
if inst is None:
inst = CampplusTRT(onnx_path, device)
_CAMPPLUS_CACHE[key] = inst
return inst
@@ -215,6 +215,44 @@ def extract_spk_embedding(prompt_wav, campplus_session, device):
return embedding
def unpad_prompt_conditioning(speech_token, speech_feat, speech_token_len):
"""Drop right-padding from per-request prompt conditioning.
The talker emits ``speech_token`` ``[1, T]`` possibly right-padded to the
batch max, plus the true ``speech_token_len``; ``speech_feat`` is at the 2:1
mel:token ratio (``[1, 2T, F]``). This trims both to the real length. Called
in the (eager) code2wav stage where a host read of the length is allowed.
Returns ``(speech_token, speech_feat)`` unchanged when length is unavailable.
"""
if not isinstance(speech_token, torch.Tensor) or speech_token_len is None:
return speech_token, speech_feat
try:
token_len = int(torch.as_tensor(speech_token_len).reshape(-1)[0].item())
except (RuntimeError, ValueError, IndexError):
return speech_token, speech_feat
if token_len <= 0 or speech_token.dim() < 2:
return speech_token, speech_feat
token_len = min(token_len, int(speech_token.shape[1]))
speech_token = speech_token[:, :token_len]
if isinstance(speech_feat, torch.Tensor) and speech_feat.dim() >= 2:
speech_feat = speech_feat[:, : 2 * token_len]
return speech_token, speech_feat
def extract_spk_embedding_trt(prompt_wav, campplus_trt, device):
"""TensorRT counterpart of ``extract_spk_embedding``.
Identical fbank front-end; the campplus forward runs on a prebuilt
TensorRT engine (GPU) instead of the CPU ONNX-Runtime session. Returns the
same ``[1, 192]`` embedding tensor on ``device``.
"""
speech = load_wav(prompt_wav, 16000)
feat = kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000)
feat = feat - feat.mean(dim=0, keepdim=True)
embedding = campplus_trt(feat)
return embedding.to(device)
def extract_text_token(text, tokenizer, allowed_special):
text_token = tokenizer.encode(text, allowed_special=allowed_special)
text_token = torch.tensor([text_token], dtype=torch.int32)
@@ -16,6 +16,7 @@ from vllm_omni.data_entry_keys import (
OmniPayloadStruct,
)
from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.model_executor.models.cosyvoice3.utils import unpad_prompt_conditioning
logger = init_logger(__name__)
@@ -192,18 +193,31 @@ def talker2code2wav_async_chunk(
info = _decode_additional_information(getattr(request, "additional_information", None))
info_embed = info.get("embed", {}) if isinstance(info, dict) else {}
prompt_payload = {}
for key in ("speech_token", "speech_feat", "embedding"):
cond_keys = ("speech_token", "speech_feat", "embedding", "speech_token_len")
for key in cond_keys:
value = _to_cpu_tensor(info_embed.get(key))
if value is not None:
prompt_payload[key] = value
if isinstance(pooling_output, dict):
po_embed = pooling_output.get("embed", {}) if isinstance(pooling_output.get("embed"), dict) else {}
for key in ("speech_token", "speech_feat", "embedding"):
for key in cond_keys:
if key in prompt_payload:
continue
value = _to_cpu_tensor(po_embed.get(key))
if value is not None:
prompt_payload[key] = value
# Drop any right-padding carried from batched talker emission so
# the chunk-routing math (prompt_token_len) and the conditioning
# sent to code2wav use the true prompt length.
if "speech_token" in prompt_payload:
st_unpad, sf_unpad = unpad_prompt_conditioning(
prompt_payload.get("speech_token"),
prompt_payload.get("speech_feat"),
prompt_payload.pop("speech_token_len", None),
)
prompt_payload["speech_token"] = st_unpad
if sf_unpad is not None:
prompt_payload["speech_feat"] = sf_unpad
prompt_token = prompt_payload.get("speech_token")
prompt_token_len = (
int(prompt_token.shape[1])
@@ -42,6 +42,13 @@ class CosyVoice3Config(PretrainedConfig):
self.qwen_pretrain_path = "CosyVoice-BlankEN"
self.campplus_onxx_path = "campplus.onnx"
self.speech_tokenizer_path = "speech_tokenizer_v3.onnx"
# Flow-decoder estimator ONNX for the optional TensorRT code2wav path.
# The fp16 (strongly-typed) ONNX gives an fp16 engine; if it is not in
# the model dir it is fetched from ``flow_estimator_onnx_repo``. Falls
# back to the bundled fp32 ONNX (fp32+TF32 engine) when unavailable.
self.flow_estimator_onnx_path = "flow.decoder.estimator.autocast_fp16.onnx"
self.flow_estimator_onnx_repo = "yuekai/Fun-CosyVoice3-0.5B-2512-FP16-ONNX"
self.flow_estimator_onnx_path_fp32 = "flow.decoder.estimator.fp32.onnx"
self.spk2info_path = "spk2info.pt"
self.version = "cosyvoice3"
self.llm = {