[Perf][TTS] Optimize cosyvoice TTFP and throughput using TensorRT (#4168)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
57c8f3abfc
commit
f1bf544208
11 files changed
+786
-87
No files matched your search
@@ -84,6 +84,11 @@ def _make_code2wav_model(
|
||||
token_mel_ratio=2 if with_stride_cfg else 0,
|
||||
)
|
||||
model.code2wav = _DummyCode2Wav(vocab_size=4, num_samples=num_samples, outputs=outputs)
|
||||
# Short-circuit the lazy TensorRT estimator swap: these tests exercise the
|
||||
# forward audio logic, not the TRT path. On a GPU CI runner the swap would
|
||||
# otherwise run and dereference ``self.model_dir`` (only set in __init__,
|
||||
# which this fixture bypasses via object.__new__).
|
||||
model._code2wav_trt_done = True
|
||||
model.source_cache_len = 4
|
||||
model.speech_window = torch.hamming_window(8, periodic=False)
|
||||
model._stream_audio_cache_by_req = {}
|
||||
|
||||
@@ -128,6 +128,7 @@ class EmbeddingsStruct(_StructBase):
|
||||
voice: torch.Tensor | None = None
|
||||
speech_feat: torch.Tensor | None = None
|
||||
speech_token: torch.Tensor | None = None
|
||||
speech_token_len: torch.Tensor | None = None
|
||||
embedding: torch.Tensor | None = None
|
||||
thinker_reply: torch.Tensor | None = None
|
||||
|
||||
|
||||
@@ -7,9 +7,22 @@
|
||||
# references stay in the yaml unconditionally; the runtime only activates
|
||||
# them when ``async_chunk: true``.
|
||||
#
|
||||
# enforce_eager=true everywhere:
|
||||
# * Stage 0 (talker) — cudagraph not verified on this checkpoint.
|
||||
# * Stage 1 (code2wav) — CUDA graphs don't work with dynamic conv shapes.
|
||||
# enforce_eager:
|
||||
# * Stage 0 (talker) — enforce_eager: false (CUDA graphs enabled).
|
||||
# * Stage 1 (code2wav) — enforce_eager: true (CUDA graphs don't work with
|
||||
# dynamic conv shapes).
|
||||
#
|
||||
# Streaming continuity is GPU-class sensitive — TUNE PER GPU.
|
||||
# The defaults below (max_num_seqs: 8, codec_chunk_frames: 15) are tuned for
|
||||
# H100, where streaming continuity at concurrency 4 stays ~100%. They push
|
||||
# more audio per scheduler step through the SharedMemoryConnector than a
|
||||
# slower GPU can keep up with: on H20-3e the same config regressed continuity
|
||||
# to ~78% at c=4 (vs ~86% with smaller batch/chunk), because code2wav can't
|
||||
# produce mel chunks fast enough to feed the streaming output → buffer
|
||||
# underruns → audible gaps.
|
||||
# On a slower GPU class, lower max_num_seqs (e.g. 4) and/or codec_chunk_frames
|
||||
# first. Raising connector_get_sleep_s also relaxes the consumer's polling
|
||||
# cadence, but trades against TTFP — keep it as a last resort.
|
||||
async_chunk: true
|
||||
dtype: float32
|
||||
|
||||
@@ -21,16 +34,16 @@ connectors:
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_chunk_frames: 15
|
||||
codec_left_context_frames: 25
|
||||
codec_vocab_size: 6561
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_batched_tokens: 32768
|
||||
max_num_seqs: 1
|
||||
max_num_seqs: 8
|
||||
gpu_memory_utilization: 0.4
|
||||
enforce_eager: true
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
devices: "0"
|
||||
@@ -49,7 +62,7 @@ stages:
|
||||
|
||||
- stage_id: 1
|
||||
max_num_batched_tokens: 32768
|
||||
max_num_seqs: 1
|
||||
max_num_seqs: 8
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
|
||||
@@ -410,6 +410,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
)
|
||||
self._fish_speech_tokenizer = None
|
||||
self._covo_audio_tokenizer = None
|
||||
# Cached per process: the CosyVoice3 Qwen tokenizer + resolved model
|
||||
# path used for dynamic-token sizing. Without this, every request
|
||||
# re-ran snapshot_download + reloaded the tokenizer (~100 ms on the
|
||||
# TTFP critical path) in _apply_cosyvoice3_dynamic_tokens.
|
||||
self._cosyvoice3_tokenizer = None
|
||||
|
||||
self._is_cosyvoice3 = (
|
||||
self._tts_stage is not None
|
||||
@@ -2658,16 +2663,21 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
sampling_params_list = copy.deepcopy(sampling_params_list)
|
||||
hf_cfg = self.model_config.hf_config
|
||||
model_path = self.engine_client.model_config.model
|
||||
if not os.path.isdir(model_path):
|
||||
from huggingface_hub import snapshot_download
|
||||
# Build the Qwen tokenizer once per process (resolving the model dir via
|
||||
# snapshot_download at most once) and reuse it across requests.
|
||||
tokenizer = self._cosyvoice3_tokenizer
|
||||
if tokenizer is None:
|
||||
model_path = self.engine_client.model_config.model
|
||||
if not os.path.isdir(model_path):
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_path = snapshot_download(model_path)
|
||||
tokenizer = get_qwen_tokenizer(
|
||||
token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path),
|
||||
skip_special_tokens=hf_cfg.skip_special_tokens,
|
||||
version=hf_cfg.version,
|
||||
)
|
||||
model_path = snapshot_download(model_path)
|
||||
tokenizer = get_qwen_tokenizer(
|
||||
token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path),
|
||||
skip_special_tokens=hf_cfg.skip_special_tokens,
|
||||
version=hf_cfg.version,
|
||||
)
|
||||
self._cosyvoice3_tokenizer = tokenizer
|
||||
_, text_token_len = extract_text_token(
|
||||
request.input,
|
||||
tokenizer,
|
||||
|
||||
@@ -141,24 +141,37 @@ class ConditionalCFM(BASECFM):
|
||||
if isinstance(self.estimator, torch.nn.Module):
|
||||
return self.estimator(x, mask, mu, t, spks, cond)
|
||||
else:
|
||||
# TensorRT estimator: bind raw device pointers. The flow runs in
|
||||
# fp32 but the engine may have fp16 I/O (strongly-typed fp16 engine),
|
||||
# so cast inputs/output to the engine's dtype at the boundary. Keep
|
||||
# references to the cast buffers alive until execute completes (a bare
|
||||
# ``.contiguous().data_ptr()`` could free the temp -> dangling ptr).
|
||||
io_dtype = getattr(self.estimator, "io_dtype", x.dtype)
|
||||
[estimator, stream], trt_engine = self.estimator.acquire_estimator()
|
||||
# NOTE need to synchronize when switching stream
|
||||
torch.cuda.current_stream().synchronize()
|
||||
with stream:
|
||||
estimator.set_input_shape("x", (2, 80, x.size(2)))
|
||||
estimator.set_input_shape("mask", (2, 1, x.size(2)))
|
||||
estimator.set_input_shape("mu", (2, 80, x.size(2)))
|
||||
x_e = x.to(io_dtype).contiguous()
|
||||
mask_e = mask.to(io_dtype).contiguous()
|
||||
mu_e = mu.to(io_dtype).contiguous()
|
||||
t_e = t.to(io_dtype).contiguous()
|
||||
spks_e = spks.to(io_dtype).contiguous()
|
||||
cond_e = cond.to(io_dtype).contiguous()
|
||||
out_e = torch.empty_like(x_e)
|
||||
estimator.set_input_shape("x", (2, 80, x_e.size(2)))
|
||||
estimator.set_input_shape("mask", (2, 1, x_e.size(2)))
|
||||
estimator.set_input_shape("mu", (2, 80, x_e.size(2)))
|
||||
estimator.set_input_shape("t", (2,))
|
||||
estimator.set_input_shape("spks", (2, 80))
|
||||
estimator.set_input_shape("cond", (2, 80, x.size(2)))
|
||||
estimator.set_input_shape("cond", (2, 80, x_e.size(2)))
|
||||
data_ptrs = [
|
||||
x.contiguous().data_ptr(),
|
||||
mask.contiguous().data_ptr(),
|
||||
mu.contiguous().data_ptr(),
|
||||
t.contiguous().data_ptr(),
|
||||
spks.contiguous().data_ptr(),
|
||||
cond.contiguous().data_ptr(),
|
||||
x.data_ptr(),
|
||||
x_e.data_ptr(),
|
||||
mask_e.data_ptr(),
|
||||
mu_e.data_ptr(),
|
||||
t_e.data_ptr(),
|
||||
spks_e.data_ptr(),
|
||||
cond_e.data_ptr(),
|
||||
out_e.data_ptr(),
|
||||
]
|
||||
for i, j in enumerate(data_ptrs):
|
||||
estimator.set_tensor_address(trt_engine.get_tensor_name(i), j)
|
||||
@@ -166,7 +179,7 @@ class ConditionalCFM(BASECFM):
|
||||
assert estimator.execute_async_v3(torch.cuda.current_stream().cuda_stream) is True
|
||||
torch.cuda.current_stream().synchronize()
|
||||
self.estimator.release_estimator(estimator, stream)
|
||||
return x
|
||||
return out_e.to(x.dtype)
|
||||
|
||||
|
||||
class CausalConditionalCFM(ConditionalCFM):
|
||||
|
||||
@@ -3,10 +3,16 @@
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from math import gcd
|
||||
from threading import Lock
|
||||
|
||||
import numpy as np
|
||||
import onnxruntime
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from huggingface_hub import snapshot_download
|
||||
from scipy.signal import resample_poly
|
||||
from transformers import Qwen2Config
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
@@ -32,12 +38,15 @@ from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.data_entry_keys import EmbeddingsStruct, OmniPayloadStruct, to_dict, to_struct
|
||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import (
|
||||
concat_text_with_prompt_ids,
|
||||
extract_speech_feat,
|
||||
extract_speech_token,
|
||||
extract_spk_embedding,
|
||||
extract_spk_embedding_trt,
|
||||
extract_text_token,
|
||||
mel_spectrogram,
|
||||
unpad_prompt_conditioning,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
|
||||
@@ -45,6 +54,23 @@ from vllm_omni.utils.speaker_cache import get_speaker_cache
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Process-wide cache of per-model mm-processor runtime components (tokenizer,
|
||||
# feat_extractor, campplus session/engine). The mm processor is re-created per
|
||||
# request (mm_processor_cache_gb: 0), so this avoids rebuilding them every time.
|
||||
_RUNTIME_COMPONENTS_CACHE: dict[str, dict] = {}
|
||||
|
||||
|
||||
def _cosyvoice3_trt_enabled() -> bool:
|
||||
"""COSYVOICE3_TRT env toggle (default on) for the optional TensorRT paths.
|
||||
|
||||
Gates both the talker speaker-embedding (campplus) engine and the code2wav
|
||||
flow-decoder estimator engine. Env-var toggle (cf. mimo_audio's
|
||||
``MIMO_AUDIO_TOKENIZER_CUDA_GRAPH``) because these run outside the stage
|
||||
worker, so deploy-yaml ``hf_overrides`` / per-stage ``env`` do not reach
|
||||
them — export ``COSYVOICE3_TRT=0`` in the launching shell to disable.
|
||||
"""
|
||||
return os.environ.get("COSYVOICE3_TRT", "1") not in ("0", "false", "False", "")
|
||||
|
||||
|
||||
class CosyVoice3MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
@@ -73,39 +99,136 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
|
||||
if cached_model_dir == model_dir:
|
||||
return
|
||||
|
||||
# If model_dir is an HF repo ID (not a local path), resolve to cache
|
||||
if not os.path.isdir(model_dir):
|
||||
from huggingface_hub import snapshot_download
|
||||
# The mm processor is re-created per request (deploy config sets
|
||||
# ``mm_processor_cache_gb: 0``), so the instance-level guard above never
|
||||
# hits across requests. Without a process-wide cache, every request
|
||||
# would re-run ``snapshot_download``, rebuild the Qwen tokenizer and
|
||||
# create a fresh ONNX campplus session — ~hundreds of ms of pure
|
||||
# overhead on the TTFP critical path. Build the heavy components once
|
||||
# per process, keyed by model_dir, and reuse them.
|
||||
comps = _RUNTIME_COMPONENTS_CACHE.get(model_dir)
|
||||
if comps is None:
|
||||
comps = self._build_runtime_components(model_dir, config)
|
||||
_RUNTIME_COMPONENTS_CACHE[model_dir] = comps
|
||||
|
||||
self.tokenizer = comps["tokenizer"]
|
||||
self.feat_extractor = comps["feat_extractor"]
|
||||
self.campplus_session = comps["campplus_session"]
|
||||
self.campplus_trt = comps["campplus_trt"]
|
||||
self._cached_model_dir = model_dir
|
||||
self._speaker_cache = get_speaker_cache()
|
||||
|
||||
def _build_runtime_components(self, model_dir: str, config: CosyVoice3Config) -> dict:
|
||||
"""Build the per-model runtime components once (cached process-wide)."""
|
||||
# If model_dir is an HF repo ID (not a local path), resolve to cache.
|
||||
if not os.path.isdir(model_dir):
|
||||
model_dir = snapshot_download(model_dir)
|
||||
|
||||
import onnxruntime
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import mel_spectrogram
|
||||
|
||||
option = onnxruntime.SessionOptions()
|
||||
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
option.intra_op_num_threads = 1
|
||||
|
||||
self.tokenizer = get_qwen_tokenizer(
|
||||
tokenizer = get_qwen_tokenizer(
|
||||
token_path=os.path.join(model_dir, config.qwen_pretrain_path),
|
||||
skip_special_tokens=config.skip_special_tokens,
|
||||
version=config.version,
|
||||
)
|
||||
self.speech_tokenizer = onnxruntime.InferenceSession(
|
||||
os.path.join(model_dir, config.speech_tokenizer_path),
|
||||
sess_options=option,
|
||||
providers=["CUDAExecutionProvider" if torch.cuda.is_available() else "CPUExecutionProvider"],
|
||||
)
|
||||
self.feat_extractor = partial(mel_spectrogram, **getattr(config, "feat_extractor", {}))
|
||||
self.campplus_session = onnxruntime.InferenceSession(
|
||||
os.path.join(model_dir, config.campplus_onxx_path),
|
||||
sess_options=option,
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
self._cached_model_dir = model_dir
|
||||
self._speaker_cache = get_speaker_cache()
|
||||
feat_extractor = partial(mel_spectrogram, **getattr(config, "feat_extractor", {}))
|
||||
|
||||
campplus_onnx_path = os.path.join(model_dir, config.campplus_onxx_path)
|
||||
# TensorRT speaker-embedding path (default on); a prebuilt TRT engine
|
||||
# runs campplus on GPU. The engine itself is cached process-wide by
|
||||
# ``get_campplus_trt``.
|
||||
campplus_trt = None
|
||||
if self._speaker_embedding_trt_enabled() and torch.cuda.is_available():
|
||||
try:
|
||||
from vllm_omni.model_executor.models.cosyvoice3.speaker_embedding_trt import (
|
||||
get_campplus_trt,
|
||||
)
|
||||
|
||||
campplus_trt = get_campplus_trt(campplus_onnx_path, device="cuda")
|
||||
logger.info("CosyVoice3: using TensorRT campplus speaker embedding")
|
||||
except Exception as exc: # pragma: no cover - defensive fallback
|
||||
logger.warning(
|
||||
"CosyVoice3: TensorRT campplus build failed (%s); falling back to ONNX-Runtime speaker embedding",
|
||||
exc,
|
||||
)
|
||||
campplus_trt = None
|
||||
|
||||
# Only build the CPU ONNX campplus session when TRT is unavailable —
|
||||
# otherwise it is never used and creating it (ORT_ENABLE_ALL graph
|
||||
# optimization) costs ~hundreds of ms.
|
||||
campplus_session = None
|
||||
if campplus_trt is None:
|
||||
option = onnxruntime.SessionOptions()
|
||||
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
option.intra_op_num_threads = 1
|
||||
campplus_session = onnxruntime.InferenceSession(
|
||||
campplus_onnx_path,
|
||||
sess_options=option,
|
||||
providers=["CPUExecutionProvider"],
|
||||
)
|
||||
|
||||
return {
|
||||
"tokenizer": tokenizer,
|
||||
"feat_extractor": feat_extractor,
|
||||
"campplus_session": campplus_session,
|
||||
"campplus_trt": campplus_trt,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _speaker_embedding_trt_enabled() -> bool:
|
||||
"""Whether to use the TensorRT campplus speaker-embedding path (default on)."""
|
||||
return _cosyvoice3_trt_enabled()
|
||||
|
||||
# Class-level cached s3tokenizer model — loaded once per process on first
|
||||
# call to ``_extract_speech_token_via_s3`` and shared across all
|
||||
# processor instances.
|
||||
_s3_model = None
|
||||
|
||||
@classmethod
|
||||
def _ensure_s3_model(cls):
|
||||
if cls._s3_model is not None:
|
||||
return cls._s3_model
|
||||
|
||||
# s3tokenizer is imported lazily (kept off the module top level) so
|
||||
# callers that don't use the CosyVoice3 talker need not install it.
|
||||
try:
|
||||
import s3tokenizer as _s3
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"CosyVoice3 speech-token extraction requires the 's3tokenizer' "
|
||||
"package; install it with `pip install s3tokenizer`."
|
||||
) from e
|
||||
|
||||
model = _s3.load_model("speech_tokenizer_v3_25hz")
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model = model.to(device).eval()
|
||||
cls._s3_model = (model, _s3, device)
|
||||
return cls._s3_model
|
||||
|
||||
def _extract_speech_token_via_s3(self, audio, return_device):
|
||||
"""Drop-in replacement for ``extract_speech_token`` that uses the
|
||||
S3Tokenizer PyTorch model on GPU. Returns the same
|
||||
``(speech_token[1, T], speech_token_len[1])`` int32 tensors as the
|
||||
ONNX path so the rest of ``_call_hf_processor`` is unchanged.
|
||||
"""
|
||||
model, _s3, dev = self._ensure_s3_model()
|
||||
|
||||
# audio is a (waveform_ndarray, sr) tuple; resample to 16 kHz mono float32.
|
||||
wav, sr = audio
|
||||
wav = np.asarray(wav, dtype=np.float32)
|
||||
if wav.ndim == 2:
|
||||
wav = wav.mean(axis=1)
|
||||
if int(sr) != 16000:
|
||||
g = gcd(int(sr), 16000)
|
||||
wav = resample_poly(wav, 16000 // g, int(sr) // g).astype(np.float32)
|
||||
audio_t = torch.from_numpy(wav)
|
||||
|
||||
mel = _s3.log_mel_spectrogram(audio_t)
|
||||
mels_p, mels_lens = _s3.padding([mel])
|
||||
with torch.inference_mode():
|
||||
codes, codes_lens = model.quantize(mels_p.to(dev), mels_lens.to(dev))
|
||||
n = int(codes_lens[0].item())
|
||||
speech_token = codes[:1, :n].to(dtype=torch.int32, device=return_device)
|
||||
speech_token_len = torch.tensor([n], dtype=torch.int32, device=return_device)
|
||||
return speech_token, speech_token_len
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
@@ -185,7 +308,11 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
|
||||
)
|
||||
return ft
|
||||
|
||||
speech_token, speech_token_len = extract_speech_token(audio, self.speech_tokenizer, device)
|
||||
# Speech-token extraction via the S3Tokenizer PyTorch model on GPU
|
||||
# (~30x faster than the bundled ``speech_tokenizer_v3.onnx`` CPU ONNX
|
||||
# path in this venv).
|
||||
speech_token, speech_token_len = self._extract_speech_token_via_s3(audio, device)
|
||||
|
||||
speech_feat, speech_feat_len = extract_speech_feat(audio, self.feat_extractor, device)
|
||||
|
||||
if config.sample_rate == 24000:
|
||||
@@ -193,7 +320,10 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
|
||||
speech_feat, speech_feat_len[:] = speech_feat[:, : 2 * token_len], 2 * token_len
|
||||
speech_token, speech_token_len[:] = speech_token[:, :token_len], token_len
|
||||
|
||||
embedding = extract_spk_embedding(audio, self.campplus_session, device)
|
||||
if self.campplus_trt is not None:
|
||||
embedding = extract_spk_embedding_trt(audio, self.campplus_trt, device)
|
||||
else:
|
||||
embedding = extract_spk_embedding(audio, self.campplus_session, device)
|
||||
|
||||
# Cache the extracted artifacts for named speakers
|
||||
if cache_key is not None:
|
||||
@@ -318,8 +448,6 @@ class CosyVoice3Model(
|
||||
self.model_stage = vllm_config.model_config.model_stage
|
||||
model_dir = vllm_config.model_config.model
|
||||
if not os.path.isdir(model_dir):
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_dir = snapshot_download(model_dir)
|
||||
self.model_dir = model_dir
|
||||
self.model = None
|
||||
@@ -377,8 +505,6 @@ class CosyVoice3Model(
|
||||
the pretrained model directory. The cache config is inherited from the parent
|
||||
to enable PagedAttention with the same memory configuration.
|
||||
"""
|
||||
from transformers import Qwen2Config
|
||||
|
||||
qwen_config_path = os.path.join(self.model_dir, self.config.llm["llm"]["pretrain_path"])
|
||||
qwen_hf_config = Qwen2Config.from_pretrained(qwen_config_path)
|
||||
|
||||
@@ -469,26 +595,32 @@ class CosyVoice3Model(
|
||||
top_k: int,
|
||||
generator: torch.Generator | None,
|
||||
) -> int:
|
||||
"""Vectorized nucleus + top-k sampling.
|
||||
|
||||
Distribution-equivalent to the reference iterative implementation: the
|
||||
keep-set is identical (token i is kept iff
|
||||
``cumsum(sorted_probs)[i] - sorted_probs[i] < top_p`` AND ``i < top_k``)
|
||||
and the renormalized sampling distribution matches, but the exact token
|
||||
drawn for a given seed is NOT guaranteed to match. The reference draws
|
||||
via ``multinomial`` over the stacked kept subset while this draws over
|
||||
the full sorted vector (zeroed outside the keep-set), so the generator
|
||||
advances over different-sized inputs and may yield a different sample.
|
||||
The win: no per-token ``.item()`` D2H syncs from the Python loop —
|
||||
those dominated the sampler CPU time in profiling.
|
||||
"""
|
||||
probs = weighted_scores.softmax(dim=0)
|
||||
sorted_prob, sorted_idx = probs.sort(descending=True, stable=True)
|
||||
kept_probs: list[torch.Tensor] = []
|
||||
kept_indices: list[torch.Tensor] = []
|
||||
cum_prob = 0.0
|
||||
max_keep = len(sorted_idx) if top_k <= 0 else min(int(top_k), len(sorted_idx))
|
||||
for i in range(len(sorted_idx)):
|
||||
if cum_prob < top_p and len(kept_probs) < max_keep:
|
||||
cum_prob += float(sorted_prob[i].item())
|
||||
kept_probs.append(sorted_prob[i])
|
||||
kept_indices.append(sorted_idx[i])
|
||||
else:
|
||||
break
|
||||
|
||||
if not kept_probs:
|
||||
return int(sorted_idx[0].item())
|
||||
|
||||
sample_probs = torch.stack(kept_probs)
|
||||
sample_idx = cls._multinomial_sample(sample_probs, generator=generator)
|
||||
return int(torch.stack(kept_indices)[int(sample_idx.item())].item())
|
||||
cum_before = sorted_prob.cumsum(dim=0) - sorted_prob
|
||||
mask = cum_before < top_p
|
||||
if top_k > 0:
|
||||
n = sorted_prob.shape[0]
|
||||
mask = mask & (torch.arange(n, device=mask.device) < min(int(top_k), n))
|
||||
weights = sorted_prob * mask.to(sorted_prob.dtype)
|
||||
# First token always passes (cum_before[0] = 0 < top_p for any top_p > 0),
|
||||
# so ``weights`` is guaranteed to have at least one nonzero entry. The
|
||||
# final ``.item()`` is the ONLY D2H sync per call.
|
||||
sample_idx = torch.multinomial(weights, 1, replacement=True, generator=generator)
|
||||
return int(sorted_idx[sample_idx].item())
|
||||
|
||||
@classmethod
|
||||
def _ras_sample_one(
|
||||
@@ -627,11 +759,21 @@ class CosyVoice3Model(
|
||||
else:
|
||||
raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.")
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> torch.Tensor:
|
||||
def embed_multimodal(self, **kwargs: object):
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
speech_token = kwargs["speech_token"]
|
||||
speech_token_emb = self.model.speech_embedding(speech_token)
|
||||
return speech_token_emb
|
||||
# vLLM's _execute_mm_encoder batches ALL multimodal items scheduled
|
||||
# in one engine step into a single call, expecting one embedding
|
||||
# tensor per item back. When >=2 requests prefill in the same step
|
||||
# (likely once mm preprocessing is fast, e.g. TensorRT speaker
|
||||
# embedding), speech_token arrives as a list of per-item tensors
|
||||
# (variable length), so embed it per item and return a list of
|
||||
# [T_i, emb] tensors. The single-item path keeps returning a
|
||||
# [1, T, emb] tensor (the caller's extend() iterates dim 0).
|
||||
if isinstance(speech_token, (list, tuple)):
|
||||
emb_dim = self.model.speech_embedding.weight.shape[1]
|
||||
return [self.model.speech_embedding(t).reshape(-1, emb_dim) for t in speech_token]
|
||||
return self.model.speech_embedding(speech_token)
|
||||
else:
|
||||
raise RuntimeError(f"embed_multimodal is only valid for {self.model_stage}.")
|
||||
|
||||
@@ -720,6 +862,121 @@ class CosyVoice3Model(
|
||||
else:
|
||||
raise RuntimeError(f"embed_input_ids is not valid for {self.model_stage}.")
|
||||
|
||||
@staticmethod
|
||||
def _split_prompt_conditioning(speech_token, speech_feat, embedding, speech_token_len):
|
||||
"""Split collated prompt conditioning into per-request lists.
|
||||
|
||||
Returns ``(speech_token_list, speech_feat_list, embedding_list,
|
||||
speech_token_len_list)``, one rank-correct tensor per request:
|
||||
``speech_token`` ``[1, T]`` (possibly right-padded), ``speech_feat``
|
||||
``[1, 2T, F]``, ``embedding`` ``[1, D]``, ``speech_token_len`` ``[1, 1]``.
|
||||
|
||||
This runs inside the talker ``forward`` which may be CUDA-graph
|
||||
captured, so it must NOT trigger any host<->device sync (no ``.item()``
|
||||
/ ``.tolist()`` / Python-int slicing by tensor value). Splitting is pure
|
||||
on-device indexing; right-padding is dropped later in the (eager)
|
||||
code2wav stage using the per-request ``speech_token_len``. Robust to
|
||||
inputs arriving as padded ``[B, ...]`` batch tensors or per-request
|
||||
lists.
|
||||
"""
|
||||
|
||||
def _rows(x, want_dim):
|
||||
if isinstance(x, (list, tuple)):
|
||||
items = list(x)
|
||||
elif isinstance(x, torch.Tensor):
|
||||
# ``x.shape[0]`` is a static Python int — no device sync.
|
||||
items = [x[i] for i in range(x.shape[0])]
|
||||
else:
|
||||
return None
|
||||
out = []
|
||||
for t in items:
|
||||
if isinstance(t, torch.Tensor):
|
||||
while t.dim() < want_dim:
|
||||
t = t.unsqueeze(0)
|
||||
if t.shape[0] != 1:
|
||||
t = t[:1]
|
||||
t = t.contiguous()
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
st_out = _rows(speech_token, 2)
|
||||
sf_out = _rows(speech_feat, 3)
|
||||
emb_out = _rows(embedding, 2)
|
||||
stl_out = _rows(speech_token_len, 2)
|
||||
return st_out, sf_out, emb_out, stl_out
|
||||
|
||||
def _resolve_flow_estimator_onnx(self) -> str | None:
|
||||
"""Locate the flow-decoder estimator ONNX for the TensorRT engine.
|
||||
|
||||
Prefers the fp16 (strongly-typed) ONNX → fp16 engine. Order: env
|
||||
``COSYVOICE3_ESTIMATOR_ONNX`` → ``<model_dir>/<fp16 name>`` → fetched
|
||||
from ``flow_estimator_onnx_repo`` → bundled fp32 ONNX (fp32+TF32). Each
|
||||
is checked for existence; returns ``None`` if nothing is found.
|
||||
"""
|
||||
env_path = os.environ.get("COSYVOICE3_ESTIMATOR_ONNX")
|
||||
if env_path and os.path.exists(env_path):
|
||||
return env_path
|
||||
|
||||
fp16_name = getattr(self.config, "flow_estimator_onnx_path", "flow.decoder.estimator.autocast_fp16.onnx")
|
||||
local_fp16 = os.path.join(self.model_dir, fp16_name)
|
||||
if os.path.exists(local_fp16):
|
||||
return local_fp16
|
||||
|
||||
repo = getattr(self.config, "flow_estimator_onnx_repo", None)
|
||||
if repo:
|
||||
try:
|
||||
fetched_dir = snapshot_download(repo, allow_patterns=[fp16_name])
|
||||
fetched = os.path.join(fetched_dir, fp16_name)
|
||||
if os.path.exists(fetched):
|
||||
return fetched
|
||||
except Exception as exc: # pragma: no cover - network/repo issues
|
||||
logger.warning("CosyVoice3 code2wav: could not fetch fp16 estimator ONNX from %s (%s)", repo, exc)
|
||||
|
||||
fp32_name = getattr(self.config, "flow_estimator_onnx_path_fp32", "flow.decoder.estimator.fp32.onnx")
|
||||
local_fp32 = os.path.join(self.model_dir, fp32_name)
|
||||
if os.path.exists(local_fp32):
|
||||
logger.info("CosyVoice3 code2wav: fp16 estimator ONNX unavailable, falling back to fp32 (%s)", local_fp32)
|
||||
return local_fp32
|
||||
return None
|
||||
|
||||
def _maybe_enable_code2wav_trt(self) -> None:
|
||||
"""Swap the flow-decoder estimator to a TensorRT engine once (lazy).
|
||||
|
||||
Runs on the first code2wav step — after weights are loaded — so the
|
||||
torch estimator is fully built first and then dropped. No-op unless
|
||||
``COSYVOICE3_TRT`` is on (default), CUDA is available, and the estimator
|
||||
ONNX ships with the model. Falls back to the torch estimator on any
|
||||
failure. The upstream ``CausalConditionalCFM.forward_estimator`` detects
|
||||
the non-``nn.Module`` estimator and drives the TRT engine.
|
||||
"""
|
||||
if getattr(self, "_code2wav_trt_done", False):
|
||||
return
|
||||
self._code2wav_trt_done = True
|
||||
if not (_cosyvoice3_trt_enabled() and torch.cuda.is_available()):
|
||||
return
|
||||
onnx_path = self._resolve_flow_estimator_onnx()
|
||||
if onnx_path is None:
|
||||
logger.warning("CosyVoice3 code2wav: no flow-estimator ONNX available; keeping torch estimator")
|
||||
return
|
||||
try:
|
||||
from vllm_omni.model_executor.models.cosyvoice3.flow_estimator_trt import (
|
||||
build_flow_estimator_trt,
|
||||
)
|
||||
|
||||
wrapper = build_flow_estimator_trt(onnx_path, device="cuda")
|
||||
# ``estimator`` is a registered nn.Module submodule; delete it first
|
||||
# (frees the torch estimator weights) so the TRT wrapper can be set
|
||||
# as a plain attribute — nn.Module.__setattr__ rejects non-Modules.
|
||||
decoder = self.code2wav.flow_model.decoder
|
||||
del decoder.estimator
|
||||
decoder.estimator = wrapper
|
||||
logger.info("CosyVoice3: using TensorRT flow-decoder estimator (code2wav)")
|
||||
except Exception as exc: # pragma: no cover - defensive fallback
|
||||
logger.warning(
|
||||
"CosyVoice3 code2wav: TensorRT estimator build failed (%s); keeping torch estimator",
|
||||
exc,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -741,18 +998,43 @@ class CosyVoice3Model(
|
||||
if "speech_token" in kwargs:
|
||||
# Prompt conditioning tensors for code2wav: live under
|
||||
# ``embed.*`` per OmniPayloadStruct schema.
|
||||
#
|
||||
# vLLM hands these mm fields to forward() as collated, padded
|
||||
# batch tensors (speech_token [B, maxT], speech_feat
|
||||
# [B, 2*maxT, F], embedding [B, D]). Emitting them raw makes the
|
||||
# downstream per-request payload split fragile at batch>1: it
|
||||
# intermittently de-batches speech_token to 1-D and leaks the
|
||||
# whole [B, D] embedding to every request, which corrupts voice
|
||||
# conditioning and crashes code2wav (`prompt_token.shape[1]`).
|
||||
# Instead, split into an explicit per-request list of unpadded,
|
||||
# correctly-ranked tensors so ``to_payload_element`` splits them
|
||||
# deterministically by request index (list[idx]).
|
||||
speech_token_list, speech_feat_list, embedding_list, speech_token_len_list = (
|
||||
self._split_prompt_conditioning(
|
||||
kwargs.get("speech_token"),
|
||||
kwargs.get("speech_feat"),
|
||||
kwargs.get("embedding"),
|
||||
kwargs.get("speech_token_len"),
|
||||
)
|
||||
)
|
||||
multimodal_outputs = to_dict(
|
||||
OmniPayloadStruct(
|
||||
embed=EmbeddingsStruct(
|
||||
speech_token=kwargs.get("speech_token"),
|
||||
speech_feat=kwargs.get("speech_feat"),
|
||||
embedding=kwargs.get("embedding"),
|
||||
speech_token=speech_token_list,
|
||||
speech_feat=speech_feat_list,
|
||||
speech_token_len=speech_token_len_list,
|
||||
embedding=embedding_list,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs)
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
# Lazily swap the flow-decoder estimator to a TensorRT engine on the
|
||||
# first code2wav step (after weights are loaded), gated by the same
|
||||
# COSYVOICE3_TRT env toggle as the talker speaker embedding.
|
||||
self._maybe_enable_code2wav_trt()
|
||||
|
||||
runtime_info = kwargs.get("model_intermediate_buffer")
|
||||
if runtime_info is None:
|
||||
runtime_info = kwargs.get("runtime_additional_information", [])
|
||||
@@ -784,6 +1066,11 @@ class CosyVoice3Model(
|
||||
speech_token = embed.speech_token if embed else None
|
||||
speech_feat = embed.speech_feat if embed else None
|
||||
embedding = embed.embedding if embed else None
|
||||
# Drop any right-padding carried from batched talker emission.
|
||||
if speech_token is not None and speech_feat is not None:
|
||||
speech_token, speech_feat = unpad_prompt_conditioning(
|
||||
speech_token, speech_feat, embed.speech_token_len if embed else None
|
||||
)
|
||||
if speech_token is None or speech_feat is None or embedding is None:
|
||||
if stream_finished and req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
|
||||
with self._stream_audio_cache_lock:
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TensorRT engine for the CosyVoice3 flow-decoder (CFM) DiT estimator.
|
||||
|
||||
The estimator is the per-step network the conditional flow-matching ODE solver
|
||||
calls during code2wav (token -> mel). It dominates code2wav latency; the
|
||||
upstream ``CausalConditionalCFM.forward_estimator`` already supports running it
|
||||
through a TensorRT engine (it switches on ``self.estimator`` not being an
|
||||
``nn.Module`` and drives it via ``acquire_estimator`` / ``execute_async_v3``).
|
||||
This module builds that engine from the bundled ``flow.decoder.estimator*.onnx``
|
||||
and wraps it so it can be dropped in for the torch estimator.
|
||||
|
||||
The estimator engine has 6 inputs ``x, mask, mu, t, spks, cond`` and one output.
|
||||
``x/mask/mu/cond`` carry a dynamic time dim; ``t``/``spks`` are fixed, so only
|
||||
the former need an optimization profile (TRT infers the latter from the ONNX).
|
||||
Shapes mirror the CosyVoice runtime.
|
||||
|
||||
Precision: TensorRT >= 11 dropped the weakly-typed FP16/INT8 builder flags, so
|
||||
fp16 only comes from a STRONGLY_TYPED network built from an fp16 ONNX
|
||||
(``*autocast_fp16*``, fp16 I/O). An fp32 ONNX is built fp32 + the TF32 matmul
|
||||
flag. ``EXPLICIT_BATCH`` is implicit (no flag) and ``ITensor.dtype`` is
|
||||
read-only, so neither is set here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import queue
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.speaker_embedding_trt import (
|
||||
_resolve_plan_path,
|
||||
_trt_logger,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Optimization-profile shapes for the dynamic-length inputs (CFG batch dim = 2,
|
||||
# 80 mel channels, time dim min/opt/max). Matches CosyVoice's token2wav.
|
||||
_DYNAMIC_INPUTS = ("x", "mask", "mu", "cond")
|
||||
_MIN_SHAPES = ((2, 80, 4), (2, 1, 4), (2, 80, 4), (2, 80, 4))
|
||||
_OPT_SHAPES = ((2, 80, 500), (2, 1, 500), (2, 80, 500), (2, 80, 500))
|
||||
_MAX_SHAPES = ((2, 80, 3000), (2, 1, 3000), (2, 80, 3000), (2, 80, 3000))
|
||||
|
||||
|
||||
def _is_fp16_onnx(onnx_path: str) -> bool:
|
||||
"""Heuristic: the project's fp16 estimator ONNX is exported strongly-typed
|
||||
and named ``*autocast_fp16*`` / ``*fp16*`` (vs ``*fp32*``)."""
|
||||
name = os.path.basename(onnx_path).lower()
|
||||
return "fp16" in name or "autocast" in name
|
||||
|
||||
|
||||
def _convert_onnx_to_trt(onnx_path: str, plan_path: str, strongly_typed: bool) -> None:
|
||||
import tensorrt as trt
|
||||
|
||||
logger.info(
|
||||
"Building flow-estimator TensorRT engine from %s (%s) ...",
|
||||
onnx_path,
|
||||
"strongly-typed/fp16" if strongly_typed else "fp32+TF32",
|
||||
)
|
||||
trt_logger = _trt_logger()
|
||||
builder = trt.Builder(trt_logger)
|
||||
# STRONGLY_TYPED takes precision from the ONNX graph (fp16 engine from an
|
||||
# fp16 ONNX) — this is the only way to get fp16 in TRT>=11, which dropped
|
||||
# the weakly-typed FP16 BuilderFlag. Otherwise EXPLICIT_BATCH is implicit,
|
||||
# so create the network with no flags.
|
||||
if strongly_typed:
|
||||
network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED))
|
||||
else:
|
||||
network = builder.create_network(0)
|
||||
parser = trt.OnnxParser(network, trt_logger)
|
||||
with open(onnx_path, "rb") as f:
|
||||
if not parser.parse(f.read()):
|
||||
errs = "; ".join(str(parser.get_error(i)) for i in range(parser.num_errors))
|
||||
raise ValueError(f"Failed to parse {onnx_path}: {errs}")
|
||||
|
||||
config = builder.create_builder_config()
|
||||
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 32) # 4 GiB
|
||||
if not strongly_typed:
|
||||
# fp32 ONNX: enable the best available reduced-precision matmul flag
|
||||
# (FP16 on older TRT, else TF32 on Ampere+/Hopper). For a strongly-typed
|
||||
# network the precision is fixed by the graph, so no flag is set.
|
||||
for _flag_name in ("FP16", "TF32"):
|
||||
_flag = getattr(trt.BuilderFlag, _flag_name, None)
|
||||
if _flag is not None:
|
||||
config.set_flag(_flag)
|
||||
break
|
||||
|
||||
profile = builder.create_optimization_profile()
|
||||
for name, mn, op, mx in zip(_DYNAMIC_INPUTS, _MIN_SHAPES, _OPT_SHAPES, _MAX_SHAPES):
|
||||
profile.set_shape(name, mn, op, mx)
|
||||
config.add_optimization_profile(profile)
|
||||
|
||||
engine_bytes = builder.build_serialized_network(network, config)
|
||||
if engine_bytes is None:
|
||||
raise RuntimeError(f"TensorRT failed to build flow-estimator engine from {onnx_path}")
|
||||
tmp = plan_path + ".tmp"
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(engine_bytes)
|
||||
os.replace(tmp, plan_path)
|
||||
logger.info("Wrote flow-estimator TensorRT engine to %s", plan_path)
|
||||
|
||||
|
||||
class TrtContextWrapper:
|
||||
"""Pool of TensorRT execution contexts for the flow estimator.
|
||||
|
||||
Exposes the ``acquire_estimator`` / ``release_estimator`` contract that
|
||||
``CausalConditionalCFM.forward_estimator`` expects.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, engine, device: str | torch.device, io_dtype: torch.dtype = torch.float32, trt_concurrent: int = 1
|
||||
):
|
||||
self.trt_engine = engine
|
||||
# Engine I/O dtype (fp16 for a strongly-typed fp16 engine). The flow runs
|
||||
# in fp32, so forward_estimator casts to/from this at the boundary.
|
||||
self.io_dtype = io_dtype
|
||||
self._pool: queue.Queue = queue.Queue(maxsize=trt_concurrent)
|
||||
for _ in range(trt_concurrent):
|
||||
ctx = engine.create_execution_context()
|
||||
assert ctx is not None, "failed to create TRT execution context (out of memory?)"
|
||||
stream = torch.cuda.stream(torch.cuda.Stream(torch.device(device)))
|
||||
self._pool.put([ctx, stream])
|
||||
|
||||
def acquire_estimator(self):
|
||||
return self._pool.get(), self.trt_engine
|
||||
|
||||
def release_estimator(self, context, stream):
|
||||
self._pool.put([context, stream])
|
||||
|
||||
|
||||
def build_flow_estimator_trt(onnx_path: str, device: str | torch.device) -> TrtContextWrapper:
|
||||
"""Build/load the flow-estimator TRT engine and return a context-pool wrapper.
|
||||
|
||||
An fp16 ONNX (``*autocast_fp16*``) is built as a strongly-typed network (the
|
||||
only way to get fp16 in TRT>=11); an fp32 ONNX is built fp32 + TF32.
|
||||
"""
|
||||
import tensorrt as trt
|
||||
|
||||
strongly_typed = _is_fp16_onnx(onnx_path)
|
||||
plan_path = _resolve_plan_path(onnx_path, prefix="flow_estimator")
|
||||
if not os.path.exists(plan_path) or os.path.getsize(plan_path) == 0:
|
||||
_convert_onnx_to_trt(onnx_path, plan_path, strongly_typed=strongly_typed)
|
||||
|
||||
runtime = trt.Runtime(_trt_logger())
|
||||
with open(plan_path, "rb") as f:
|
||||
engine = runtime.deserialize_cuda_engine(f.read())
|
||||
if engine is None:
|
||||
raise RuntimeError(f"Failed to deserialize flow-estimator TensorRT engine {plan_path}")
|
||||
logger.info("Loaded flow-estimator TensorRT engine (%s)", plan_path)
|
||||
io_dtype = torch.float16 if strongly_typed else torch.float32
|
||||
return TrtContextWrapper(engine, device=device, io_dtype=io_dtype)
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TensorRT speaker-embedding (campplus) extractor for CosyVoice3.
|
||||
|
||||
Drop-in accelerator for the ONNX-Runtime campplus session used in the
|
||||
multimodal processor. ``campplus.onnx`` maps an fbank feature
|
||||
``[B, T, 80]`` to a 192-d speaker embedding ``[B, 192]``; on this stack the
|
||||
ONNX path only has a CPU execution provider, so it dominates per-request
|
||||
prompt-processing latency. Building a TensorRT engine once and running it on
|
||||
GPU removes that bottleneck.
|
||||
|
||||
The engine is built lazily from ``campplus.onnx`` on first use and cached as
|
||||
a ``.plan`` file (engines are device/TRT-version specific, so the cache key
|
||||
includes both). Set ``COSYVOICE3_TRT_CACHE`` to override the cache dir.
|
||||
|
||||
Written against TensorRT >= 10/11: ``EXPLICIT_BATCH`` is the implicit default
|
||||
(create the network with no flags) and ``ITensor.dtype`` is read-only (I/O
|
||||
dtypes come from the ONNX graph), so neither is set here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import threading
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# campplus fbank sequence length bounds for the optimization profile.
|
||||
_MIN_T = 4
|
||||
_OPT_T = 500
|
||||
_MAX_T = 3000
|
||||
_EMBED_DIM = 192
|
||||
|
||||
|
||||
def _trt_logger():
|
||||
import tensorrt as trt
|
||||
|
||||
return trt.Logger(trt.Logger.WARNING)
|
||||
|
||||
|
||||
def _convert_onnx_to_trt(onnx_path: str, plan_path: str) -> None:
|
||||
import tensorrt as trt
|
||||
|
||||
logger.info("Building campplus TensorRT engine from %s ...", onnx_path)
|
||||
trt_logger = _trt_logger()
|
||||
builder = trt.Builder(trt_logger)
|
||||
# EXPLICIT_BATCH is implicit in TRT>=10; create the network with no flags.
|
||||
network = builder.create_network(0)
|
||||
parser = trt.OnnxParser(network, trt_logger)
|
||||
with open(onnx_path, "rb") as f:
|
||||
if not parser.parse(f.read()):
|
||||
errs = "; ".join(str(parser.get_error(i)) for i in range(parser.num_errors))
|
||||
raise ValueError(f"Failed to parse {onnx_path}: {errs}")
|
||||
|
||||
config = builder.create_builder_config()
|
||||
config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 31) # 2 GiB
|
||||
|
||||
input_name = network.get_input(0).name
|
||||
profile = builder.create_optimization_profile()
|
||||
profile.set_shape(
|
||||
input_name,
|
||||
(1, _MIN_T, 80),
|
||||
(1, _OPT_T, 80),
|
||||
(1, _MAX_T, 80),
|
||||
)
|
||||
config.add_optimization_profile(profile)
|
||||
|
||||
engine_bytes = builder.build_serialized_network(network, config)
|
||||
if engine_bytes is None:
|
||||
raise RuntimeError(f"TensorRT failed to build campplus engine from {onnx_path}")
|
||||
tmp = plan_path + ".tmp"
|
||||
with open(tmp, "wb") as f:
|
||||
f.write(engine_bytes)
|
||||
os.replace(tmp, plan_path)
|
||||
logger.info("Wrote campplus TensorRT engine to %s", plan_path)
|
||||
|
||||
|
||||
def _resolve_plan_path(onnx_path: str, prefix: str = "campplus") -> str:
|
||||
cache_dir = os.environ.get("COSYVOICE3_TRT_CACHE") or os.path.join(
|
||||
os.path.expanduser("~"), ".cache", "vllm_omni", "cosyvoice3_trt"
|
||||
)
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
try:
|
||||
# No-arg form targets the current device (avoids the banned
|
||||
# torch.cuda.current_device per ruff TID251).
|
||||
dev_name = torch.cuda.get_device_name()
|
||||
except Exception:
|
||||
dev_name = "unknown"
|
||||
import tensorrt as trt
|
||||
|
||||
st = os.stat(onnx_path)
|
||||
key = f"{os.path.abspath(onnx_path)}|{st.st_size}|{int(st.st_mtime)}|{dev_name}|trt{trt.__version__}"
|
||||
digest = hashlib.sha1(key.encode()).hexdigest()[:16]
|
||||
return os.path.join(cache_dir, f"{prefix}_{digest}.plan")
|
||||
|
||||
|
||||
class CampplusTRT:
|
||||
"""Lazily-built TensorRT campplus speaker-embedding extractor."""
|
||||
|
||||
def __init__(self, onnx_path: str, device: str | torch.device):
|
||||
import tensorrt as trt
|
||||
|
||||
self.device = torch.device(device)
|
||||
plan_path = _resolve_plan_path(onnx_path)
|
||||
if not os.path.exists(plan_path) or os.path.getsize(plan_path) == 0:
|
||||
_convert_onnx_to_trt(onnx_path, plan_path)
|
||||
|
||||
runtime = trt.Runtime(_trt_logger())
|
||||
with open(plan_path, "rb") as f:
|
||||
self.engine = runtime.deserialize_cuda_engine(f.read())
|
||||
if self.engine is None:
|
||||
raise RuntimeError(f"Failed to deserialize campplus TensorRT engine {plan_path}")
|
||||
self.context = self.engine.create_execution_context()
|
||||
self.input_name = self.engine.get_tensor_name(0)
|
||||
self.output_name = self.engine.get_tensor_name(1)
|
||||
# TRT execution contexts are not safe for concurrent enqueue; serialize.
|
||||
self._lock = threading.Lock()
|
||||
logger.info("Loaded campplus TensorRT engine (%s)", plan_path)
|
||||
|
||||
@torch.inference_mode()
|
||||
def __call__(self, feat: torch.Tensor) -> torch.Tensor:
|
||||
"""feat: ``[T, 80]`` fbank features. Returns ``[1, 192]`` embedding."""
|
||||
x = feat.unsqueeze(0).to(self.device, dtype=torch.float32).contiguous()
|
||||
out = torch.empty((x.shape[0], _EMBED_DIM), device=self.device, dtype=torch.float32)
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
with self._lock:
|
||||
self.context.set_input_shape(self.input_name, tuple(x.shape))
|
||||
self.context.set_tensor_address(self.input_name, x.data_ptr())
|
||||
self.context.set_tensor_address(self.output_name, out.data_ptr())
|
||||
ok = self.context.execute_async_v3(stream.cuda_stream)
|
||||
if not ok:
|
||||
raise RuntimeError("campplus TensorRT execute_async_v3 failed")
|
||||
stream.synchronize()
|
||||
return out
|
||||
|
||||
|
||||
# Process-wide cache: the mm processor is re-created per request (the deploy
|
||||
# config disables the mm-processor object cache), so building a fresh
|
||||
# CampplusTRT each time would re-deserialize the ~33 MB engine and allocate a
|
||||
# new execution context on every request — inflating TTFP. Cache by
|
||||
# (engine path, device) and reuse; the context serializes concurrent enqueues
|
||||
# via its own lock, so a single shared instance is safe across requests.
|
||||
_CAMPPLUS_CACHE: dict[tuple[str, str], CampplusTRT] = {}
|
||||
|
||||
|
||||
def get_campplus_trt(onnx_path: str, device: str | torch.device) -> CampplusTRT:
|
||||
"""Return a process-wide cached :class:`CampplusTRT` (engine + context)."""
|
||||
key = (os.path.abspath(onnx_path), str(device))
|
||||
inst = _CAMPPLUS_CACHE.get(key)
|
||||
if inst is None:
|
||||
inst = CampplusTRT(onnx_path, device)
|
||||
_CAMPPLUS_CACHE[key] = inst
|
||||
return inst
|
||||
@@ -215,6 +215,44 @@ def extract_spk_embedding(prompt_wav, campplus_session, device):
|
||||
return embedding
|
||||
|
||||
|
||||
def unpad_prompt_conditioning(speech_token, speech_feat, speech_token_len):
|
||||
"""Drop right-padding from per-request prompt conditioning.
|
||||
|
||||
The talker emits ``speech_token`` ``[1, T]`` possibly right-padded to the
|
||||
batch max, plus the true ``speech_token_len``; ``speech_feat`` is at the 2:1
|
||||
mel:token ratio (``[1, 2T, F]``). This trims both to the real length. Called
|
||||
in the (eager) code2wav stage where a host read of the length is allowed.
|
||||
Returns ``(speech_token, speech_feat)`` unchanged when length is unavailable.
|
||||
"""
|
||||
if not isinstance(speech_token, torch.Tensor) or speech_token_len is None:
|
||||
return speech_token, speech_feat
|
||||
try:
|
||||
token_len = int(torch.as_tensor(speech_token_len).reshape(-1)[0].item())
|
||||
except (RuntimeError, ValueError, IndexError):
|
||||
return speech_token, speech_feat
|
||||
if token_len <= 0 or speech_token.dim() < 2:
|
||||
return speech_token, speech_feat
|
||||
token_len = min(token_len, int(speech_token.shape[1]))
|
||||
speech_token = speech_token[:, :token_len]
|
||||
if isinstance(speech_feat, torch.Tensor) and speech_feat.dim() >= 2:
|
||||
speech_feat = speech_feat[:, : 2 * token_len]
|
||||
return speech_token, speech_feat
|
||||
|
||||
|
||||
def extract_spk_embedding_trt(prompt_wav, campplus_trt, device):
|
||||
"""TensorRT counterpart of ``extract_spk_embedding``.
|
||||
|
||||
Identical fbank front-end; the campplus forward runs on a prebuilt
|
||||
TensorRT engine (GPU) instead of the CPU ONNX-Runtime session. Returns the
|
||||
same ``[1, 192]`` embedding tensor on ``device``.
|
||||
"""
|
||||
speech = load_wav(prompt_wav, 16000)
|
||||
feat = kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000)
|
||||
feat = feat - feat.mean(dim=0, keepdim=True)
|
||||
embedding = campplus_trt(feat)
|
||||
return embedding.to(device)
|
||||
|
||||
|
||||
def extract_text_token(text, tokenizer, allowed_special):
|
||||
text_token = tokenizer.encode(text, allowed_special=allowed_special)
|
||||
text_token = torch.tensor([text_token], dtype=torch.int32)
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm_omni.data_entry_keys import (
|
||||
OmniPayloadStruct,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniTokensPrompt
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import unpad_prompt_conditioning
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -192,18 +193,31 @@ def talker2code2wav_async_chunk(
|
||||
info = _decode_additional_information(getattr(request, "additional_information", None))
|
||||
info_embed = info.get("embed", {}) if isinstance(info, dict) else {}
|
||||
prompt_payload = {}
|
||||
for key in ("speech_token", "speech_feat", "embedding"):
|
||||
cond_keys = ("speech_token", "speech_feat", "embedding", "speech_token_len")
|
||||
for key in cond_keys:
|
||||
value = _to_cpu_tensor(info_embed.get(key))
|
||||
if value is not None:
|
||||
prompt_payload[key] = value
|
||||
if isinstance(pooling_output, dict):
|
||||
po_embed = pooling_output.get("embed", {}) if isinstance(pooling_output.get("embed"), dict) else {}
|
||||
for key in ("speech_token", "speech_feat", "embedding"):
|
||||
for key in cond_keys:
|
||||
if key in prompt_payload:
|
||||
continue
|
||||
value = _to_cpu_tensor(po_embed.get(key))
|
||||
if value is not None:
|
||||
prompt_payload[key] = value
|
||||
# Drop any right-padding carried from batched talker emission so
|
||||
# the chunk-routing math (prompt_token_len) and the conditioning
|
||||
# sent to code2wav use the true prompt length.
|
||||
if "speech_token" in prompt_payload:
|
||||
st_unpad, sf_unpad = unpad_prompt_conditioning(
|
||||
prompt_payload.get("speech_token"),
|
||||
prompt_payload.get("speech_feat"),
|
||||
prompt_payload.pop("speech_token_len", None),
|
||||
)
|
||||
prompt_payload["speech_token"] = st_unpad
|
||||
if sf_unpad is not None:
|
||||
prompt_payload["speech_feat"] = sf_unpad
|
||||
prompt_token = prompt_payload.get("speech_token")
|
||||
prompt_token_len = (
|
||||
int(prompt_token.shape[1])
|
||||
|
||||
@@ -42,6 +42,13 @@ class CosyVoice3Config(PretrainedConfig):
|
||||
self.qwen_pretrain_path = "CosyVoice-BlankEN"
|
||||
self.campplus_onxx_path = "campplus.onnx"
|
||||
self.speech_tokenizer_path = "speech_tokenizer_v3.onnx"
|
||||
# Flow-decoder estimator ONNX for the optional TensorRT code2wav path.
|
||||
# The fp16 (strongly-typed) ONNX gives an fp16 engine; if it is not in
|
||||
# the model dir it is fetched from ``flow_estimator_onnx_repo``. Falls
|
||||
# back to the bundled fp32 ONNX (fp32+TF32 engine) when unavailable.
|
||||
self.flow_estimator_onnx_path = "flow.decoder.estimator.autocast_fp16.onnx"
|
||||
self.flow_estimator_onnx_repo = "yuekai/Fun-CosyVoice3-0.5B-2512-FP16-ONNX"
|
||||
self.flow_estimator_onnx_path_fp32 = "flow.decoder.estimator.fp32.onnx"
|
||||
self.spk2info_path = "spk2info.pt"
|
||||
self.version = "cosyvoice3"
|
||||
self.llm = {
|
||||
|
||||
Reference in new issue
Block a user