From f1bf5442085e2517177b269246f05cdf547fea61 Mon Sep 17 00:00:00 2001 From: Yuekai Zhang Date: Wed, 10 Jun 2026 16:42:34 +0800 Subject: [PATCH] [Perf][TTS] Optimize cosyvoice TTFP and throughput using TensorRT (#4168) Signed-off-by: Yuekai Zhang Co-authored-by: Claude Opus 4.8 (1M context) --- .../test_cosyvoice3_model_helpers.py | 5 + vllm_omni/data_entry_keys.py | 1 + vllm_omni/deploy/cosyvoice3.yaml | 27 +- .../entrypoints/openai/serving_speech.py | 28 +- .../models/cosyvoice3/code2wav_core/cfm.py | 37 +- .../models/cosyvoice3/cosyvoice3.py | 401 +++++++++++++++--- .../models/cosyvoice3/flow_estimator_trt.py | 154 +++++++ .../cosyvoice3/speaker_embedding_trt.py | 157 +++++++ .../model_executor/models/cosyvoice3/utils.py | 38 ++ .../stage_input_processors/cosyvoice3.py | 18 +- .../transformers_utils/configs/cosyvoice3.py | 7 + 11 files changed, 786 insertions(+), 87 deletions(-) create mode 100644 vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py create mode 100644 vllm_omni/model_executor/models/cosyvoice3/speaker_embedding_trt.py diff --git a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_model_helpers.py b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_model_helpers.py index b0afc9592..a2c3142c7 100644 --- a/tests/model_executor/models/cosyvoice3/test_cosyvoice3_model_helpers.py +++ b/tests/model_executor/models/cosyvoice3/test_cosyvoice3_model_helpers.py @@ -84,6 +84,11 @@ def _make_code2wav_model( token_mel_ratio=2 if with_stride_cfg else 0, ) model.code2wav = _DummyCode2Wav(vocab_size=4, num_samples=num_samples, outputs=outputs) + # Short-circuit the lazy TensorRT estimator swap: these tests exercise the + # forward audio logic, not the TRT path. On a GPU CI runner the swap would + # otherwise run and dereference ``self.model_dir`` (only set in __init__, + # which this fixture bypasses via object.__new__). + model._code2wav_trt_done = True model.source_cache_len = 4 model.speech_window = torch.hamming_window(8, periodic=False) model._stream_audio_cache_by_req = {} diff --git a/vllm_omni/data_entry_keys.py b/vllm_omni/data_entry_keys.py index 2584a7242..eec3e5c33 100644 --- a/vllm_omni/data_entry_keys.py +++ b/vllm_omni/data_entry_keys.py @@ -128,6 +128,7 @@ class EmbeddingsStruct(_StructBase): voice: torch.Tensor | None = None speech_feat: torch.Tensor | None = None speech_token: torch.Tensor | None = None + speech_token_len: torch.Tensor | None = None embedding: torch.Tensor | None = None thinker_reply: torch.Tensor | None = None diff --git a/vllm_omni/deploy/cosyvoice3.yaml b/vllm_omni/deploy/cosyvoice3.yaml index 4bfd4ab85..d81cf4a9b 100644 --- a/vllm_omni/deploy/cosyvoice3.yaml +++ b/vllm_omni/deploy/cosyvoice3.yaml @@ -7,9 +7,22 @@ # references stay in the yaml unconditionally; the runtime only activates # them when ``async_chunk: true``. # -# enforce_eager=true everywhere: -# * Stage 0 (talker) — cudagraph not verified on this checkpoint. -# * Stage 1 (code2wav) — CUDA graphs don't work with dynamic conv shapes. +# enforce_eager: +# * Stage 0 (talker) — enforce_eager: false (CUDA graphs enabled). +# * Stage 1 (code2wav) — enforce_eager: true (CUDA graphs don't work with +# dynamic conv shapes). +# +# Streaming continuity is GPU-class sensitive — TUNE PER GPU. +# The defaults below (max_num_seqs: 8, codec_chunk_frames: 15) are tuned for +# H100, where streaming continuity at concurrency 4 stays ~100%. They push +# more audio per scheduler step through the SharedMemoryConnector than a +# slower GPU can keep up with: on H20-3e the same config regressed continuity +# to ~78% at c=4 (vs ~86% with smaller batch/chunk), because code2wav can't +# produce mel chunks fast enough to feed the streaming output → buffer +# underruns → audible gaps. +# On a slower GPU class, lower max_num_seqs (e.g. 4) and/or codec_chunk_frames +# first. Raising connector_get_sleep_s also relaxes the consumer's polling +# cadence, but trades against TTFP — keep it as a last resort. async_chunk: true dtype: float32 @@ -21,16 +34,16 @@ connectors: connector_get_sleep_s: 0.01 connector_get_max_wait_first_chunk: 3000 connector_get_max_wait: 300 - codec_chunk_frames: 25 + codec_chunk_frames: 15 codec_left_context_frames: 25 codec_vocab_size: 6561 stages: - stage_id: 0 max_num_batched_tokens: 32768 - max_num_seqs: 1 + max_num_seqs: 8 gpu_memory_utilization: 0.4 - enforce_eager: true + enforce_eager: false trust_remote_code: true enable_prefix_caching: false devices: "0" @@ -49,7 +62,7 @@ stages: - stage_id: 1 max_num_batched_tokens: 32768 - max_num_seqs: 1 + max_num_seqs: 8 gpu_memory_utilization: 0.2 enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 1857c6fc8..c5db707d6 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -410,6 +410,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ) self._fish_speech_tokenizer = None self._covo_audio_tokenizer = None + # Cached per process: the CosyVoice3 Qwen tokenizer + resolved model + # path used for dynamic-token sizing. Without this, every request + # re-ran snapshot_download + reloaded the tokenizer (~100 ms on the + # TTFP critical path) in _apply_cosyvoice3_dynamic_tokens. + self._cosyvoice3_tokenizer = None self._is_cosyvoice3 = ( self._tts_stage is not None @@ -2658,16 +2663,21 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): sampling_params_list = copy.deepcopy(sampling_params_list) hf_cfg = self.model_config.hf_config - model_path = self.engine_client.model_config.model - if not os.path.isdir(model_path): - from huggingface_hub import snapshot_download + # Build the Qwen tokenizer once per process (resolving the model dir via + # snapshot_download at most once) and reuse it across requests. + tokenizer = self._cosyvoice3_tokenizer + if tokenizer is None: + model_path = self.engine_client.model_config.model + if not os.path.isdir(model_path): + from huggingface_hub import snapshot_download - model_path = snapshot_download(model_path) - tokenizer = get_qwen_tokenizer( - token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path), - skip_special_tokens=hf_cfg.skip_special_tokens, - version=hf_cfg.version, - ) + model_path = snapshot_download(model_path) + tokenizer = get_qwen_tokenizer( + token_path=os.path.join(model_path, hf_cfg.qwen_pretrain_path), + skip_special_tokens=hf_cfg.skip_special_tokens, + version=hf_cfg.version, + ) + self._cosyvoice3_tokenizer = tokenizer _, text_token_len = extract_text_token( request.input, tokenizer, diff --git a/vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py b/vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py index 36ff0d456..78c1ea66f 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py +++ b/vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py @@ -141,24 +141,37 @@ class ConditionalCFM(BASECFM): if isinstance(self.estimator, torch.nn.Module): return self.estimator(x, mask, mu, t, spks, cond) else: + # TensorRT estimator: bind raw device pointers. The flow runs in + # fp32 but the engine may have fp16 I/O (strongly-typed fp16 engine), + # so cast inputs/output to the engine's dtype at the boundary. Keep + # references to the cast buffers alive until execute completes (a bare + # ``.contiguous().data_ptr()`` could free the temp -> dangling ptr). + io_dtype = getattr(self.estimator, "io_dtype", x.dtype) [estimator, stream], trt_engine = self.estimator.acquire_estimator() # NOTE need to synchronize when switching stream torch.cuda.current_stream().synchronize() with stream: - estimator.set_input_shape("x", (2, 80, x.size(2))) - estimator.set_input_shape("mask", (2, 1, x.size(2))) - estimator.set_input_shape("mu", (2, 80, x.size(2))) + x_e = x.to(io_dtype).contiguous() + mask_e = mask.to(io_dtype).contiguous() + mu_e = mu.to(io_dtype).contiguous() + t_e = t.to(io_dtype).contiguous() + spks_e = spks.to(io_dtype).contiguous() + cond_e = cond.to(io_dtype).contiguous() + out_e = torch.empty_like(x_e) + estimator.set_input_shape("x", (2, 80, x_e.size(2))) + estimator.set_input_shape("mask", (2, 1, x_e.size(2))) + estimator.set_input_shape("mu", (2, 80, x_e.size(2))) estimator.set_input_shape("t", (2,)) estimator.set_input_shape("spks", (2, 80)) - estimator.set_input_shape("cond", (2, 80, x.size(2))) + estimator.set_input_shape("cond", (2, 80, x_e.size(2))) data_ptrs = [ - x.contiguous().data_ptr(), - mask.contiguous().data_ptr(), - mu.contiguous().data_ptr(), - t.contiguous().data_ptr(), - spks.contiguous().data_ptr(), - cond.contiguous().data_ptr(), - x.data_ptr(), + x_e.data_ptr(), + mask_e.data_ptr(), + mu_e.data_ptr(), + t_e.data_ptr(), + spks_e.data_ptr(), + cond_e.data_ptr(), + out_e.data_ptr(), ] for i, j in enumerate(data_ptrs): estimator.set_tensor_address(trt_engine.get_tensor_name(i), j) @@ -166,7 +179,7 @@ class ConditionalCFM(BASECFM): assert estimator.execute_async_v3(torch.cuda.current_stream().cuda_stream) is True torch.cuda.current_stream().synchronize() self.estimator.release_estimator(estimator, stream) - return x + return out_e.to(x.dtype) class CausalConditionalCFM(ConditionalCFM): diff --git a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py index d20312a97..1a387cdcc 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py +++ b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py @@ -3,10 +3,16 @@ import os from collections.abc import Iterable, Mapping, Sequence from functools import partial +from math import gcd from threading import Lock +import numpy as np +import onnxruntime import torch import torch.nn as nn +from huggingface_hub import snapshot_download +from scipy.signal import resample_poly +from transformers import Qwen2Config from transformers.feature_extraction_utils import BatchFeature from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions @@ -32,12 +38,15 @@ from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.sample.sampler import Sampler from vllm_omni.data_entry_keys import EmbeddingsStruct, OmniPayloadStruct, to_dict, to_struct +from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer from vllm_omni.model_executor.models.cosyvoice3.utils import ( concat_text_with_prompt_ids, extract_speech_feat, - extract_speech_token, extract_spk_embedding, + extract_spk_embedding_trt, extract_text_token, + mel_spectrogram, + unpad_prompt_conditioning, ) from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config @@ -45,6 +54,23 @@ from vllm_omni.utils.speaker_cache import get_speaker_cache logger = init_logger(__name__) +# Process-wide cache of per-model mm-processor runtime components (tokenizer, +# feat_extractor, campplus session/engine). The mm processor is re-created per +# request (mm_processor_cache_gb: 0), so this avoids rebuilding them every time. +_RUNTIME_COMPONENTS_CACHE: dict[str, dict] = {} + + +def _cosyvoice3_trt_enabled() -> bool: + """COSYVOICE3_TRT env toggle (default on) for the optional TensorRT paths. + + Gates both the talker speaker-embedding (campplus) engine and the code2wav + flow-decoder estimator engine. Env-var toggle (cf. mimo_audio's + ``MIMO_AUDIO_TOKENIZER_CUDA_GRAPH``) because these run outside the stage + worker, so deploy-yaml ``hf_overrides`` / per-stage ``env`` do not reach + them — export ``COSYVOICE3_TRT=0`` in the launching shell to disable. + """ + return os.environ.get("COSYVOICE3_TRT", "1") not in ("0", "false", "False", "") + class CosyVoice3MultiModalProcessingInfo(BaseProcessingInfo): def get_hf_config(self): @@ -73,39 +99,136 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal if cached_model_dir == model_dir: return - # If model_dir is an HF repo ID (not a local path), resolve to cache - if not os.path.isdir(model_dir): - from huggingface_hub import snapshot_download + # The mm processor is re-created per request (deploy config sets + # ``mm_processor_cache_gb: 0``), so the instance-level guard above never + # hits across requests. Without a process-wide cache, every request + # would re-run ``snapshot_download``, rebuild the Qwen tokenizer and + # create a fresh ONNX campplus session — ~hundreds of ms of pure + # overhead on the TTFP critical path. Build the heavy components once + # per process, keyed by model_dir, and reuse them. + comps = _RUNTIME_COMPONENTS_CACHE.get(model_dir) + if comps is None: + comps = self._build_runtime_components(model_dir, config) + _RUNTIME_COMPONENTS_CACHE[model_dir] = comps + self.tokenizer = comps["tokenizer"] + self.feat_extractor = comps["feat_extractor"] + self.campplus_session = comps["campplus_session"] + self.campplus_trt = comps["campplus_trt"] + self._cached_model_dir = model_dir + self._speaker_cache = get_speaker_cache() + + def _build_runtime_components(self, model_dir: str, config: CosyVoice3Config) -> dict: + """Build the per-model runtime components once (cached process-wide).""" + # If model_dir is an HF repo ID (not a local path), resolve to cache. + if not os.path.isdir(model_dir): model_dir = snapshot_download(model_dir) - import onnxruntime - - from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer - from vllm_omni.model_executor.models.cosyvoice3.utils import mel_spectrogram - - option = onnxruntime.SessionOptions() - option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - option.intra_op_num_threads = 1 - - self.tokenizer = get_qwen_tokenizer( + tokenizer = get_qwen_tokenizer( token_path=os.path.join(model_dir, config.qwen_pretrain_path), skip_special_tokens=config.skip_special_tokens, version=config.version, ) - self.speech_tokenizer = onnxruntime.InferenceSession( - os.path.join(model_dir, config.speech_tokenizer_path), - sess_options=option, - providers=["CUDAExecutionProvider" if torch.cuda.is_available() else "CPUExecutionProvider"], - ) - self.feat_extractor = partial(mel_spectrogram, **getattr(config, "feat_extractor", {})) - self.campplus_session = onnxruntime.InferenceSession( - os.path.join(model_dir, config.campplus_onxx_path), - sess_options=option, - providers=["CPUExecutionProvider"], - ) - self._cached_model_dir = model_dir - self._speaker_cache = get_speaker_cache() + feat_extractor = partial(mel_spectrogram, **getattr(config, "feat_extractor", {})) + + campplus_onnx_path = os.path.join(model_dir, config.campplus_onxx_path) + # TensorRT speaker-embedding path (default on); a prebuilt TRT engine + # runs campplus on GPU. The engine itself is cached process-wide by + # ``get_campplus_trt``. + campplus_trt = None + if self._speaker_embedding_trt_enabled() and torch.cuda.is_available(): + try: + from vllm_omni.model_executor.models.cosyvoice3.speaker_embedding_trt import ( + get_campplus_trt, + ) + + campplus_trt = get_campplus_trt(campplus_onnx_path, device="cuda") + logger.info("CosyVoice3: using TensorRT campplus speaker embedding") + except Exception as exc: # pragma: no cover - defensive fallback + logger.warning( + "CosyVoice3: TensorRT campplus build failed (%s); falling back to ONNX-Runtime speaker embedding", + exc, + ) + campplus_trt = None + + # Only build the CPU ONNX campplus session when TRT is unavailable — + # otherwise it is never used and creating it (ORT_ENABLE_ALL graph + # optimization) costs ~hundreds of ms. + campplus_session = None + if campplus_trt is None: + option = onnxruntime.SessionOptions() + option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + option.intra_op_num_threads = 1 + campplus_session = onnxruntime.InferenceSession( + campplus_onnx_path, + sess_options=option, + providers=["CPUExecutionProvider"], + ) + + return { + "tokenizer": tokenizer, + "feat_extractor": feat_extractor, + "campplus_session": campplus_session, + "campplus_trt": campplus_trt, + } + + @staticmethod + def _speaker_embedding_trt_enabled() -> bool: + """Whether to use the TensorRT campplus speaker-embedding path (default on).""" + return _cosyvoice3_trt_enabled() + + # Class-level cached s3tokenizer model — loaded once per process on first + # call to ``_extract_speech_token_via_s3`` and shared across all + # processor instances. + _s3_model = None + + @classmethod + def _ensure_s3_model(cls): + if cls._s3_model is not None: + return cls._s3_model + + # s3tokenizer is imported lazily (kept off the module top level) so + # callers that don't use the CosyVoice3 talker need not install it. + try: + import s3tokenizer as _s3 + except ImportError as e: + raise ImportError( + "CosyVoice3 speech-token extraction requires the 's3tokenizer' " + "package; install it with `pip install s3tokenizer`." + ) from e + + model = _s3.load_model("speech_tokenizer_v3_25hz") + device = "cuda" if torch.cuda.is_available() else "cpu" + model = model.to(device).eval() + cls._s3_model = (model, _s3, device) + return cls._s3_model + + def _extract_speech_token_via_s3(self, audio, return_device): + """Drop-in replacement for ``extract_speech_token`` that uses the + S3Tokenizer PyTorch model on GPU. Returns the same + ``(speech_token[1, T], speech_token_len[1])`` int32 tensors as the + ONNX path so the rest of ``_call_hf_processor`` is unchanged. + """ + model, _s3, dev = self._ensure_s3_model() + + # audio is a (waveform_ndarray, sr) tuple; resample to 16 kHz mono float32. + wav, sr = audio + wav = np.asarray(wav, dtype=np.float32) + if wav.ndim == 2: + wav = wav.mean(axis=1) + if int(sr) != 16000: + g = gcd(int(sr), 16000) + wav = resample_poly(wav, 16000 // g, int(sr) // g).astype(np.float32) + audio_t = torch.from_numpy(wav) + + mel = _s3.log_mel_spectrogram(audio_t) + mels_p, mels_lens = _s3.padding([mel]) + with torch.inference_mode(): + codes, codes_lens = model.quantize(mels_p.to(dev), mels_lens.to(dev)) + n = int(codes_lens[0].item()) + speech_token = codes[:1, :n].to(dtype=torch.int32, device=return_device) + speech_token_len = torch.tensor([n], dtype=torch.int32, device=return_device) + return speech_token, speech_token_len def _call_hf_processor( self, @@ -185,7 +308,11 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal ) return ft - speech_token, speech_token_len = extract_speech_token(audio, self.speech_tokenizer, device) + # Speech-token extraction via the S3Tokenizer PyTorch model on GPU + # (~30x faster than the bundled ``speech_tokenizer_v3.onnx`` CPU ONNX + # path in this venv). + speech_token, speech_token_len = self._extract_speech_token_via_s3(audio, device) + speech_feat, speech_feat_len = extract_speech_feat(audio, self.feat_extractor, device) if config.sample_rate == 24000: @@ -193,7 +320,10 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal speech_feat, speech_feat_len[:] = speech_feat[:, : 2 * token_len], 2 * token_len speech_token, speech_token_len[:] = speech_token[:, :token_len], token_len - embedding = extract_spk_embedding(audio, self.campplus_session, device) + if self.campplus_trt is not None: + embedding = extract_spk_embedding_trt(audio, self.campplus_trt, device) + else: + embedding = extract_spk_embedding(audio, self.campplus_session, device) # Cache the extracted artifacts for named speakers if cache_key is not None: @@ -318,8 +448,6 @@ class CosyVoice3Model( self.model_stage = vllm_config.model_config.model_stage model_dir = vllm_config.model_config.model if not os.path.isdir(model_dir): - from huggingface_hub import snapshot_download - model_dir = snapshot_download(model_dir) self.model_dir = model_dir self.model = None @@ -377,8 +505,6 @@ class CosyVoice3Model( the pretrained model directory. The cache config is inherited from the parent to enable PagedAttention with the same memory configuration. """ - from transformers import Qwen2Config - qwen_config_path = os.path.join(self.model_dir, self.config.llm["llm"]["pretrain_path"]) qwen_hf_config = Qwen2Config.from_pretrained(qwen_config_path) @@ -469,26 +595,32 @@ class CosyVoice3Model( top_k: int, generator: torch.Generator | None, ) -> int: + """Vectorized nucleus + top-k sampling. + + Distribution-equivalent to the reference iterative implementation: the + keep-set is identical (token i is kept iff + ``cumsum(sorted_probs)[i] - sorted_probs[i] < top_p`` AND ``i < top_k``) + and the renormalized sampling distribution matches, but the exact token + drawn for a given seed is NOT guaranteed to match. The reference draws + via ``multinomial`` over the stacked kept subset while this draws over + the full sorted vector (zeroed outside the keep-set), so the generator + advances over different-sized inputs and may yield a different sample. + The win: no per-token ``.item()`` D2H syncs from the Python loop — + those dominated the sampler CPU time in profiling. + """ probs = weighted_scores.softmax(dim=0) sorted_prob, sorted_idx = probs.sort(descending=True, stable=True) - kept_probs: list[torch.Tensor] = [] - kept_indices: list[torch.Tensor] = [] - cum_prob = 0.0 - max_keep = len(sorted_idx) if top_k <= 0 else min(int(top_k), len(sorted_idx)) - for i in range(len(sorted_idx)): - if cum_prob < top_p and len(kept_probs) < max_keep: - cum_prob += float(sorted_prob[i].item()) - kept_probs.append(sorted_prob[i]) - kept_indices.append(sorted_idx[i]) - else: - break - - if not kept_probs: - return int(sorted_idx[0].item()) - - sample_probs = torch.stack(kept_probs) - sample_idx = cls._multinomial_sample(sample_probs, generator=generator) - return int(torch.stack(kept_indices)[int(sample_idx.item())].item()) + cum_before = sorted_prob.cumsum(dim=0) - sorted_prob + mask = cum_before < top_p + if top_k > 0: + n = sorted_prob.shape[0] + mask = mask & (torch.arange(n, device=mask.device) < min(int(top_k), n)) + weights = sorted_prob * mask.to(sorted_prob.dtype) + # First token always passes (cum_before[0] = 0 < top_p for any top_p > 0), + # so ``weights`` is guaranteed to have at least one nonzero entry. The + # final ``.item()`` is the ONLY D2H sync per call. + sample_idx = torch.multinomial(weights, 1, replacement=True, generator=generator) + return int(sorted_idx[sample_idx].item()) @classmethod def _ras_sample_one( @@ -627,11 +759,21 @@ class CosyVoice3Model( else: raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.") - def embed_multimodal(self, **kwargs: object) -> torch.Tensor: + def embed_multimodal(self, **kwargs: object): if self.model_stage == "cosyvoice3_talker": speech_token = kwargs["speech_token"] - speech_token_emb = self.model.speech_embedding(speech_token) - return speech_token_emb + # vLLM's _execute_mm_encoder batches ALL multimodal items scheduled + # in one engine step into a single call, expecting one embedding + # tensor per item back. When >=2 requests prefill in the same step + # (likely once mm preprocessing is fast, e.g. TensorRT speaker + # embedding), speech_token arrives as a list of per-item tensors + # (variable length), so embed it per item and return a list of + # [T_i, emb] tensors. The single-item path keeps returning a + # [1, T, emb] tensor (the caller's extend() iterates dim 0). + if isinstance(speech_token, (list, tuple)): + emb_dim = self.model.speech_embedding.weight.shape[1] + return [self.model.speech_embedding(t).reshape(-1, emb_dim) for t in speech_token] + return self.model.speech_embedding(speech_token) else: raise RuntimeError(f"embed_multimodal is only valid for {self.model_stage}.") @@ -720,6 +862,121 @@ class CosyVoice3Model( else: raise RuntimeError(f"embed_input_ids is not valid for {self.model_stage}.") + @staticmethod + def _split_prompt_conditioning(speech_token, speech_feat, embedding, speech_token_len): + """Split collated prompt conditioning into per-request lists. + + Returns ``(speech_token_list, speech_feat_list, embedding_list, + speech_token_len_list)``, one rank-correct tensor per request: + ``speech_token`` ``[1, T]`` (possibly right-padded), ``speech_feat`` + ``[1, 2T, F]``, ``embedding`` ``[1, D]``, ``speech_token_len`` ``[1, 1]``. + + This runs inside the talker ``forward`` which may be CUDA-graph + captured, so it must NOT trigger any host<->device sync (no ``.item()`` + / ``.tolist()`` / Python-int slicing by tensor value). Splitting is pure + on-device indexing; right-padding is dropped later in the (eager) + code2wav stage using the per-request ``speech_token_len``. Robust to + inputs arriving as padded ``[B, ...]`` batch tensors or per-request + lists. + """ + + def _rows(x, want_dim): + if isinstance(x, (list, tuple)): + items = list(x) + elif isinstance(x, torch.Tensor): + # ``x.shape[0]`` is a static Python int — no device sync. + items = [x[i] for i in range(x.shape[0])] + else: + return None + out = [] + for t in items: + if isinstance(t, torch.Tensor): + while t.dim() < want_dim: + t = t.unsqueeze(0) + if t.shape[0] != 1: + t = t[:1] + t = t.contiguous() + out.append(t) + return out + + st_out = _rows(speech_token, 2) + sf_out = _rows(speech_feat, 3) + emb_out = _rows(embedding, 2) + stl_out = _rows(speech_token_len, 2) + return st_out, sf_out, emb_out, stl_out + + def _resolve_flow_estimator_onnx(self) -> str | None: + """Locate the flow-decoder estimator ONNX for the TensorRT engine. + + Prefers the fp16 (strongly-typed) ONNX → fp16 engine. Order: env + ``COSYVOICE3_ESTIMATOR_ONNX`` → ``/`` → fetched + from ``flow_estimator_onnx_repo`` → bundled fp32 ONNX (fp32+TF32). Each + is checked for existence; returns ``None`` if nothing is found. + """ + env_path = os.environ.get("COSYVOICE3_ESTIMATOR_ONNX") + if env_path and os.path.exists(env_path): + return env_path + + fp16_name = getattr(self.config, "flow_estimator_onnx_path", "flow.decoder.estimator.autocast_fp16.onnx") + local_fp16 = os.path.join(self.model_dir, fp16_name) + if os.path.exists(local_fp16): + return local_fp16 + + repo = getattr(self.config, "flow_estimator_onnx_repo", None) + if repo: + try: + fetched_dir = snapshot_download(repo, allow_patterns=[fp16_name]) + fetched = os.path.join(fetched_dir, fp16_name) + if os.path.exists(fetched): + return fetched + except Exception as exc: # pragma: no cover - network/repo issues + logger.warning("CosyVoice3 code2wav: could not fetch fp16 estimator ONNX from %s (%s)", repo, exc) + + fp32_name = getattr(self.config, "flow_estimator_onnx_path_fp32", "flow.decoder.estimator.fp32.onnx") + local_fp32 = os.path.join(self.model_dir, fp32_name) + if os.path.exists(local_fp32): + logger.info("CosyVoice3 code2wav: fp16 estimator ONNX unavailable, falling back to fp32 (%s)", local_fp32) + return local_fp32 + return None + + def _maybe_enable_code2wav_trt(self) -> None: + """Swap the flow-decoder estimator to a TensorRT engine once (lazy). + + Runs on the first code2wav step — after weights are loaded — so the + torch estimator is fully built first and then dropped. No-op unless + ``COSYVOICE3_TRT`` is on (default), CUDA is available, and the estimator + ONNX ships with the model. Falls back to the torch estimator on any + failure. The upstream ``CausalConditionalCFM.forward_estimator`` detects + the non-``nn.Module`` estimator and drives the TRT engine. + """ + if getattr(self, "_code2wav_trt_done", False): + return + self._code2wav_trt_done = True + if not (_cosyvoice3_trt_enabled() and torch.cuda.is_available()): + return + onnx_path = self._resolve_flow_estimator_onnx() + if onnx_path is None: + logger.warning("CosyVoice3 code2wav: no flow-estimator ONNX available; keeping torch estimator") + return + try: + from vllm_omni.model_executor.models.cosyvoice3.flow_estimator_trt import ( + build_flow_estimator_trt, + ) + + wrapper = build_flow_estimator_trt(onnx_path, device="cuda") + # ``estimator`` is a registered nn.Module submodule; delete it first + # (frees the torch estimator weights) so the TRT wrapper can be set + # as a plain attribute — nn.Module.__setattr__ rejects non-Modules. + decoder = self.code2wav.flow_model.decoder + del decoder.estimator + decoder.estimator = wrapper + logger.info("CosyVoice3: using TensorRT flow-decoder estimator (code2wav)") + except Exception as exc: # pragma: no cover - defensive fallback + logger.warning( + "CosyVoice3 code2wav: TensorRT estimator build failed (%s); keeping torch estimator", + exc, + ) + def forward( self, input_ids: torch.Tensor, @@ -741,18 +998,43 @@ class CosyVoice3Model( if "speech_token" in kwargs: # Prompt conditioning tensors for code2wav: live under # ``embed.*`` per OmniPayloadStruct schema. + # + # vLLM hands these mm fields to forward() as collated, padded + # batch tensors (speech_token [B, maxT], speech_feat + # [B, 2*maxT, F], embedding [B, D]). Emitting them raw makes the + # downstream per-request payload split fragile at batch>1: it + # intermittently de-batches speech_token to 1-D and leaks the + # whole [B, D] embedding to every request, which corrupts voice + # conditioning and crashes code2wav (`prompt_token.shape[1]`). + # Instead, split into an explicit per-request list of unpadded, + # correctly-ranked tensors so ``to_payload_element`` splits them + # deterministically by request index (list[idx]). + speech_token_list, speech_feat_list, embedding_list, speech_token_len_list = ( + self._split_prompt_conditioning( + kwargs.get("speech_token"), + kwargs.get("speech_feat"), + kwargs.get("embedding"), + kwargs.get("speech_token_len"), + ) + ) multimodal_outputs = to_dict( OmniPayloadStruct( embed=EmbeddingsStruct( - speech_token=kwargs.get("speech_token"), - speech_feat=kwargs.get("speech_feat"), - embedding=kwargs.get("embedding"), + speech_token=speech_token_list, + speech_feat=speech_feat_list, + speech_token_len=speech_token_len_list, + embedding=embedding_list, ), ) ) return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs) elif self.model_stage == "cosyvoice3_code2wav": + # Lazily swap the flow-decoder estimator to a TensorRT engine on the + # first code2wav step (after weights are loaded), gated by the same + # COSYVOICE3_TRT env toggle as the talker speaker embedding. + self._maybe_enable_code2wav_trt() + runtime_info = kwargs.get("model_intermediate_buffer") if runtime_info is None: runtime_info = kwargs.get("runtime_additional_information", []) @@ -784,6 +1066,11 @@ class CosyVoice3Model( speech_token = embed.speech_token if embed else None speech_feat = embed.speech_feat if embed else None embedding = embed.embedding if embed else None + # Drop any right-padding carried from batched talker emission. + if speech_token is not None and speech_feat is not None: + speech_token, speech_feat = unpad_prompt_conditioning( + speech_token, speech_feat, embed.speech_token_len if embed else None + ) if speech_token is None or speech_feat is None or embedding is None: if stream_finished and req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"): with self._stream_audio_cache_lock: diff --git a/vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py b/vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py new file mode 100644 index 000000000..230fb14fb --- /dev/null +++ b/vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""TensorRT engine for the CosyVoice3 flow-decoder (CFM) DiT estimator. + +The estimator is the per-step network the conditional flow-matching ODE solver +calls during code2wav (token -> mel). It dominates code2wav latency; the +upstream ``CausalConditionalCFM.forward_estimator`` already supports running it +through a TensorRT engine (it switches on ``self.estimator`` not being an +``nn.Module`` and drives it via ``acquire_estimator`` / ``execute_async_v3``). +This module builds that engine from the bundled ``flow.decoder.estimator*.onnx`` +and wraps it so it can be dropped in for the torch estimator. + +The estimator engine has 6 inputs ``x, mask, mu, t, spks, cond`` and one output. +``x/mask/mu/cond`` carry a dynamic time dim; ``t``/``spks`` are fixed, so only +the former need an optimization profile (TRT infers the latter from the ONNX). +Shapes mirror the CosyVoice runtime. + +Precision: TensorRT >= 11 dropped the weakly-typed FP16/INT8 builder flags, so +fp16 only comes from a STRONGLY_TYPED network built from an fp16 ONNX +(``*autocast_fp16*``, fp16 I/O). An fp32 ONNX is built fp32 + the TF32 matmul +flag. ``EXPLICIT_BATCH`` is implicit (no flag) and ``ITensor.dtype`` is +read-only, so neither is set here. +""" + +from __future__ import annotations + +import os +import queue + +import torch +from vllm.logger import init_logger + +from vllm_omni.model_executor.models.cosyvoice3.speaker_embedding_trt import ( + _resolve_plan_path, + _trt_logger, +) + +logger = init_logger(__name__) + +# Optimization-profile shapes for the dynamic-length inputs (CFG batch dim = 2, +# 80 mel channels, time dim min/opt/max). Matches CosyVoice's token2wav. +_DYNAMIC_INPUTS = ("x", "mask", "mu", "cond") +_MIN_SHAPES = ((2, 80, 4), (2, 1, 4), (2, 80, 4), (2, 80, 4)) +_OPT_SHAPES = ((2, 80, 500), (2, 1, 500), (2, 80, 500), (2, 80, 500)) +_MAX_SHAPES = ((2, 80, 3000), (2, 1, 3000), (2, 80, 3000), (2, 80, 3000)) + + +def _is_fp16_onnx(onnx_path: str) -> bool: + """Heuristic: the project's fp16 estimator ONNX is exported strongly-typed + and named ``*autocast_fp16*`` / ``*fp16*`` (vs ``*fp32*``).""" + name = os.path.basename(onnx_path).lower() + return "fp16" in name or "autocast" in name + + +def _convert_onnx_to_trt(onnx_path: str, plan_path: str, strongly_typed: bool) -> None: + import tensorrt as trt + + logger.info( + "Building flow-estimator TensorRT engine from %s (%s) ...", + onnx_path, + "strongly-typed/fp16" if strongly_typed else "fp32+TF32", + ) + trt_logger = _trt_logger() + builder = trt.Builder(trt_logger) + # STRONGLY_TYPED takes precision from the ONNX graph (fp16 engine from an + # fp16 ONNX) — this is the only way to get fp16 in TRT>=11, which dropped + # the weakly-typed FP16 BuilderFlag. Otherwise EXPLICIT_BATCH is implicit, + # so create the network with no flags. + if strongly_typed: + network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED)) + else: + network = builder.create_network(0) + parser = trt.OnnxParser(network, trt_logger) + with open(onnx_path, "rb") as f: + if not parser.parse(f.read()): + errs = "; ".join(str(parser.get_error(i)) for i in range(parser.num_errors)) + raise ValueError(f"Failed to parse {onnx_path}: {errs}") + + config = builder.create_builder_config() + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 32) # 4 GiB + if not strongly_typed: + # fp32 ONNX: enable the best available reduced-precision matmul flag + # (FP16 on older TRT, else TF32 on Ampere+/Hopper). For a strongly-typed + # network the precision is fixed by the graph, so no flag is set. + for _flag_name in ("FP16", "TF32"): + _flag = getattr(trt.BuilderFlag, _flag_name, None) + if _flag is not None: + config.set_flag(_flag) + break + + profile = builder.create_optimization_profile() + for name, mn, op, mx in zip(_DYNAMIC_INPUTS, _MIN_SHAPES, _OPT_SHAPES, _MAX_SHAPES): + profile.set_shape(name, mn, op, mx) + config.add_optimization_profile(profile) + + engine_bytes = builder.build_serialized_network(network, config) + if engine_bytes is None: + raise RuntimeError(f"TensorRT failed to build flow-estimator engine from {onnx_path}") + tmp = plan_path + ".tmp" + with open(tmp, "wb") as f: + f.write(engine_bytes) + os.replace(tmp, plan_path) + logger.info("Wrote flow-estimator TensorRT engine to %s", plan_path) + + +class TrtContextWrapper: + """Pool of TensorRT execution contexts for the flow estimator. + + Exposes the ``acquire_estimator`` / ``release_estimator`` contract that + ``CausalConditionalCFM.forward_estimator`` expects. + """ + + def __init__( + self, engine, device: str | torch.device, io_dtype: torch.dtype = torch.float32, trt_concurrent: int = 1 + ): + self.trt_engine = engine + # Engine I/O dtype (fp16 for a strongly-typed fp16 engine). The flow runs + # in fp32, so forward_estimator casts to/from this at the boundary. + self.io_dtype = io_dtype + self._pool: queue.Queue = queue.Queue(maxsize=trt_concurrent) + for _ in range(trt_concurrent): + ctx = engine.create_execution_context() + assert ctx is not None, "failed to create TRT execution context (out of memory?)" + stream = torch.cuda.stream(torch.cuda.Stream(torch.device(device))) + self._pool.put([ctx, stream]) + + def acquire_estimator(self): + return self._pool.get(), self.trt_engine + + def release_estimator(self, context, stream): + self._pool.put([context, stream]) + + +def build_flow_estimator_trt(onnx_path: str, device: str | torch.device) -> TrtContextWrapper: + """Build/load the flow-estimator TRT engine and return a context-pool wrapper. + + An fp16 ONNX (``*autocast_fp16*``) is built as a strongly-typed network (the + only way to get fp16 in TRT>=11); an fp32 ONNX is built fp32 + TF32. + """ + import tensorrt as trt + + strongly_typed = _is_fp16_onnx(onnx_path) + plan_path = _resolve_plan_path(onnx_path, prefix="flow_estimator") + if not os.path.exists(plan_path) or os.path.getsize(plan_path) == 0: + _convert_onnx_to_trt(onnx_path, plan_path, strongly_typed=strongly_typed) + + runtime = trt.Runtime(_trt_logger()) + with open(plan_path, "rb") as f: + engine = runtime.deserialize_cuda_engine(f.read()) + if engine is None: + raise RuntimeError(f"Failed to deserialize flow-estimator TensorRT engine {plan_path}") + logger.info("Loaded flow-estimator TensorRT engine (%s)", plan_path) + io_dtype = torch.float16 if strongly_typed else torch.float32 + return TrtContextWrapper(engine, device=device, io_dtype=io_dtype) diff --git a/vllm_omni/model_executor/models/cosyvoice3/speaker_embedding_trt.py b/vllm_omni/model_executor/models/cosyvoice3/speaker_embedding_trt.py new file mode 100644 index 000000000..61aacd917 --- /dev/null +++ b/vllm_omni/model_executor/models/cosyvoice3/speaker_embedding_trt.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""TensorRT speaker-embedding (campplus) extractor for CosyVoice3. + +Drop-in accelerator for the ONNX-Runtime campplus session used in the +multimodal processor. ``campplus.onnx`` maps an fbank feature +``[B, T, 80]`` to a 192-d speaker embedding ``[B, 192]``; on this stack the +ONNX path only has a CPU execution provider, so it dominates per-request +prompt-processing latency. Building a TensorRT engine once and running it on +GPU removes that bottleneck. + +The engine is built lazily from ``campplus.onnx`` on first use and cached as +a ``.plan`` file (engines are device/TRT-version specific, so the cache key +includes both). Set ``COSYVOICE3_TRT_CACHE`` to override the cache dir. + +Written against TensorRT >= 10/11: ``EXPLICIT_BATCH`` is the implicit default +(create the network with no flags) and ``ITensor.dtype`` is read-only (I/O +dtypes come from the ONNX graph), so neither is set here. +""" + +from __future__ import annotations + +import hashlib +import os +import threading + +import torch +from vllm.logger import init_logger + +logger = init_logger(__name__) + +# campplus fbank sequence length bounds for the optimization profile. +_MIN_T = 4 +_OPT_T = 500 +_MAX_T = 3000 +_EMBED_DIM = 192 + + +def _trt_logger(): + import tensorrt as trt + + return trt.Logger(trt.Logger.WARNING) + + +def _convert_onnx_to_trt(onnx_path: str, plan_path: str) -> None: + import tensorrt as trt + + logger.info("Building campplus TensorRT engine from %s ...", onnx_path) + trt_logger = _trt_logger() + builder = trt.Builder(trt_logger) + # EXPLICIT_BATCH is implicit in TRT>=10; create the network with no flags. + network = builder.create_network(0) + parser = trt.OnnxParser(network, trt_logger) + with open(onnx_path, "rb") as f: + if not parser.parse(f.read()): + errs = "; ".join(str(parser.get_error(i)) for i in range(parser.num_errors)) + raise ValueError(f"Failed to parse {onnx_path}: {errs}") + + config = builder.create_builder_config() + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, 1 << 31) # 2 GiB + + input_name = network.get_input(0).name + profile = builder.create_optimization_profile() + profile.set_shape( + input_name, + (1, _MIN_T, 80), + (1, _OPT_T, 80), + (1, _MAX_T, 80), + ) + config.add_optimization_profile(profile) + + engine_bytes = builder.build_serialized_network(network, config) + if engine_bytes is None: + raise RuntimeError(f"TensorRT failed to build campplus engine from {onnx_path}") + tmp = plan_path + ".tmp" + with open(tmp, "wb") as f: + f.write(engine_bytes) + os.replace(tmp, plan_path) + logger.info("Wrote campplus TensorRT engine to %s", plan_path) + + +def _resolve_plan_path(onnx_path: str, prefix: str = "campplus") -> str: + cache_dir = os.environ.get("COSYVOICE3_TRT_CACHE") or os.path.join( + os.path.expanduser("~"), ".cache", "vllm_omni", "cosyvoice3_trt" + ) + os.makedirs(cache_dir, exist_ok=True) + try: + # No-arg form targets the current device (avoids the banned + # torch.cuda.current_device per ruff TID251). + dev_name = torch.cuda.get_device_name() + except Exception: + dev_name = "unknown" + import tensorrt as trt + + st = os.stat(onnx_path) + key = f"{os.path.abspath(onnx_path)}|{st.st_size}|{int(st.st_mtime)}|{dev_name}|trt{trt.__version__}" + digest = hashlib.sha1(key.encode()).hexdigest()[:16] + return os.path.join(cache_dir, f"{prefix}_{digest}.plan") + + +class CampplusTRT: + """Lazily-built TensorRT campplus speaker-embedding extractor.""" + + def __init__(self, onnx_path: str, device: str | torch.device): + import tensorrt as trt + + self.device = torch.device(device) + plan_path = _resolve_plan_path(onnx_path) + if not os.path.exists(plan_path) or os.path.getsize(plan_path) == 0: + _convert_onnx_to_trt(onnx_path, plan_path) + + runtime = trt.Runtime(_trt_logger()) + with open(plan_path, "rb") as f: + self.engine = runtime.deserialize_cuda_engine(f.read()) + if self.engine is None: + raise RuntimeError(f"Failed to deserialize campplus TensorRT engine {plan_path}") + self.context = self.engine.create_execution_context() + self.input_name = self.engine.get_tensor_name(0) + self.output_name = self.engine.get_tensor_name(1) + # TRT execution contexts are not safe for concurrent enqueue; serialize. + self._lock = threading.Lock() + logger.info("Loaded campplus TensorRT engine (%s)", plan_path) + + @torch.inference_mode() + def __call__(self, feat: torch.Tensor) -> torch.Tensor: + """feat: ``[T, 80]`` fbank features. Returns ``[1, 192]`` embedding.""" + x = feat.unsqueeze(0).to(self.device, dtype=torch.float32).contiguous() + out = torch.empty((x.shape[0], _EMBED_DIM), device=self.device, dtype=torch.float32) + stream = torch.cuda.current_stream(self.device) + with self._lock: + self.context.set_input_shape(self.input_name, tuple(x.shape)) + self.context.set_tensor_address(self.input_name, x.data_ptr()) + self.context.set_tensor_address(self.output_name, out.data_ptr()) + ok = self.context.execute_async_v3(stream.cuda_stream) + if not ok: + raise RuntimeError("campplus TensorRT execute_async_v3 failed") + stream.synchronize() + return out + + +# Process-wide cache: the mm processor is re-created per request (the deploy +# config disables the mm-processor object cache), so building a fresh +# CampplusTRT each time would re-deserialize the ~33 MB engine and allocate a +# new execution context on every request — inflating TTFP. Cache by +# (engine path, device) and reuse; the context serializes concurrent enqueues +# via its own lock, so a single shared instance is safe across requests. +_CAMPPLUS_CACHE: dict[tuple[str, str], CampplusTRT] = {} + + +def get_campplus_trt(onnx_path: str, device: str | torch.device) -> CampplusTRT: + """Return a process-wide cached :class:`CampplusTRT` (engine + context).""" + key = (os.path.abspath(onnx_path), str(device)) + inst = _CAMPPLUS_CACHE.get(key) + if inst is None: + inst = CampplusTRT(onnx_path, device) + _CAMPPLUS_CACHE[key] = inst + return inst diff --git a/vllm_omni/model_executor/models/cosyvoice3/utils.py b/vllm_omni/model_executor/models/cosyvoice3/utils.py index 0bf0cccb1..5827e6bbb 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/utils.py +++ b/vllm_omni/model_executor/models/cosyvoice3/utils.py @@ -215,6 +215,44 @@ def extract_spk_embedding(prompt_wav, campplus_session, device): return embedding +def unpad_prompt_conditioning(speech_token, speech_feat, speech_token_len): + """Drop right-padding from per-request prompt conditioning. + + The talker emits ``speech_token`` ``[1, T]`` possibly right-padded to the + batch max, plus the true ``speech_token_len``; ``speech_feat`` is at the 2:1 + mel:token ratio (``[1, 2T, F]``). This trims both to the real length. Called + in the (eager) code2wav stage where a host read of the length is allowed. + Returns ``(speech_token, speech_feat)`` unchanged when length is unavailable. + """ + if not isinstance(speech_token, torch.Tensor) or speech_token_len is None: + return speech_token, speech_feat + try: + token_len = int(torch.as_tensor(speech_token_len).reshape(-1)[0].item()) + except (RuntimeError, ValueError, IndexError): + return speech_token, speech_feat + if token_len <= 0 or speech_token.dim() < 2: + return speech_token, speech_feat + token_len = min(token_len, int(speech_token.shape[1])) + speech_token = speech_token[:, :token_len] + if isinstance(speech_feat, torch.Tensor) and speech_feat.dim() >= 2: + speech_feat = speech_feat[:, : 2 * token_len] + return speech_token, speech_feat + + +def extract_spk_embedding_trt(prompt_wav, campplus_trt, device): + """TensorRT counterpart of ``extract_spk_embedding``. + + Identical fbank front-end; the campplus forward runs on a prebuilt + TensorRT engine (GPU) instead of the CPU ONNX-Runtime session. Returns the + same ``[1, 192]`` embedding tensor on ``device``. + """ + speech = load_wav(prompt_wav, 16000) + feat = kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000) + feat = feat - feat.mean(dim=0, keepdim=True) + embedding = campplus_trt(feat) + return embedding.to(device) + + def extract_text_token(text, tokenizer, allowed_special): text_token = tokenizer.encode(text, allowed_special=allowed_special) text_token = torch.tensor([text_token], dtype=torch.int32) diff --git a/vllm_omni/model_executor/stage_input_processors/cosyvoice3.py b/vllm_omni/model_executor/stage_input_processors/cosyvoice3.py index 4c7245e77..2a0fdd869 100644 --- a/vllm_omni/model_executor/stage_input_processors/cosyvoice3.py +++ b/vllm_omni/model_executor/stage_input_processors/cosyvoice3.py @@ -16,6 +16,7 @@ from vllm_omni.data_entry_keys import ( OmniPayloadStruct, ) from vllm_omni.inputs.data import OmniTokensPrompt +from vllm_omni.model_executor.models.cosyvoice3.utils import unpad_prompt_conditioning logger = init_logger(__name__) @@ -192,18 +193,31 @@ def talker2code2wav_async_chunk( info = _decode_additional_information(getattr(request, "additional_information", None)) info_embed = info.get("embed", {}) if isinstance(info, dict) else {} prompt_payload = {} - for key in ("speech_token", "speech_feat", "embedding"): + cond_keys = ("speech_token", "speech_feat", "embedding", "speech_token_len") + for key in cond_keys: value = _to_cpu_tensor(info_embed.get(key)) if value is not None: prompt_payload[key] = value if isinstance(pooling_output, dict): po_embed = pooling_output.get("embed", {}) if isinstance(pooling_output.get("embed"), dict) else {} - for key in ("speech_token", "speech_feat", "embedding"): + for key in cond_keys: if key in prompt_payload: continue value = _to_cpu_tensor(po_embed.get(key)) if value is not None: prompt_payload[key] = value + # Drop any right-padding carried from batched talker emission so + # the chunk-routing math (prompt_token_len) and the conditioning + # sent to code2wav use the true prompt length. + if "speech_token" in prompt_payload: + st_unpad, sf_unpad = unpad_prompt_conditioning( + prompt_payload.get("speech_token"), + prompt_payload.get("speech_feat"), + prompt_payload.pop("speech_token_len", None), + ) + prompt_payload["speech_token"] = st_unpad + if sf_unpad is not None: + prompt_payload["speech_feat"] = sf_unpad prompt_token = prompt_payload.get("speech_token") prompt_token_len = ( int(prompt_token.shape[1]) diff --git a/vllm_omni/transformers_utils/configs/cosyvoice3.py b/vllm_omni/transformers_utils/configs/cosyvoice3.py index 9accc3cc8..b08cec063 100644 --- a/vllm_omni/transformers_utils/configs/cosyvoice3.py +++ b/vllm_omni/transformers_utils/configs/cosyvoice3.py @@ -42,6 +42,13 @@ class CosyVoice3Config(PretrainedConfig): self.qwen_pretrain_path = "CosyVoice-BlankEN" self.campplus_onxx_path = "campplus.onnx" self.speech_tokenizer_path = "speech_tokenizer_v3.onnx" + # Flow-decoder estimator ONNX for the optional TensorRT code2wav path. + # The fp16 (strongly-typed) ONNX gives an fp16 engine; if it is not in + # the model dir it is fetched from ``flow_estimator_onnx_repo``. Falls + # back to the bundled fp32 ONNX (fp32+TF32 engine) when unavailable. + self.flow_estimator_onnx_path = "flow.decoder.estimator.autocast_fp16.onnx" + self.flow_estimator_onnx_repo = "yuekai/Fun-CosyVoice3-0.5B-2512-FP16-ONNX" + self.flow_estimator_onnx_path_fp32 = "flow.decoder.estimator.fp32.onnx" self.spk2info_path = "spk2info.pt" self.version = "cosyvoice3" self.llm = {