[Model][Core] Enable async_chunk streaming pipeline for CosyVoice3 (#1703)

Signed-off-by: linyueqian <linyueqian@outlook.com>
Signed-off-by: indevn <indevn@outlook.com>
Co-authored-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
indevnandlinyueqian authored and GitHub committed 2026-04-05 06:11:33 +00:00
1 parent d92439c155
commit 6fc38e0467
18 files changed
+2094 -131

No files matched your search

@@ -174,7 +174,7 @@ class CausalConditionalCFM(ConditionalCFM):
super().__init__(in_channels, cfm_params, n_spks, spk_emb_dim, estimator)
@torch.inference_mode()
def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None):
def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None, streaming: bool = False):
"""Forward diffusion
Args:
@@ -277,7 +277,9 @@ class CausalMaskedDiffWithDiT(torch.nn.Module):
prompt_feat,
prompt_feat_len,
embedding,
finalize,
streaming: bool = True,
finalize: bool = False,
n_timesteps: int = 10,
):
assert token.shape[0] == 1
# xvec projection
@@ -314,7 +316,8 @@ class CausalMaskedDiffWithDiT(torch.nn.Module):
mask=mask.unsqueeze(1),
spks=embedding,
cond=conds,
n_timesteps=10,
n_timesteps=max(1, int(n_timesteps)),
streaming=streaming,
)
feat = feat[:, :, mel_len1:]
@@ -7,7 +7,9 @@ class CosyVoice3Config(PretrainedConfig):
model_type = "cosyvoice3"
def __init__(self, **kwargs):
# Set speech EOS so vLLM stops generation at the right token
# Set primary speech EOS so vLLM stops generation at the right token.
# The official CosyVoice3 treats ALL tokens >= speech_token_size
# (6561-6760) as stop signals; see stop_token_ids in the YAML configs.
kwargs.setdefault("eos_token_id", 6562)
super().__init__(**kwargs)
self.sample_rate = 24000
@@ -2,14 +2,16 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from functools import partial
from threading import Lock
import numpy as np
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsMultiModal
@@ -26,6 +28,9 @@ from vllm.multimodal.processing import (
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
from vllm_omni.model_executor.models.cosyvoice3.utils import (
@@ -267,6 +272,8 @@ class CosyVoice3Model(
supports_multimodal_raw_input_only = True
supports_multimodal = True
requires_raw_input_tokens = True
prefer_model_sampler = True
_sampling_eps = 1e-5
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -305,6 +312,8 @@ class CosyVoice3Model(
self.code2wav = CosyVoice3Code2Wav(self.config)
self.model = self.code2wav.flow_model
self.hift = self.code2wav.hift
# Keep additional information synchronized for async_chunk updates.
self.enable_update_additional_information = True
# Expose streaming parameters
self.token_overlap_len = self.code2wav.token_overlap_len
@@ -313,6 +322,9 @@ class CosyVoice3Model(
self.mel_cache_len = self.code2wav.mel_cache_len
self.source_cache_len = self.code2wav.source_cache_len
self.speech_window = self.code2wav.speech_window
self._stream_audio_cache_by_req: dict[str, torch.Tensor] = {}
self._stream_audio_cache_lock = Lock()
self._stream_vocoder_cache_by_req: dict[str, dict[str, torch.Tensor]] = {}
else:
raise ValueError(f"Model stage not supported {self.model_stage}")
@@ -331,19 +343,277 @@ class CosyVoice3Model(
# Use parent's cache config - critical for PagedAttention to work correctly
return parent_config.with_hf_config(qwen_hf_config, architectures=["Qwen2Model"])
@staticmethod
def _as_tensor(value: object) -> torch.Tensor | None:
"""Extract tensor payload from runtime info fields."""
if isinstance(value, list):
if not value:
return None
value = value[0]
if isinstance(value, torch.Tensor):
return value
return None
@staticmethod
def _as_str(value: object) -> str | None:
"""Extract string payload from runtime info fields."""
if isinstance(value, list):
if not value:
return None
value = value[0]
if value is None:
return None
return str(value)
@staticmethod
def _as_bool(value: object) -> bool:
"""Extract boolean payload from runtime info fields."""
if isinstance(value, list):
if not value:
return False
value = value[0]
if isinstance(value, torch.Tensor):
if value.numel() == 0:
return False
return bool(value.reshape(-1)[0].item())
if value is None:
return False
return bool(value)
@staticmethod
def _cross_fade_audio(audio: torch.Tensor, prev_tail: torch.Tensor) -> torch.Tensor:
"""Blend previous chunk tail into current chunk head using a Hamming window.
This mirrors upstream CosyVoice's `fade_in_out(...)` semantics:
update the current head in-place using a 2*overlap window, then
concatenate the unchanged remainder.
"""
if audio.numel() == 0 or prev_tail.numel() == 0:
return audio
overlap = min(int(audio.numel()), int(prev_tail.numel()))
if overlap <= 0:
return audio
window = torch.hamming_window(2 * overlap, periodic=False, dtype=audio.dtype, device=audio.device)
fade_in = window[:overlap]
fade_out = window[overlap:]
blended = audio[:overlap] * fade_in + prev_tail[-overlap:].to(device=audio.device, dtype=audio.dtype) * fade_out
if overlap == int(audio.numel()):
return blended
return torch.cat([blended, audio[overlap:]], dim=0)
def _stitch_stream_audio(self, req_id: str | None, audio: torch.Tensor, stream_finished: bool) -> torch.Tensor:
"""Pass-through stitching for async_chunk.
Chunk overlap is already removed in mel domain via token_offset_tokens.
Applying an additional waveform-domain fade/cache step introduces either
duplicated overlap (if no tail trim) or duration shrink (if tail trim).
"""
if req_id is not None and stream_finished and hasattr(self, "_stream_audio_cache_by_req"):
with self._stream_audio_cache_lock:
self._stream_audio_cache_by_req.pop(req_id, None)
if hasattr(self, "_stream_vocoder_cache_by_req"):
self._stream_vocoder_cache_by_req.pop(req_id, None)
return audio
@staticmethod
def _split_request_ids(ids: torch.Tensor, seq_token_counts: list[int] | None = None) -> list[torch.Tensor]:
"""Split concatenated input_ids into per-request segments."""
if seq_token_counts is not None:
boundaries = [0]
for count in seq_token_counts:
boundaries.append(boundaries[-1] + int(count))
total = ids.numel()
return [ids[boundaries[i] : min(boundaries[i + 1], total)] for i in range(len(seq_token_counts))]
if is_forward_context_available():
slices = get_forward_context().ubatch_slices
if slices is not None and len(slices) > 1 and not any(hasattr(s, "token_slice") for s in slices):
boundaries = [0]
for s in slices:
boundaries.append(boundaries[-1] + int(s))
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
return [ids]
def _sanitize_codec_tokens(self, req_ids: torch.Tensor) -> torch.Tensor:
"""Filter non-code tokens before feeding flow token embedding."""
vocab_size = int(self.code2wav.input_embedding.num_embeddings)
valid_mask = (req_ids >= 0) & (req_ids < vocab_size)
return req_ids[valid_mask]
@staticmethod
def _req_scalar(param: torch.Tensor | None, req_idx: int, default: float | int) -> float | int:
if param is None or param.numel() == 0:
return default
index = min(req_idx, int(param.numel()) - 1)
value = param.reshape(-1)[index].item()
if isinstance(default, int):
return int(value)
return float(value)
@staticmethod
def _multinomial_sample(probs: torch.Tensor, generator: torch.Generator | None = None) -> torch.Tensor:
return torch.multinomial(probs, 1, replacement=True, generator=generator).reshape(())
@classmethod
def _nucleus_sample_one(
cls,
weighted_scores: torch.Tensor,
*,
top_p: float,
top_k: int,
generator: torch.Generator | None,
) -> int:
probs = weighted_scores.softmax(dim=0)
sorted_prob, sorted_idx = probs.sort(descending=True, stable=True)
kept_probs: list[torch.Tensor] = []
kept_indices: list[torch.Tensor] = []
cum_prob = 0.0
max_keep = len(sorted_idx) if top_k <= 0 else min(int(top_k), len(sorted_idx))
for i in range(len(sorted_idx)):
if cum_prob < top_p and len(kept_probs) < max_keep:
cum_prob += float(sorted_prob[i].item())
kept_probs.append(sorted_prob[i])
kept_indices.append(sorted_idx[i])
else:
break
if not kept_probs:
return int(sorted_idx[0].item())
sample_probs = torch.stack(kept_probs)
sample_idx = cls._multinomial_sample(sample_probs, generator=generator)
return int(torch.stack(kept_indices)[int(sample_idx.item())].item())
@classmethod
def _ras_sample_one(
cls,
weighted_scores: torch.Tensor,
decoded_tokens: Sequence[int],
*,
top_p: float,
top_k: int,
win_size: int,
tau_r: float,
generator: torch.Generator | None,
) -> int:
top_id = cls._nucleus_sample_one(
weighted_scores,
top_p=top_p,
top_k=top_k,
generator=generator,
)
if win_size > 0 and decoded_tokens:
recent = torch.as_tensor(
list(decoded_tokens[-win_size:]),
device=weighted_scores.device,
dtype=torch.long,
)
rep_num = int((recent == top_id).sum().item())
if rep_num >= win_size * tau_r:
weighted_scores = weighted_scores.clone()
weighted_scores[top_id] = float("-inf")
fallback_probs = weighted_scores.softmax(dim=0)
top_id = int(cls._multinomial_sample(fallback_probs, generator=generator).item())
return top_id
def _cosyvoice3_ras_enabled(self, sampling_metadata: SamplingMetadata) -> bool:
if self.model_stage != "cosyvoice3_talker":
return False
if sampling_metadata.max_num_logprobs is not None:
return False
if sampling_metadata.temperature is None:
return False
if bool(sampling_metadata.bad_words_token_ids):
return False
if torch.any(sampling_metadata.frequency_penalties != 0):
return False
if torch.any(sampling_metadata.presence_penalties != 0):
return False
return True
def sample(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> SamplerOutput | None:
if logits is None or logits.numel() == 0:
return None
if self.model_stage != "cosyvoice3_talker":
return None
sampler = getattr(self, "_talker_sampler", None)
if sampler is None:
sampler = Sampler()
self._talker_sampler = sampler
if not self._cosyvoice3_ras_enabled(sampling_metadata):
return sampler(logits=logits, sampling_metadata=sampling_metadata)
logits = logits.to(torch.float32)
sampling_for_processors = replace(sampling_metadata, no_penalties=True)
logits = sampler.apply_logits_processors(logits, sampling_for_processors, predict_bonus_token=False)
sampling_cfg = dict(self.config.llm.get("sampling", {}))
default_top_p = float(sampling_cfg.get("top_p", 0.8))
default_top_k = int(sampling_cfg.get("top_k", 25))
win_size = int(sampling_cfg.get("win_size", 10))
tau_r = float(sampling_cfg.get("tau_r", 0.1))
sampled_ids: list[int] = []
for req_idx in range(int(logits.shape[0])):
row_logits = logits[req_idx]
temperature = float(self._req_scalar(sampling_metadata.temperature, req_idx, 1.0))
if temperature < self._sampling_eps:
sampled_ids.append(int(torch.argmax(row_logits).item()))
continue
top_p = float(self._req_scalar(sampling_metadata.top_p, req_idx, default_top_p))
top_k = int(self._req_scalar(sampling_metadata.top_k, req_idx, default_top_k))
generator = sampling_metadata.generators.get(req_idx)
weighted_scores = torch.log_softmax(row_logits / max(temperature, self._sampling_eps), dim=0)
decoded_tokens = (
sampling_metadata.output_token_ids[req_idx] if req_idx < len(sampling_metadata.output_token_ids) else []
)
sampled_ids.append(
self._ras_sample_one(
weighted_scores,
decoded_tokens,
top_p=top_p,
top_k=top_k,
win_size=win_size,
tau_r=tau_r,
generator=generator,
)
)
sampled = torch.tensor(sampled_ids, device=logits.device, dtype=torch.int32)
return SamplerOutput(sampled_token_ids=sampled.unsqueeze(-1), logprobs_tensors=None)
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if self.model_stage == "cosyvoice3_talker":
logits = self.model.llm_decoder(hidden_states)
# The decoder outputs speech_token_size + 200 logits. The official
# CosyVoice3 treats ALL tokens >= speech_token_size (the last 200)
# as stop signals. Merge their probabilities into a single EOS
# token (6562) via logsumexp so that vLLM's stop_token_ids=[6562]
# fires with the correct aggregate stop probability.
speech_token_size = self.config.llm["speech_token_size"]
eos_idx = self.config.llm["eos_token_id"]
stop_logits = logits[..., speech_token_size:] # last 200
merged_stop = torch.logsumexp(stop_logits, dim=-1, keepdim=True)
logits[..., speech_token_size:] = float("-inf") # mask all
logits[..., eos_idx] = merged_stop.squeeze(-1) # restore merged
# Pad to full vocab_size for vLLM token handling.
vocab_size = self.config.vocab_size
pad_size = vocab_size - logits.size(-1)
pad_shape = logits.shape[:-1] + (pad_size,)
pad = logits.new_full(pad_shape, float("-inf"))
eos_token_val = logits[..., self.config.llm["eos_token_id"]].clone()
logits[..., -200:] = float("-inf")
logits[..., self.config.llm["eos_token_id"]] = eos_token_val
logits = torch.cat([logits, pad], dim=-1)
if pad_size > 0:
pad_shape = logits.shape[:-1] + (pad_size,)
pad = logits.new_full(pad_shape, float("-inf"))
logits = torch.cat([logits, pad], dim=-1)
return logits
else:
raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.")
@@ -380,6 +650,7 @@ class CosyVoice3Model(
hidden = int(self.config.hidden_size)
return torch.zeros(
(input_ids.shape[0], hidden),
device=input_ids.device,
)
else:
raise RuntimeError(f"embed_input_ids is not valid for {self.model_stage}.")
@@ -412,28 +683,116 @@ class CosyVoice3Model(
return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs)
elif self.model_stage == "cosyvoice3_code2wav":
runtime_info = kwargs.get("runtime_additional_information", [])
if not runtime_info:
length = 30 * 24000
audio = np.zeros((length,))
return OmniOutput(text_hidden_states=None, multimodal_outputs={"audio": audio})
runtime_info = kwargs.get("model_intermediate_buffer")
if runtime_info is None:
runtime_info = kwargs.get("runtime_additional_information", [])
if "runtime_additional_information" in kwargs and "model_intermediate_buffer" not in kwargs:
logger.warning_once("runtime_additional_information is deprecated, use model_intermediate_buffer")
# Remove the last eos token and add batch dimension
token = input_ids[..., :-1].unsqueeze(0)
seq_token_counts = kwargs.get("seq_token_counts")
flat_ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(flat_ids, seq_token_counts)
# Generate audio using code2wav
tts_speech = self.code2wav(
token=token,
prompt_token=runtime_info[0]["speech_token"][:1],
prompt_feat=runtime_info[0]["speech_feat"][:1],
embedding=runtime_info[0]["embedding"][:1],
n_timesteps=10,
)
num_reqs = max(1, len(request_ids_list))
sample_rate = torch.tensor(int(self.config.sample_rate), dtype=torch.int32)
empty_audio = torch.zeros((0,), dtype=torch.float32, device=input_ids.device)
audios: list[torch.Tensor] = [empty_audio] * num_reqs
srs: list[torch.Tensor] = [sample_rate] * num_reqs
if not isinstance(runtime_info, list):
runtime_info = []
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"audio": tts_speech, "sr": torch.tensor(22050)},
)
for idx, req_ids in enumerate(request_ids_list):
info = runtime_info[idx] if idx < len(runtime_info) and isinstance(runtime_info[idx], dict) else {}
req_id = self._as_str(info.get("req_id")) if info else None
stream_finished = self._as_bool(info.get("stream_finished")) if info else False
speech_token = self._as_tensor(info.get("speech_token")) if info else None
speech_feat = self._as_tensor(info.get("speech_feat")) if info else None
embedding = self._as_tensor(info.get("embedding")) if info else None
if speech_token is None or speech_feat is None or embedding is None:
if stream_finished and req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
with self._stream_audio_cache_lock:
self._stream_vocoder_cache_by_req.pop(req_id, None)
audios[idx] = self._stitch_stream_audio(req_id, empty_audio, stream_finished)
if (
req_ids.numel() > 0
and info
and ("token_offset" in info or "left_context_size" in info or "generated_len" in info)
):
info_keys = ",".join(sorted(info.keys())) if info else ""
logger.warning_once(
"CosyVoice3 code2wav missing prompt conditioning for non-empty codec tokens: "
"raw_len=%d info_keys=%s",
int(req_ids.numel()),
info_keys,
)
continue
token = self._sanitize_codec_tokens(req_ids)
if token.numel() == 0:
audios[idx] = self._stitch_stream_audio(req_id, empty_audio, stream_finished)
if req_ids.numel() > 0:
logger.warning_once(
"CosyVoice3 code2wav received no valid codec tokens after filtering: "
"raw_len=%d raw_range=[%d,%d] vocab_size=%d",
req_ids.numel(),
int(req_ids.min().item()),
int(req_ids.max().item()),
int(self.code2wav.input_embedding.num_embeddings),
)
continue
# `generated_len` is injected for many models by the generic
# runner, so only explicit chunk-routing fields should switch
# code2wav into the streaming path.
uses_streaming_decode = bool(info) and (
"stream_finished" in info or "token_offset" in info or "left_context_size" in info
)
if uses_streaming_decode:
token_offset = 0
try:
if info and "token_offset" in info:
token_offset = max(0, int(info.get("token_offset", 0)))
elif info:
token_offset = max(0, int(info.get("left_context_size", 0)))
except (TypeError, ValueError):
token_offset = 0
cache_state = None
if req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
with self._stream_audio_cache_lock:
cache_state = self._stream_vocoder_cache_by_req.get(req_id)
tts_speech, new_cache_state = self.code2wav.forward_streaming(
token=token.unsqueeze(0),
prompt_token=speech_token[:1],
prompt_feat=speech_feat[:1],
embedding=embedding[:1],
cache_state=cache_state,
n_timesteps=10,
token_offset_tokens=token_offset,
finalize=stream_finished,
)
if req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
with self._stream_audio_cache_lock:
if new_cache_state is None or stream_finished:
self._stream_vocoder_cache_by_req.pop(req_id, None)
else:
self._stream_vocoder_cache_by_req[req_id] = new_cache_state
else:
tts_speech = self.code2wav.forward(
token=token.unsqueeze(0),
prompt_token=speech_token[:1],
prompt_feat=speech_feat[:1],
embedding=embedding[:1],
n_timesteps=10,
)
audio = tts_speech.reshape(-1).to(dtype=torch.float32)
audios[idx] = self._stitch_stream_audio(req_id, audio, stream_finished)
return OmniOutput(text_hidden_states=None, multimodal_outputs={"audio": audios, "sr": srs})
else:
raise ValueError(f"Unsupported model_stage: {self.model_stage}")
@@ -11,11 +11,12 @@ This module contains the code2wav (token-to-waveform) stage which uses:
from __future__ import annotations
from contextlib import nullcontext
import numpy as np
import torch
import torch.nn as nn
from omegaconf import DictConfig
from torch.nn import functional as F
from vllm.logger import init_logger
from vllm_omni.diffusion.models.cosyvoice3_audio.cosyvoice3_dit import DiT
@@ -29,7 +30,6 @@ from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan import (
)
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.layers import PreLookaheadLayer
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
from vllm_omni.model_executor.models.cosyvoice3.utils import make_pad_mask
logger = init_logger(__name__)
@@ -150,6 +150,140 @@ class CosyVoice3Code2Wav(nn.Module):
"""Speaker embedding affine layer."""
return self.flow_model.spk_embed_affine_layer
@torch.inference_mode()
def _forward_mel(
self,
token: torch.Tensor,
prompt_token: torch.Tensor,
prompt_feat: torch.Tensor,
embedding: torch.Tensor,
n_timesteps: int = 10,
token_offset_tokens: int = 0,
streaming: bool = True,
finalize: bool = False,
) -> torch.Tensor:
"""Generate mel features via the upstream flow-model inference path."""
flow_weight = next(self.flow_model.parameters())
device = flow_weight.device
dtype = flow_weight.dtype
token = token.to(device=device, dtype=torch.int32)
prompt_token = prompt_token.to(device=device, dtype=torch.int32)
prompt_feat = prompt_feat.to(device=device, dtype=dtype)
embedding = embedding.to(device=device, dtype=dtype)
token_len = torch.tensor([token.shape[1]], device=device, dtype=torch.int32)
prompt_token_len = torch.tensor([prompt_token.shape[1]], device=device, dtype=torch.int32)
prompt_feat_len = torch.tensor([prompt_feat.shape[1]], device=device, dtype=torch.int32)
with nullcontext():
feat, _ = self.flow_model.inference(
token=token,
token_len=token_len,
prompt_token=prompt_token,
prompt_token_len=prompt_token_len,
prompt_feat=prompt_feat,
prompt_feat_len=prompt_feat_len,
embedding=embedding,
streaming=streaming,
finalize=finalize,
n_timesteps=n_timesteps,
)
trim_mel = max(0, int(token_offset_tokens)) * int(self.token_mel_ratio)
if trim_mel > 0:
feat = feat[:, :, trim_mel:]
return feat
@staticmethod
def _fade_speech(
speech: torch.Tensor,
prev_speech: torch.Tensor,
) -> torch.Tensor:
"""Blend previous speech tail into current speech head."""
if speech.numel() == 0 or prev_speech.numel() == 0:
return speech
overlap = min(int(speech.shape[-1]), int(prev_speech.shape[-1]))
if overlap <= 0:
return speech
window = torch.hamming_window(2 * overlap, periodic=False, dtype=speech.dtype, device=speech.device)
fade_in = window[:overlap].view(1, -1)
fade_out = window[overlap:].view(1, -1)
blended_head = (
speech[:, :overlap] * fade_in
+ prev_speech[:, -overlap:].to(device=speech.device, dtype=speech.dtype) * fade_out
)
if overlap == int(speech.shape[-1]):
return blended_head
return torch.cat([blended_head, speech[:, overlap:]], dim=-1)
@torch.inference_mode()
def forward_streaming(
self,
token: torch.Tensor,
prompt_token: torch.Tensor,
prompt_feat: torch.Tensor,
embedding: torch.Tensor,
*,
cache_state: dict[str, torch.Tensor] | None = None,
n_timesteps: int = 10,
token_offset_tokens: int = 0,
finalize: bool = False,
) -> tuple[torch.Tensor, dict[str, torch.Tensor] | None]:
"""Decode streaming audio using cumulative mel + emitted-speech offset.
This mirrors upstream CosyVoice3 streaming semantics more closely than
waveform-domain overlap-add: keep a cumulative mel history per request,
re-run causal HiFT on the history, and emit only the newly grown speech
suffix. That preserves causal look-right handling without double
trimming or duplicated overlap at chunk boundaries.
"""
with nullcontext():
feat = self._forward_mel(
token=token,
prompt_token=prompt_token,
prompt_feat=prompt_feat,
embedding=embedding,
n_timesteps=n_timesteps,
token_offset_tokens=token_offset_tokens,
streaming=True,
finalize=finalize,
)
hift_weight = self.hift.m_source.l_linear.weight
chunk_mel = feat.to(device=hift_weight.device, dtype=hift_weight.dtype)
cached_mel = None if not cache_state else cache_state.get("mel")
speech_offset_obj = None if not cache_state else cache_state.get("speech_offset")
try:
speech_offset = int(speech_offset_obj) if speech_offset_obj is not None else 0
except (TypeError, ValueError):
speech_offset = 0
if isinstance(cached_mel, torch.Tensor) and cached_mel.numel() > 0:
cached_mel = cached_mel.to(device=chunk_mel.device, dtype=chunk_mel.dtype)
tts_mel = torch.cat([cached_mel, chunk_mel], dim=-1) if chunk_mel.numel() > 0 else cached_mel
else:
tts_mel = chunk_mel
if tts_mel.shape[-1] == 0:
tts_speech = torch.zeros((chunk_mel.shape[0], 1, 0), device=chunk_mel.device, dtype=chunk_mel.dtype)
else:
with nullcontext():
tts_speech, _ = self.hift.inference(speech_feat=tts_mel, finalize=finalize)
tts_speech = tts_speech.reshape(tts_speech.shape[0], -1)
speech_offset = max(0, min(speech_offset, int(tts_speech.shape[-1])))
emitted_speech = tts_speech[:, speech_offset:]
if finalize:
return emitted_speech.reshape(emitted_speech.shape[0], 1, -1), None
new_state = {
"mel": tts_mel.detach().cpu().contiguous(),
"speech_offset": int(tts_speech.shape[-1]),
}
return emitted_speech.reshape(emitted_speech.shape[0], 1, -1), new_state
@torch.inference_mode()
def forward(
self,
@@ -159,76 +293,18 @@ class CosyVoice3Code2Wav(nn.Module):
embedding: torch.Tensor,
n_timesteps: int = 10,
) -> torch.Tensor:
"""Generate audio waveform from speech tokens.
Args:
token: Speech tokens from talker stage [batch, seq_len]
prompt_token: Prompt speech tokens [batch, prompt_len]
prompt_feat: Prompt mel features [batch, feat_len, mel_dim]
embedding: Speaker embedding [batch, spk_dim]
n_timesteps: Number of diffusion steps
Returns:
Audio waveform [batch, 1, audio_len]
"""
device = token.device
dtype = next(self.flow_model.parameters()).dtype
# Normalize and project speaker embedding
embedding = embedding.to(device=device, dtype=dtype)
embedding = F.normalize(embedding, dim=1)
embedding = self.spk_embed_affine_layer(embedding)
# Prepare tokens
prompt_token = prompt_token.to(device=device)
token_len1, token_len2 = prompt_token.shape[1], token.shape[1]
prompt_token_len = torch.tensor([token_len1], device=device, dtype=torch.int32)
token_len = torch.tensor([token_len2], device=device, dtype=torch.int32)
# Concatenate prompt and target tokens
full_token = torch.cat([prompt_token, token], dim=1)
full_token_len = prompt_token_len + token_len
# Create mask
mask = (~make_pad_mask(full_token_len)).unsqueeze(-1).to(embedding)
# Token embedding (clamp to valid codebook range; EOS/padding tokens may exceed vocab_size)
token_emb = (
self.input_embedding(torch.clamp(full_token, min=0, max=self.input_embedding.num_embeddings - 1)) * mask
)
# Pre-lookahead processing
h = self.pre_lookahead_layer(token_emb)
h = h.repeat_interleave(self.token_mel_ratio, dim=1)
# Calculate mel lengths
mel_len1 = prompt_feat.shape[1]
mel_len2 = h.shape[1] - mel_len1
# Build conditioning
conds = torch.zeros(
[1, mel_len1 + mel_len2, self.output_size],
device=device,
dtype=h.dtype,
)
conds[:, :mel_len1] = prompt_feat
conds = conds.transpose(1, 2)
# Create mel mask
mel_mask = (~make_pad_mask(torch.tensor([mel_len1 + mel_len2]))).to(h)
# Run flow matching decoder
feat, _ = self.decoder(
mu=h.transpose(1, 2).contiguous(),
mask=mel_mask.unsqueeze(1),
spks=embedding,
cond=conds,
"""Generate audio waveform from speech tokens."""
feat = self._forward_mel(
token=token,
prompt_token=prompt_token,
prompt_feat=prompt_feat,
embedding=embedding,
n_timesteps=n_timesteps,
token_offset_tokens=0,
streaming=False,
finalize=True,
)
# Extract generated portion (after prompt)
feat = feat[:, :, mel_len1:]
# Run vocoder
hift_weight = self.hift.m_source.l_linear.weight
tts_mel = feat.to(device=hift_weight.device, dtype=hift_weight.dtype)
@@ -240,7 +316,7 @@ class CosyVoice3Code2Wav(nn.Module):
dtype=tts_mel.dtype,
)
else:
tts_speech, _ = self.hift.inference(speech_feat=tts_mel)
tts_speech, _ = self.hift.inference(speech_feat=tts_mel, finalize=True)
return tts_speech