[Model][Core] Enable async_chunk streaming pipeline for CosyVoice3 (#1703)
Signed-off-by: linyueqian <linyueqian@outlook.com> Signed-off-by: indevn <indevn@outlook.com> Co-authored-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
1 parent
d92439c155
commit
6fc38e0467
18 files changed
+2094
-131
No files matched your search
Binary file not shown.
@@ -174,7 +174,7 @@ class CausalConditionalCFM(ConditionalCFM):
|
||||
super().__init__(in_channels, cfm_params, n_spks, spk_emb_dim, estimator)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None):
|
||||
def forward(self, mu, mask, n_timesteps, temperature=1.0, spks=None, cond=None, streaming: bool = False):
|
||||
"""Forward diffusion
|
||||
|
||||
Args:
|
||||
@@ -277,7 +277,9 @@ class CausalMaskedDiffWithDiT(torch.nn.Module):
|
||||
prompt_feat,
|
||||
prompt_feat_len,
|
||||
embedding,
|
||||
finalize,
|
||||
streaming: bool = True,
|
||||
finalize: bool = False,
|
||||
n_timesteps: int = 10,
|
||||
):
|
||||
assert token.shape[0] == 1
|
||||
# xvec projection
|
||||
@@ -314,7 +316,8 @@ class CausalMaskedDiffWithDiT(torch.nn.Module):
|
||||
mask=mask.unsqueeze(1),
|
||||
spks=embedding,
|
||||
cond=conds,
|
||||
n_timesteps=10,
|
||||
n_timesteps=max(1, int(n_timesteps)),
|
||||
streaming=streaming,
|
||||
)
|
||||
|
||||
feat = feat[:, :, mel_len1:]
|
||||
|
||||
@@ -7,7 +7,9 @@ class CosyVoice3Config(PretrainedConfig):
|
||||
model_type = "cosyvoice3"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
# Set speech EOS so vLLM stops generation at the right token
|
||||
# Set primary speech EOS so vLLM stops generation at the right token.
|
||||
# The official CosyVoice3 treats ALL tokens >= speech_token_size
|
||||
# (6561-6760) as stop signals; see stop_token_ids in the YAML configs.
|
||||
kwargs.setdefault("eos_token_id", 6562)
|
||||
super().__init__(**kwargs)
|
||||
self.sample_rate = 24000
|
||||
|
||||
@@ -2,14 +2,16 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from functools import partial
|
||||
from threading import Lock
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
@@ -26,6 +28,9 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import (
|
||||
@@ -267,6 +272,8 @@ class CosyVoice3Model(
|
||||
supports_multimodal_raw_input_only = True
|
||||
supports_multimodal = True
|
||||
requires_raw_input_tokens = True
|
||||
prefer_model_sampler = True
|
||||
_sampling_eps = 1e-5
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
@@ -305,6 +312,8 @@ class CosyVoice3Model(
|
||||
self.code2wav = CosyVoice3Code2Wav(self.config)
|
||||
self.model = self.code2wav.flow_model
|
||||
self.hift = self.code2wav.hift
|
||||
# Keep additional information synchronized for async_chunk updates.
|
||||
self.enable_update_additional_information = True
|
||||
|
||||
# Expose streaming parameters
|
||||
self.token_overlap_len = self.code2wav.token_overlap_len
|
||||
@@ -313,6 +322,9 @@ class CosyVoice3Model(
|
||||
self.mel_cache_len = self.code2wav.mel_cache_len
|
||||
self.source_cache_len = self.code2wav.source_cache_len
|
||||
self.speech_window = self.code2wav.speech_window
|
||||
self._stream_audio_cache_by_req: dict[str, torch.Tensor] = {}
|
||||
self._stream_audio_cache_lock = Lock()
|
||||
self._stream_vocoder_cache_by_req: dict[str, dict[str, torch.Tensor]] = {}
|
||||
else:
|
||||
raise ValueError(f"Model stage not supported {self.model_stage}")
|
||||
|
||||
@@ -331,19 +343,277 @@ class CosyVoice3Model(
|
||||
# Use parent's cache config - critical for PagedAttention to work correctly
|
||||
return parent_config.with_hf_config(qwen_hf_config, architectures=["Qwen2Model"])
|
||||
|
||||
@staticmethod
|
||||
def _as_tensor(value: object) -> torch.Tensor | None:
|
||||
"""Extract tensor payload from runtime info fields."""
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
return None
|
||||
value = value[0]
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _as_str(value: object) -> str | None:
|
||||
"""Extract string payload from runtime info fields."""
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
return None
|
||||
value = value[0]
|
||||
if value is None:
|
||||
return None
|
||||
return str(value)
|
||||
|
||||
@staticmethod
|
||||
def _as_bool(value: object) -> bool:
|
||||
"""Extract boolean payload from runtime info fields."""
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
return False
|
||||
value = value[0]
|
||||
if isinstance(value, torch.Tensor):
|
||||
if value.numel() == 0:
|
||||
return False
|
||||
return bool(value.reshape(-1)[0].item())
|
||||
if value is None:
|
||||
return False
|
||||
return bool(value)
|
||||
|
||||
@staticmethod
|
||||
def _cross_fade_audio(audio: torch.Tensor, prev_tail: torch.Tensor) -> torch.Tensor:
|
||||
"""Blend previous chunk tail into current chunk head using a Hamming window.
|
||||
|
||||
This mirrors upstream CosyVoice's `fade_in_out(...)` semantics:
|
||||
update the current head in-place using a 2*overlap window, then
|
||||
concatenate the unchanged remainder.
|
||||
"""
|
||||
if audio.numel() == 0 or prev_tail.numel() == 0:
|
||||
return audio
|
||||
overlap = min(int(audio.numel()), int(prev_tail.numel()))
|
||||
if overlap <= 0:
|
||||
return audio
|
||||
window = torch.hamming_window(2 * overlap, periodic=False, dtype=audio.dtype, device=audio.device)
|
||||
fade_in = window[:overlap]
|
||||
fade_out = window[overlap:]
|
||||
blended = audio[:overlap] * fade_in + prev_tail[-overlap:].to(device=audio.device, dtype=audio.dtype) * fade_out
|
||||
if overlap == int(audio.numel()):
|
||||
return blended
|
||||
return torch.cat([blended, audio[overlap:]], dim=0)
|
||||
|
||||
def _stitch_stream_audio(self, req_id: str | None, audio: torch.Tensor, stream_finished: bool) -> torch.Tensor:
|
||||
"""Pass-through stitching for async_chunk.
|
||||
|
||||
Chunk overlap is already removed in mel domain via token_offset_tokens.
|
||||
Applying an additional waveform-domain fade/cache step introduces either
|
||||
duplicated overlap (if no tail trim) or duration shrink (if tail trim).
|
||||
"""
|
||||
if req_id is not None and stream_finished and hasattr(self, "_stream_audio_cache_by_req"):
|
||||
with self._stream_audio_cache_lock:
|
||||
self._stream_audio_cache_by_req.pop(req_id, None)
|
||||
if hasattr(self, "_stream_vocoder_cache_by_req"):
|
||||
self._stream_vocoder_cache_by_req.pop(req_id, None)
|
||||
return audio
|
||||
|
||||
@staticmethod
|
||||
def _split_request_ids(ids: torch.Tensor, seq_token_counts: list[int] | None = None) -> list[torch.Tensor]:
|
||||
"""Split concatenated input_ids into per-request segments."""
|
||||
if seq_token_counts is not None:
|
||||
boundaries = [0]
|
||||
for count in seq_token_counts:
|
||||
boundaries.append(boundaries[-1] + int(count))
|
||||
total = ids.numel()
|
||||
return [ids[boundaries[i] : min(boundaries[i + 1], total)] for i in range(len(seq_token_counts))]
|
||||
|
||||
if is_forward_context_available():
|
||||
slices = get_forward_context().ubatch_slices
|
||||
if slices is not None and len(slices) > 1 and not any(hasattr(s, "token_slice") for s in slices):
|
||||
boundaries = [0]
|
||||
for s in slices:
|
||||
boundaries.append(boundaries[-1] + int(s))
|
||||
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
|
||||
|
||||
return [ids]
|
||||
|
||||
def _sanitize_codec_tokens(self, req_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Filter non-code tokens before feeding flow token embedding."""
|
||||
vocab_size = int(self.code2wav.input_embedding.num_embeddings)
|
||||
valid_mask = (req_ids >= 0) & (req_ids < vocab_size)
|
||||
return req_ids[valid_mask]
|
||||
|
||||
@staticmethod
|
||||
def _req_scalar(param: torch.Tensor | None, req_idx: int, default: float | int) -> float | int:
|
||||
if param is None or param.numel() == 0:
|
||||
return default
|
||||
index = min(req_idx, int(param.numel()) - 1)
|
||||
value = param.reshape(-1)[index].item()
|
||||
if isinstance(default, int):
|
||||
return int(value)
|
||||
return float(value)
|
||||
|
||||
@staticmethod
|
||||
def _multinomial_sample(probs: torch.Tensor, generator: torch.Generator | None = None) -> torch.Tensor:
|
||||
return torch.multinomial(probs, 1, replacement=True, generator=generator).reshape(())
|
||||
|
||||
@classmethod
|
||||
def _nucleus_sample_one(
|
||||
cls,
|
||||
weighted_scores: torch.Tensor,
|
||||
*,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
generator: torch.Generator | None,
|
||||
) -> int:
|
||||
probs = weighted_scores.softmax(dim=0)
|
||||
sorted_prob, sorted_idx = probs.sort(descending=True, stable=True)
|
||||
kept_probs: list[torch.Tensor] = []
|
||||
kept_indices: list[torch.Tensor] = []
|
||||
cum_prob = 0.0
|
||||
max_keep = len(sorted_idx) if top_k <= 0 else min(int(top_k), len(sorted_idx))
|
||||
for i in range(len(sorted_idx)):
|
||||
if cum_prob < top_p and len(kept_probs) < max_keep:
|
||||
cum_prob += float(sorted_prob[i].item())
|
||||
kept_probs.append(sorted_prob[i])
|
||||
kept_indices.append(sorted_idx[i])
|
||||
else:
|
||||
break
|
||||
|
||||
if not kept_probs:
|
||||
return int(sorted_idx[0].item())
|
||||
|
||||
sample_probs = torch.stack(kept_probs)
|
||||
sample_idx = cls._multinomial_sample(sample_probs, generator=generator)
|
||||
return int(torch.stack(kept_indices)[int(sample_idx.item())].item())
|
||||
|
||||
@classmethod
|
||||
def _ras_sample_one(
|
||||
cls,
|
||||
weighted_scores: torch.Tensor,
|
||||
decoded_tokens: Sequence[int],
|
||||
*,
|
||||
top_p: float,
|
||||
top_k: int,
|
||||
win_size: int,
|
||||
tau_r: float,
|
||||
generator: torch.Generator | None,
|
||||
) -> int:
|
||||
top_id = cls._nucleus_sample_one(
|
||||
weighted_scores,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
generator=generator,
|
||||
)
|
||||
if win_size > 0 and decoded_tokens:
|
||||
recent = torch.as_tensor(
|
||||
list(decoded_tokens[-win_size:]),
|
||||
device=weighted_scores.device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
rep_num = int((recent == top_id).sum().item())
|
||||
if rep_num >= win_size * tau_r:
|
||||
weighted_scores = weighted_scores.clone()
|
||||
weighted_scores[top_id] = float("-inf")
|
||||
fallback_probs = weighted_scores.softmax(dim=0)
|
||||
top_id = int(cls._multinomial_sample(fallback_probs, generator=generator).item())
|
||||
return top_id
|
||||
|
||||
def _cosyvoice3_ras_enabled(self, sampling_metadata: SamplingMetadata) -> bool:
|
||||
if self.model_stage != "cosyvoice3_talker":
|
||||
return False
|
||||
if sampling_metadata.max_num_logprobs is not None:
|
||||
return False
|
||||
if sampling_metadata.temperature is None:
|
||||
return False
|
||||
if bool(sampling_metadata.bad_words_token_ids):
|
||||
return False
|
||||
if torch.any(sampling_metadata.frequency_penalties != 0):
|
||||
return False
|
||||
if torch.any(sampling_metadata.presence_penalties != 0):
|
||||
return False
|
||||
return True
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> SamplerOutput | None:
|
||||
if logits is None or logits.numel() == 0:
|
||||
return None
|
||||
if self.model_stage != "cosyvoice3_talker":
|
||||
return None
|
||||
|
||||
sampler = getattr(self, "_talker_sampler", None)
|
||||
if sampler is None:
|
||||
sampler = Sampler()
|
||||
self._talker_sampler = sampler
|
||||
|
||||
if not self._cosyvoice3_ras_enabled(sampling_metadata):
|
||||
return sampler(logits=logits, sampling_metadata=sampling_metadata)
|
||||
|
||||
logits = logits.to(torch.float32)
|
||||
sampling_for_processors = replace(sampling_metadata, no_penalties=True)
|
||||
logits = sampler.apply_logits_processors(logits, sampling_for_processors, predict_bonus_token=False)
|
||||
|
||||
sampling_cfg = dict(self.config.llm.get("sampling", {}))
|
||||
default_top_p = float(sampling_cfg.get("top_p", 0.8))
|
||||
default_top_k = int(sampling_cfg.get("top_k", 25))
|
||||
win_size = int(sampling_cfg.get("win_size", 10))
|
||||
tau_r = float(sampling_cfg.get("tau_r", 0.1))
|
||||
|
||||
sampled_ids: list[int] = []
|
||||
for req_idx in range(int(logits.shape[0])):
|
||||
row_logits = logits[req_idx]
|
||||
|
||||
temperature = float(self._req_scalar(sampling_metadata.temperature, req_idx, 1.0))
|
||||
if temperature < self._sampling_eps:
|
||||
sampled_ids.append(int(torch.argmax(row_logits).item()))
|
||||
continue
|
||||
|
||||
top_p = float(self._req_scalar(sampling_metadata.top_p, req_idx, default_top_p))
|
||||
top_k = int(self._req_scalar(sampling_metadata.top_k, req_idx, default_top_k))
|
||||
generator = sampling_metadata.generators.get(req_idx)
|
||||
weighted_scores = torch.log_softmax(row_logits / max(temperature, self._sampling_eps), dim=0)
|
||||
decoded_tokens = (
|
||||
sampling_metadata.output_token_ids[req_idx] if req_idx < len(sampling_metadata.output_token_ids) else []
|
||||
)
|
||||
sampled_ids.append(
|
||||
self._ras_sample_one(
|
||||
weighted_scores,
|
||||
decoded_tokens,
|
||||
top_p=top_p,
|
||||
top_k=top_k,
|
||||
win_size=win_size,
|
||||
tau_r=tau_r,
|
||||
generator=generator,
|
||||
)
|
||||
)
|
||||
|
||||
sampled = torch.tensor(sampled_ids, device=logits.device, dtype=torch.int32)
|
||||
return SamplerOutput(sampled_token_ids=sampled.unsqueeze(-1), logprobs_tensors=None)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
logits = self.model.llm_decoder(hidden_states)
|
||||
# The decoder outputs speech_token_size + 200 logits. The official
|
||||
# CosyVoice3 treats ALL tokens >= speech_token_size (the last 200)
|
||||
# as stop signals. Merge their probabilities into a single EOS
|
||||
# token (6562) via logsumexp so that vLLM's stop_token_ids=[6562]
|
||||
# fires with the correct aggregate stop probability.
|
||||
speech_token_size = self.config.llm["speech_token_size"]
|
||||
eos_idx = self.config.llm["eos_token_id"]
|
||||
stop_logits = logits[..., speech_token_size:] # last 200
|
||||
merged_stop = torch.logsumexp(stop_logits, dim=-1, keepdim=True)
|
||||
logits[..., speech_token_size:] = float("-inf") # mask all
|
||||
logits[..., eos_idx] = merged_stop.squeeze(-1) # restore merged
|
||||
# Pad to full vocab_size for vLLM token handling.
|
||||
vocab_size = self.config.vocab_size
|
||||
pad_size = vocab_size - logits.size(-1)
|
||||
pad_shape = logits.shape[:-1] + (pad_size,)
|
||||
pad = logits.new_full(pad_shape, float("-inf"))
|
||||
eos_token_val = logits[..., self.config.llm["eos_token_id"]].clone()
|
||||
logits[..., -200:] = float("-inf")
|
||||
logits[..., self.config.llm["eos_token_id"]] = eos_token_val
|
||||
logits = torch.cat([logits, pad], dim=-1)
|
||||
if pad_size > 0:
|
||||
pad_shape = logits.shape[:-1] + (pad_size,)
|
||||
pad = logits.new_full(pad_shape, float("-inf"))
|
||||
logits = torch.cat([logits, pad], dim=-1)
|
||||
return logits
|
||||
else:
|
||||
raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.")
|
||||
@@ -380,6 +650,7 @@ class CosyVoice3Model(
|
||||
hidden = int(self.config.hidden_size)
|
||||
return torch.zeros(
|
||||
(input_ids.shape[0], hidden),
|
||||
device=input_ids.device,
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(f"embed_input_ids is not valid for {self.model_stage}.")
|
||||
@@ -412,28 +683,116 @@ class CosyVoice3Model(
|
||||
|
||||
return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs)
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
runtime_info = kwargs.get("runtime_additional_information", [])
|
||||
if not runtime_info:
|
||||
length = 30 * 24000
|
||||
audio = np.zeros((length,))
|
||||
return OmniOutput(text_hidden_states=None, multimodal_outputs={"audio": audio})
|
||||
runtime_info = kwargs.get("model_intermediate_buffer")
|
||||
if runtime_info is None:
|
||||
runtime_info = kwargs.get("runtime_additional_information", [])
|
||||
if "runtime_additional_information" in kwargs and "model_intermediate_buffer" not in kwargs:
|
||||
logger.warning_once("runtime_additional_information is deprecated, use model_intermediate_buffer")
|
||||
|
||||
# Remove the last eos token and add batch dimension
|
||||
token = input_ids[..., :-1].unsqueeze(0)
|
||||
seq_token_counts = kwargs.get("seq_token_counts")
|
||||
flat_ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
request_ids_list = self._split_request_ids(flat_ids, seq_token_counts)
|
||||
|
||||
# Generate audio using code2wav
|
||||
tts_speech = self.code2wav(
|
||||
token=token,
|
||||
prompt_token=runtime_info[0]["speech_token"][:1],
|
||||
prompt_feat=runtime_info[0]["speech_feat"][:1],
|
||||
embedding=runtime_info[0]["embedding"][:1],
|
||||
n_timesteps=10,
|
||||
)
|
||||
num_reqs = max(1, len(request_ids_list))
|
||||
sample_rate = torch.tensor(int(self.config.sample_rate), dtype=torch.int32)
|
||||
empty_audio = torch.zeros((0,), dtype=torch.float32, device=input_ids.device)
|
||||
audios: list[torch.Tensor] = [empty_audio] * num_reqs
|
||||
srs: list[torch.Tensor] = [sample_rate] * num_reqs
|
||||
if not isinstance(runtime_info, list):
|
||||
runtime_info = []
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"audio": tts_speech, "sr": torch.tensor(22050)},
|
||||
)
|
||||
for idx, req_ids in enumerate(request_ids_list):
|
||||
info = runtime_info[idx] if idx < len(runtime_info) and isinstance(runtime_info[idx], dict) else {}
|
||||
req_id = self._as_str(info.get("req_id")) if info else None
|
||||
stream_finished = self._as_bool(info.get("stream_finished")) if info else False
|
||||
speech_token = self._as_tensor(info.get("speech_token")) if info else None
|
||||
speech_feat = self._as_tensor(info.get("speech_feat")) if info else None
|
||||
embedding = self._as_tensor(info.get("embedding")) if info else None
|
||||
if speech_token is None or speech_feat is None or embedding is None:
|
||||
if stream_finished and req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
|
||||
with self._stream_audio_cache_lock:
|
||||
self._stream_vocoder_cache_by_req.pop(req_id, None)
|
||||
audios[idx] = self._stitch_stream_audio(req_id, empty_audio, stream_finished)
|
||||
if (
|
||||
req_ids.numel() > 0
|
||||
and info
|
||||
and ("token_offset" in info or "left_context_size" in info or "generated_len" in info)
|
||||
):
|
||||
info_keys = ",".join(sorted(info.keys())) if info else ""
|
||||
logger.warning_once(
|
||||
"CosyVoice3 code2wav missing prompt conditioning for non-empty codec tokens: "
|
||||
"raw_len=%d info_keys=%s",
|
||||
int(req_ids.numel()),
|
||||
info_keys,
|
||||
)
|
||||
continue
|
||||
|
||||
token = self._sanitize_codec_tokens(req_ids)
|
||||
if token.numel() == 0:
|
||||
audios[idx] = self._stitch_stream_audio(req_id, empty_audio, stream_finished)
|
||||
if req_ids.numel() > 0:
|
||||
logger.warning_once(
|
||||
"CosyVoice3 code2wav received no valid codec tokens after filtering: "
|
||||
"raw_len=%d raw_range=[%d,%d] vocab_size=%d",
|
||||
req_ids.numel(),
|
||||
int(req_ids.min().item()),
|
||||
int(req_ids.max().item()),
|
||||
int(self.code2wav.input_embedding.num_embeddings),
|
||||
)
|
||||
continue
|
||||
|
||||
# `generated_len` is injected for many models by the generic
|
||||
# runner, so only explicit chunk-routing fields should switch
|
||||
# code2wav into the streaming path.
|
||||
uses_streaming_decode = bool(info) and (
|
||||
"stream_finished" in info or "token_offset" in info or "left_context_size" in info
|
||||
)
|
||||
if uses_streaming_decode:
|
||||
token_offset = 0
|
||||
try:
|
||||
if info and "token_offset" in info:
|
||||
token_offset = max(0, int(info.get("token_offset", 0)))
|
||||
elif info:
|
||||
token_offset = max(0, int(info.get("left_context_size", 0)))
|
||||
except (TypeError, ValueError):
|
||||
token_offset = 0
|
||||
|
||||
cache_state = None
|
||||
if req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
|
||||
with self._stream_audio_cache_lock:
|
||||
cache_state = self._stream_vocoder_cache_by_req.get(req_id)
|
||||
|
||||
tts_speech, new_cache_state = self.code2wav.forward_streaming(
|
||||
token=token.unsqueeze(0),
|
||||
prompt_token=speech_token[:1],
|
||||
prompt_feat=speech_feat[:1],
|
||||
embedding=embedding[:1],
|
||||
cache_state=cache_state,
|
||||
n_timesteps=10,
|
||||
token_offset_tokens=token_offset,
|
||||
finalize=stream_finished,
|
||||
)
|
||||
|
||||
if req_id is not None and hasattr(self, "_stream_vocoder_cache_by_req"):
|
||||
with self._stream_audio_cache_lock:
|
||||
if new_cache_state is None or stream_finished:
|
||||
self._stream_vocoder_cache_by_req.pop(req_id, None)
|
||||
else:
|
||||
self._stream_vocoder_cache_by_req[req_id] = new_cache_state
|
||||
else:
|
||||
tts_speech = self.code2wav.forward(
|
||||
token=token.unsqueeze(0),
|
||||
prompt_token=speech_token[:1],
|
||||
prompt_feat=speech_feat[:1],
|
||||
embedding=embedding[:1],
|
||||
n_timesteps=10,
|
||||
)
|
||||
|
||||
audio = tts_speech.reshape(-1).to(dtype=torch.float32)
|
||||
|
||||
audios[idx] = self._stitch_stream_audio(req_id, audio, stream_finished)
|
||||
|
||||
return OmniOutput(text_hidden_states=None, multimodal_outputs={"audio": audios, "sr": srs})
|
||||
else:
|
||||
raise ValueError(f"Unsupported model_stage: {self.model_stage}")
|
||||
|
||||
|
||||
@@ -11,11 +11,12 @@ This module contains the code2wav (token-to-waveform) stage which uses:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import nullcontext
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from omegaconf import DictConfig
|
||||
from torch.nn import functional as F
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.cosyvoice3_audio.cosyvoice3_dit import DiT
|
||||
@@ -29,7 +30,6 @@ from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan import (
|
||||
)
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.layers import PreLookaheadLayer
|
||||
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import make_pad_mask
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -150,6 +150,140 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
"""Speaker embedding affine layer."""
|
||||
return self.flow_model.spk_embed_affine_layer
|
||||
|
||||
@torch.inference_mode()
|
||||
def _forward_mel(
|
||||
self,
|
||||
token: torch.Tensor,
|
||||
prompt_token: torch.Tensor,
|
||||
prompt_feat: torch.Tensor,
|
||||
embedding: torch.Tensor,
|
||||
n_timesteps: int = 10,
|
||||
token_offset_tokens: int = 0,
|
||||
streaming: bool = True,
|
||||
finalize: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Generate mel features via the upstream flow-model inference path."""
|
||||
flow_weight = next(self.flow_model.parameters())
|
||||
device = flow_weight.device
|
||||
dtype = flow_weight.dtype
|
||||
|
||||
token = token.to(device=device, dtype=torch.int32)
|
||||
prompt_token = prompt_token.to(device=device, dtype=torch.int32)
|
||||
prompt_feat = prompt_feat.to(device=device, dtype=dtype)
|
||||
embedding = embedding.to(device=device, dtype=dtype)
|
||||
token_len = torch.tensor([token.shape[1]], device=device, dtype=torch.int32)
|
||||
prompt_token_len = torch.tensor([prompt_token.shape[1]], device=device, dtype=torch.int32)
|
||||
prompt_feat_len = torch.tensor([prompt_feat.shape[1]], device=device, dtype=torch.int32)
|
||||
|
||||
with nullcontext():
|
||||
feat, _ = self.flow_model.inference(
|
||||
token=token,
|
||||
token_len=token_len,
|
||||
prompt_token=prompt_token,
|
||||
prompt_token_len=prompt_token_len,
|
||||
prompt_feat=prompt_feat,
|
||||
prompt_feat_len=prompt_feat_len,
|
||||
embedding=embedding,
|
||||
streaming=streaming,
|
||||
finalize=finalize,
|
||||
n_timesteps=n_timesteps,
|
||||
)
|
||||
|
||||
trim_mel = max(0, int(token_offset_tokens)) * int(self.token_mel_ratio)
|
||||
if trim_mel > 0:
|
||||
feat = feat[:, :, trim_mel:]
|
||||
|
||||
return feat
|
||||
|
||||
@staticmethod
|
||||
def _fade_speech(
|
||||
speech: torch.Tensor,
|
||||
prev_speech: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Blend previous speech tail into current speech head."""
|
||||
if speech.numel() == 0 or prev_speech.numel() == 0:
|
||||
return speech
|
||||
overlap = min(int(speech.shape[-1]), int(prev_speech.shape[-1]))
|
||||
if overlap <= 0:
|
||||
return speech
|
||||
window = torch.hamming_window(2 * overlap, periodic=False, dtype=speech.dtype, device=speech.device)
|
||||
fade_in = window[:overlap].view(1, -1)
|
||||
fade_out = window[overlap:].view(1, -1)
|
||||
blended_head = (
|
||||
speech[:, :overlap] * fade_in
|
||||
+ prev_speech[:, -overlap:].to(device=speech.device, dtype=speech.dtype) * fade_out
|
||||
)
|
||||
if overlap == int(speech.shape[-1]):
|
||||
return blended_head
|
||||
return torch.cat([blended_head, speech[:, overlap:]], dim=-1)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward_streaming(
|
||||
self,
|
||||
token: torch.Tensor,
|
||||
prompt_token: torch.Tensor,
|
||||
prompt_feat: torch.Tensor,
|
||||
embedding: torch.Tensor,
|
||||
*,
|
||||
cache_state: dict[str, torch.Tensor] | None = None,
|
||||
n_timesteps: int = 10,
|
||||
token_offset_tokens: int = 0,
|
||||
finalize: bool = False,
|
||||
) -> tuple[torch.Tensor, dict[str, torch.Tensor] | None]:
|
||||
"""Decode streaming audio using cumulative mel + emitted-speech offset.
|
||||
|
||||
This mirrors upstream CosyVoice3 streaming semantics more closely than
|
||||
waveform-domain overlap-add: keep a cumulative mel history per request,
|
||||
re-run causal HiFT on the history, and emit only the newly grown speech
|
||||
suffix. That preserves causal look-right handling without double
|
||||
trimming or duplicated overlap at chunk boundaries.
|
||||
"""
|
||||
with nullcontext():
|
||||
feat = self._forward_mel(
|
||||
token=token,
|
||||
prompt_token=prompt_token,
|
||||
prompt_feat=prompt_feat,
|
||||
embedding=embedding,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=token_offset_tokens,
|
||||
streaming=True,
|
||||
finalize=finalize,
|
||||
)
|
||||
hift_weight = self.hift.m_source.l_linear.weight
|
||||
chunk_mel = feat.to(device=hift_weight.device, dtype=hift_weight.dtype)
|
||||
|
||||
cached_mel = None if not cache_state else cache_state.get("mel")
|
||||
speech_offset_obj = None if not cache_state else cache_state.get("speech_offset")
|
||||
try:
|
||||
speech_offset = int(speech_offset_obj) if speech_offset_obj is not None else 0
|
||||
except (TypeError, ValueError):
|
||||
speech_offset = 0
|
||||
|
||||
if isinstance(cached_mel, torch.Tensor) and cached_mel.numel() > 0:
|
||||
cached_mel = cached_mel.to(device=chunk_mel.device, dtype=chunk_mel.dtype)
|
||||
tts_mel = torch.cat([cached_mel, chunk_mel], dim=-1) if chunk_mel.numel() > 0 else cached_mel
|
||||
else:
|
||||
tts_mel = chunk_mel
|
||||
|
||||
if tts_mel.shape[-1] == 0:
|
||||
tts_speech = torch.zeros((chunk_mel.shape[0], 1, 0), device=chunk_mel.device, dtype=chunk_mel.dtype)
|
||||
else:
|
||||
with nullcontext():
|
||||
tts_speech, _ = self.hift.inference(speech_feat=tts_mel, finalize=finalize)
|
||||
|
||||
tts_speech = tts_speech.reshape(tts_speech.shape[0], -1)
|
||||
speech_offset = max(0, min(speech_offset, int(tts_speech.shape[-1])))
|
||||
emitted_speech = tts_speech[:, speech_offset:]
|
||||
|
||||
if finalize:
|
||||
return emitted_speech.reshape(emitted_speech.shape[0], 1, -1), None
|
||||
|
||||
new_state = {
|
||||
"mel": tts_mel.detach().cpu().contiguous(),
|
||||
"speech_offset": int(tts_speech.shape[-1]),
|
||||
}
|
||||
return emitted_speech.reshape(emitted_speech.shape[0], 1, -1), new_state
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
@@ -159,76 +293,18 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
embedding: torch.Tensor,
|
||||
n_timesteps: int = 10,
|
||||
) -> torch.Tensor:
|
||||
"""Generate audio waveform from speech tokens.
|
||||
|
||||
Args:
|
||||
token: Speech tokens from talker stage [batch, seq_len]
|
||||
prompt_token: Prompt speech tokens [batch, prompt_len]
|
||||
prompt_feat: Prompt mel features [batch, feat_len, mel_dim]
|
||||
embedding: Speaker embedding [batch, spk_dim]
|
||||
n_timesteps: Number of diffusion steps
|
||||
|
||||
Returns:
|
||||
Audio waveform [batch, 1, audio_len]
|
||||
"""
|
||||
device = token.device
|
||||
dtype = next(self.flow_model.parameters()).dtype
|
||||
|
||||
# Normalize and project speaker embedding
|
||||
embedding = embedding.to(device=device, dtype=dtype)
|
||||
embedding = F.normalize(embedding, dim=1)
|
||||
embedding = self.spk_embed_affine_layer(embedding)
|
||||
|
||||
# Prepare tokens
|
||||
prompt_token = prompt_token.to(device=device)
|
||||
token_len1, token_len2 = prompt_token.shape[1], token.shape[1]
|
||||
prompt_token_len = torch.tensor([token_len1], device=device, dtype=torch.int32)
|
||||
token_len = torch.tensor([token_len2], device=device, dtype=torch.int32)
|
||||
|
||||
# Concatenate prompt and target tokens
|
||||
full_token = torch.cat([prompt_token, token], dim=1)
|
||||
full_token_len = prompt_token_len + token_len
|
||||
|
||||
# Create mask
|
||||
mask = (~make_pad_mask(full_token_len)).unsqueeze(-1).to(embedding)
|
||||
|
||||
# Token embedding (clamp to valid codebook range; EOS/padding tokens may exceed vocab_size)
|
||||
token_emb = (
|
||||
self.input_embedding(torch.clamp(full_token, min=0, max=self.input_embedding.num_embeddings - 1)) * mask
|
||||
)
|
||||
|
||||
# Pre-lookahead processing
|
||||
h = self.pre_lookahead_layer(token_emb)
|
||||
h = h.repeat_interleave(self.token_mel_ratio, dim=1)
|
||||
|
||||
# Calculate mel lengths
|
||||
mel_len1 = prompt_feat.shape[1]
|
||||
mel_len2 = h.shape[1] - mel_len1
|
||||
|
||||
# Build conditioning
|
||||
conds = torch.zeros(
|
||||
[1, mel_len1 + mel_len2, self.output_size],
|
||||
device=device,
|
||||
dtype=h.dtype,
|
||||
)
|
||||
conds[:, :mel_len1] = prompt_feat
|
||||
conds = conds.transpose(1, 2)
|
||||
|
||||
# Create mel mask
|
||||
mel_mask = (~make_pad_mask(torch.tensor([mel_len1 + mel_len2]))).to(h)
|
||||
|
||||
# Run flow matching decoder
|
||||
feat, _ = self.decoder(
|
||||
mu=h.transpose(1, 2).contiguous(),
|
||||
mask=mel_mask.unsqueeze(1),
|
||||
spks=embedding,
|
||||
cond=conds,
|
||||
"""Generate audio waveform from speech tokens."""
|
||||
feat = self._forward_mel(
|
||||
token=token,
|
||||
prompt_token=prompt_token,
|
||||
prompt_feat=prompt_feat,
|
||||
embedding=embedding,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=0,
|
||||
streaming=False,
|
||||
finalize=True,
|
||||
)
|
||||
|
||||
# Extract generated portion (after prompt)
|
||||
feat = feat[:, :, mel_len1:]
|
||||
|
||||
# Run vocoder
|
||||
hift_weight = self.hift.m_source.l_linear.weight
|
||||
tts_mel = feat.to(device=hift_weight.device, dtype=hift_weight.dtype)
|
||||
@@ -240,7 +316,7 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
dtype=tts_mel.dtype,
|
||||
)
|
||||
else:
|
||||
tts_speech, _ = self.hift.inference(speech_feat=tts_mel)
|
||||
tts_speech, _ = self.hift.inference(speech_feat=tts_mel, finalize=True)
|
||||
|
||||
return tts_speech
|
||||
|
||||
|
||||
Reference in new issue
Block a user