Optimize CosyVoice3 Stage1 flow batching (#4876)
Signed-off-by: gerayking <399geray@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
28291df179
commit
9edc1b4667
10 files changed
+825
-198
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""
|
||||
CosyVoice3 Code2Wav Stage - Converts speech tokens to audio waveforms.
|
||||
|
||||
@@ -11,6 +11,9 @@ This module contains the code2wav (token-to-waveform) stage which uses:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -27,6 +30,10 @@ from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan import (
|
||||
CausalHiFTGenerator,
|
||||
)
|
||||
from vllm_omni.model_executor.models.cosyvoice3.code2wav_core.layers import PreLookaheadLayer
|
||||
from vllm_omni.model_executor.models.cosyvoice3.runtime import (
|
||||
cosyvoice3_batch_flow_debug,
|
||||
cosyvoice3_batch_flow_profile,
|
||||
)
|
||||
from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -159,6 +166,9 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
token_offset_tokens: int = 0,
|
||||
streaming: bool = True,
|
||||
finalize: bool = False,
|
||||
token_lens: torch.Tensor | None = None,
|
||||
prompt_token_lens: torch.Tensor | None = None,
|
||||
prompt_feat_lens: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Generate mel features via the upstream flow-model inference path."""
|
||||
flow_weight = next(self.flow_model.parameters())
|
||||
@@ -169,9 +179,22 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
prompt_token = prompt_token.to(device=device, dtype=torch.int32)
|
||||
prompt_feat = prompt_feat.to(device=device, dtype=dtype)
|
||||
embedding = embedding.to(device=device, dtype=dtype)
|
||||
token_len = torch.tensor([token.shape[1]], device=device, dtype=torch.int32)
|
||||
prompt_token_len = torch.tensor([prompt_token.shape[1]], device=device, dtype=torch.int32)
|
||||
prompt_feat_len = torch.tensor([prompt_feat.shape[1]], device=device, dtype=torch.int32)
|
||||
batch_size = int(token.shape[0])
|
||||
token_len = (
|
||||
token_lens.to(device=device, dtype=torch.int32)
|
||||
if token_lens is not None
|
||||
else torch.full((batch_size,), token.shape[1], device=device, dtype=torch.int32)
|
||||
)
|
||||
prompt_token_len = (
|
||||
prompt_token_lens.to(device=device, dtype=torch.int32)
|
||||
if prompt_token_lens is not None
|
||||
else torch.full((batch_size,), prompt_token.shape[1], device=device, dtype=torch.int32)
|
||||
)
|
||||
prompt_feat_len = (
|
||||
prompt_feat_lens.to(device=device, dtype=torch.int32)
|
||||
if prompt_feat_lens is not None
|
||||
else torch.full((batch_size,), prompt_feat.shape[1], device=device, dtype=torch.int32)
|
||||
)
|
||||
|
||||
feat, _ = self.flow_model.inference(
|
||||
token=token,
|
||||
@@ -192,37 +215,13 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
|
||||
return feat
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward_streaming(
|
||||
def _stream_hift_from_feat(
|
||||
self,
|
||||
token: torch.Tensor,
|
||||
prompt_token: torch.Tensor,
|
||||
prompt_feat: torch.Tensor,
|
||||
embedding: torch.Tensor,
|
||||
feat: torch.Tensor,
|
||||
*,
|
||||
cache_state: dict[str, torch.Tensor] | None = None,
|
||||
n_timesteps: int = 10,
|
||||
token_offset_tokens: int = 0,
|
||||
finalize: bool = False,
|
||||
) -> tuple[torch.Tensor, dict[str, torch.Tensor] | None]:
|
||||
"""Decode streaming audio using cumulative mel + emitted-speech offset.
|
||||
|
||||
This mirrors upstream CosyVoice3 streaming semantics more closely than
|
||||
waveform-domain overlap-add: keep a cumulative mel history per request,
|
||||
re-run causal HiFT on the history, and emit only the newly grown speech
|
||||
suffix. That preserves causal look-right handling without double
|
||||
trimming or duplicated overlap at chunk boundaries.
|
||||
"""
|
||||
feat = self._forward_mel(
|
||||
token=token,
|
||||
prompt_token=prompt_token,
|
||||
prompt_feat=prompt_feat,
|
||||
embedding=embedding,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=token_offset_tokens,
|
||||
streaming=True,
|
||||
finalize=finalize,
|
||||
)
|
||||
hift_weight = self.hift.m_source.l_linear.weight
|
||||
chunk_mel = feat.to(device=hift_weight.device, dtype=hift_weight.dtype)
|
||||
|
||||
@@ -257,6 +256,160 @@ class CosyVoice3Code2Wav(nn.Module):
|
||||
}
|
||||
return emitted_speech.reshape(emitted_speech.shape[0], 1, -1), new_state
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward_streaming_batch(
|
||||
self,
|
||||
items: list[dict[str, object]],
|
||||
*,
|
||||
n_timesteps: int = 10,
|
||||
) -> list[tuple[torch.Tensor, dict[str, torch.Tensor] | None]]:
|
||||
"""Batch the flow-matching mel path, then run HiFT per request.
|
||||
|
||||
Items are grouped by prompt condition shape and finalization state.
|
||||
Codec tokens may have different lengths; those are padded within the
|
||||
group and passed to the flow as per-row token lengths.
|
||||
"""
|
||||
results: list[tuple[torch.Tensor, dict[str, torch.Tensor] | None] | None] = [None] * len(items)
|
||||
groups: dict[tuple[int, int, int, bool], list[tuple[int, dict[str, object]]]] = {}
|
||||
for index, item in enumerate(items):
|
||||
token = item["token"]
|
||||
prompt_token = item["prompt_token"]
|
||||
prompt_feat = item["prompt_feat"]
|
||||
embedding = item["embedding"]
|
||||
assert isinstance(token, torch.Tensor)
|
||||
assert isinstance(prompt_token, torch.Tensor)
|
||||
assert isinstance(prompt_feat, torch.Tensor)
|
||||
assert isinstance(embedding, torch.Tensor)
|
||||
key = (
|
||||
int(prompt_token.shape[1]),
|
||||
int(prompt_feat.shape[1]),
|
||||
int(embedding.shape[1]),
|
||||
bool(item.get("finalize", False)),
|
||||
)
|
||||
groups.setdefault(key, []).append((index, item))
|
||||
|
||||
if cosyvoice3_batch_flow_debug():
|
||||
group_summary = {key: len(group) for key, group in groups.items()}
|
||||
group_size_distribution = Counter(group_summary.values())
|
||||
batchable_items = sum(size for size in group_summary.values() if size > 1)
|
||||
logger.info(
|
||||
"CosyVoice3 code2wav debug: forward_streaming_batch items=%d "
|
||||
"groups=%s group_size_distribution=%s batchable_items=%d",
|
||||
len(items),
|
||||
group_summary,
|
||||
dict(sorted(group_size_distribution.items())),
|
||||
batchable_items,
|
||||
)
|
||||
|
||||
for _key, group in groups.items():
|
||||
if len(group) == 1:
|
||||
index, item = group[0]
|
||||
result = self.forward_streaming(
|
||||
token=item["token"], # type: ignore[arg-type]
|
||||
prompt_token=item["prompt_token"], # type: ignore[arg-type]
|
||||
prompt_feat=item["prompt_feat"], # type: ignore[arg-type]
|
||||
embedding=item["embedding"], # type: ignore[arg-type]
|
||||
cache_state=item.get("cache_state"), # type: ignore[arg-type]
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=int(item.get("token_offset_tokens", 0)),
|
||||
finalize=bool(item.get("finalize", False)),
|
||||
)
|
||||
results[index] = result
|
||||
continue
|
||||
|
||||
token_tensors = [item["token"] for _, item in group]
|
||||
assert all(isinstance(token, torch.Tensor) for token in token_tensors)
|
||||
token_lens = torch.tensor(
|
||||
[int(token.shape[1]) for token in token_tensors], # type: ignore[union-attr]
|
||||
dtype=torch.int32,
|
||||
)
|
||||
max_token_len = int(token_lens.max().item())
|
||||
padded_tokens = []
|
||||
for token in token_tensors:
|
||||
assert isinstance(token, torch.Tensor)
|
||||
if int(token.shape[1]) == max_token_len:
|
||||
padded_tokens.append(token)
|
||||
else:
|
||||
pad = torch.zeros(
|
||||
(token.shape[0], max_token_len - int(token.shape[1])),
|
||||
device=token.device,
|
||||
dtype=token.dtype,
|
||||
)
|
||||
padded_tokens.append(torch.cat([token, pad], dim=1))
|
||||
tokens = torch.cat(padded_tokens, dim=0)
|
||||
prompt_tokens = torch.cat([item["prompt_token"] for _, item in group], dim=0) # type: ignore[list-item]
|
||||
prompt_feats = torch.cat([item["prompt_feat"] for _, item in group], dim=0) # type: ignore[list-item]
|
||||
embeddings = torch.cat([item["embedding"] for _, item in group], dim=0) # type: ignore[list-item]
|
||||
prompt_token_lens = torch.full((len(group),), prompt_tokens.shape[1], dtype=torch.int32)
|
||||
prompt_feat_lens = torch.full((len(group),), prompt_feats.shape[1], dtype=torch.int32)
|
||||
finalize = bool(group[0][1].get("finalize", False))
|
||||
|
||||
with cosyvoice3_batch_flow_profile(f"cosyvoice3_flow_batch_b{len(group)}_t{tokens.shape[1]}"):
|
||||
feat = self._forward_mel(
|
||||
token=tokens,
|
||||
prompt_token=prompt_tokens,
|
||||
prompt_feat=prompt_feats,
|
||||
embedding=embeddings,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=0,
|
||||
streaming=True,
|
||||
finalize=finalize,
|
||||
token_lens=token_lens,
|
||||
prompt_token_lens=prompt_token_lens,
|
||||
prompt_feat_lens=prompt_feat_lens,
|
||||
)
|
||||
|
||||
for row, (index, item) in enumerate(group):
|
||||
trim_mel = max(0, int(item.get("token_offset_tokens", 0))) * int(self.token_mel_ratio)
|
||||
valid_tokens = int(token_lens[row].item())
|
||||
if not finalize:
|
||||
valid_tokens = max(0, valid_tokens - int(self.flow_model.pre_lookahead_len))
|
||||
valid_mel = valid_tokens * int(self.token_mel_ratio)
|
||||
row_feat = feat[row : row + 1, :, :valid_mel]
|
||||
if trim_mel > 0:
|
||||
row_feat = row_feat[:, :, trim_mel:]
|
||||
results[index] = self._stream_hift_from_feat(
|
||||
row_feat,
|
||||
cache_state=item.get("cache_state"), # type: ignore[arg-type]
|
||||
finalize=finalize,
|
||||
)
|
||||
|
||||
assert all(result is not None for result in results), "every streaming item must produce exactly one result"
|
||||
return cast(list[tuple[torch.Tensor, dict[str, torch.Tensor] | None]], results)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward_streaming(
|
||||
self,
|
||||
token: torch.Tensor,
|
||||
prompt_token: torch.Tensor,
|
||||
prompt_feat: torch.Tensor,
|
||||
embedding: torch.Tensor,
|
||||
*,
|
||||
cache_state: dict[str, torch.Tensor] | None = None,
|
||||
n_timesteps: int = 10,
|
||||
token_offset_tokens: int = 0,
|
||||
finalize: bool = False,
|
||||
) -> tuple[torch.Tensor, dict[str, torch.Tensor] | None]:
|
||||
"""Decode streaming audio using cumulative mel + emitted-speech offset.
|
||||
|
||||
This mirrors upstream CosyVoice3 streaming semantics more closely than
|
||||
waveform-domain overlap-add: keep a cumulative mel history per request,
|
||||
re-run causal HiFT on the history, and emit only the newly grown speech
|
||||
suffix. That preserves causal look-right handling without double
|
||||
trimming or duplicated overlap at chunk boundaries.
|
||||
"""
|
||||
feat = self._forward_mel(
|
||||
token=token,
|
||||
prompt_token=prompt_token,
|
||||
prompt_feat=prompt_feat,
|
||||
embedding=embedding,
|
||||
n_timesteps=n_timesteps,
|
||||
token_offset_tokens=token_offset_tokens,
|
||||
streaming=True,
|
||||
finalize=finalize,
|
||||
)
|
||||
return self._stream_hift_from_feat(feat, cache_state=cache_state, finalize=finalize)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in new issue
Block a user