[TTS][Perf] Optimize Fish Speech S2 Pro high-concurrency serving (#3773)

Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Sy03andYueqian Lin authored and GitHub committed 2026-05-27 13:25:49 -05:00
1 parent 37eebff674
commit d37912b071
23 files changed
+3453 -92

No files matched your search

@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm_omni.model_executor.models.fish_speech.configuration_fish_speech import (
FishSpeechFastARConfig,
FishSpeechSlowARConfig,
)
from vllm_omni.model_executor.models.fish_speech.fish_speech_fast_ar import FishSpeechFastAR
from vllm_omni.model_executor.models.fish_speech.fish_speech_slow_ar import (
FishSpeechSlowARForConditionalGeneration,
)
def test_fast_ar_reuses_dense_position_id_buffer(monkeypatch):
fast_config = FishSpeechFastARConfig(
vocab_size=16,
num_codebooks=4,
dim=8,
n_head=2,
n_local_heads=1,
head_dim=4,
n_layer=1,
intermediate_size=16,
max_seq_len=5,
)
slow_config = FishSpeechSlowARConfig(
vocab_size=32,
dim=8,
n_head=2,
n_local_heads=1,
head_dim=4,
n_layer=1,
intermediate_size=16,
codebook_size=16,
num_codebooks=4,
semantic_begin_id=4,
semantic_end_id=19,
)
fast_ar = object.__new__(FishSpeechFastAR)
nn.Module.__init__(fast_ar)
fast_ar.config = fast_config
fast_ar.slow_ar_config = slow_config
fast_ar.fast_project_in = nn.Identity()
fast_ar.fast_embeddings = nn.Embedding(fast_config.vocab_size, fast_config.hidden_size)
fast_ar.fast_output = nn.Linear(fast_config.hidden_size, fast_config.vocab_size, bias=False)
fast_ar.fast_norm = nn.Identity()
fast_ar._num_codebooks = fast_config.num_codebooks
fast_ar._fast_dim = fast_config.hidden_size
fast_ar._embed_buf = None
fast_ar._pos_ids = None
fast_ar._k_cache = None
fast_ar._v_cache = None
fast_ar._compiled_model_fwd = None
fast_ar._compile_attempted = True
fast_ar._compile_failed = False
fast_ar._disable_compile_for_graph = False
seen_position_ids: list[torch.Tensor] = []
def fake_run_model_one(step_input, step_pos_ids, cache_pos):
seen_position_ids.append(step_pos_ids)
return torch.zeros(
step_input.shape[0],
fast_config.hidden_size,
dtype=step_input.dtype,
device=step_input.device,
)
monkeypatch.setattr(fast_ar, "_run_model_one", fake_run_model_one)
hidden = torch.zeros(3, slow_config.hidden_size, dtype=torch.float32)
semantic = torch.full((3,), slow_config.semantic_begin_id, dtype=torch.long)
fast_ar(hidden, semantic, do_sample=False)
assert fast_ar._pos_ids is not None
assert fast_ar._pos_ids.shape == (3, fast_config.num_codebooks + 1)
assert len(seen_position_ids) == fast_config.num_codebooks
for step, step_pos_ids in enumerate(seen_position_ids):
assert step_pos_ids.shape == (3,)
assert step_pos_ids.untyped_storage().data_ptr() == fast_ar._pos_ids.untyped_storage().data_ptr()
assert step_pos_ids.stride() == (fast_ar._pos_ids.stride(0),)
expected = torch.full((3,), step)
assert torch.equal(step_pos_ids.cpu(), expected)
def test_talker_mtp_does_not_mutate_input():
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
nn.Module.__init__(model)
model._semantic_begin_id = 4
model._semantic_end_id = 19
model._codebook_size = 16
model._num_codebooks = 4
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
model.fast_ar = lambda **_: torch.tensor(
[
[0, 1, 2, 3],
[15, 14, 13, 12],
],
dtype=torch.long,
)
input_ids = torch.tensor([4, 2], dtype=torch.long)
input_embeds = torch.randn(2, 8)
input_embeds_before = input_embeds.clone()
last_hidden = torch.randn(2, 8, dtype=torch.bfloat16)
text_step = torch.zeros(2, 8, dtype=torch.bfloat16)
out1, _ = FishSpeechSlowARForConditionalGeneration.talker_mtp(
model,
input_ids,
input_embeds,
last_hidden,
text_step,
)
assert torch.equal(input_embeds, input_embeds_before)
assert out1.untyped_storage().data_ptr() != input_embeds.untyped_storage().data_ptr()
def test_talker_mtp_forwards_sampling_params():
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
nn.Module.__init__(model)
model._semantic_begin_id = 4
model._semantic_end_id = 19
model._codebook_size = 16
model._num_codebooks = 4
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
seen = {}
def fake_fast_ar(**kwargs):
seen.update(kwargs)
return torch.tensor([[0, 1, 2, 3]], dtype=torch.long)
model.fast_ar = fake_fast_ar
input_ids = torch.tensor([4], dtype=torch.long)
input_embeds = torch.randn(1, 8)
last_hidden = torch.randn(1, 8, dtype=torch.bfloat16)
text_step = torch.zeros(1, 8, dtype=torch.bfloat16)
FishSpeechSlowARForConditionalGeneration.talker_mtp(
model,
input_ids,
input_embeds,
last_hidden,
text_step,
do_sample=False,
temperature=0.1,
top_k=5,
top_p=0.7,
)
assert seen["do_sample"] is False
assert seen["temperature"] == 0.1
assert seen["top_k"] == 5
assert seen["top_p"] == 0.7
@@ -1,5 +1,6 @@
import functools
import math
from types import SimpleNamespace
import pytest
import torch
@@ -26,6 +27,39 @@ class _FakeCodec:
return wav, audio_lengths
class _RecordingCodec:
def __init__(self):
self.calls = []
def decode(self, codes_bqf: torch.Tensor, feature_lengths: torch.Tensor):
self.calls.append((tuple(codes_bqf.shape), tuple(int(x) for x in feature_lengths.tolist())))
batch = codes_bqf.shape[0]
wav_len = int(codes_bqf.shape[-1]) * 10
wav = torch.arange(batch * wav_len, dtype=torch.float32).view(batch, 1, wav_len)
audio_lengths = feature_lengths.to(torch.long) * 10
return wav, audio_lengths
class _FakeOneArgCodec:
def decode(self, codes_bqf: torch.Tensor):
batch = codes_bqf.shape[0]
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
class _FakeFromIndicesCodec:
def __init__(self):
self.from_indices_calls = 0
def from_indices(self, codes_bqf: torch.Tensor):
self.from_indices_calls += 1
batch = codes_bqf.shape[0]
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
def decode(self, codes_bqf: torch.Tensor):
del codes_bqf
raise AssertionError("decode() should not be used for index tensors when from_indices() exists")
class _FakeTokenizer:
def __init__(self, mapping, unk_token_id=-1):
self._mapping = mapping
@@ -40,6 +74,7 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = _FakeCodec()
decoder._codec_decode_takes_lengths = True
decoder._num_codebooks = 10
decoder._output_sample_rate = 44100
decoder._hop_length = 512
@@ -62,6 +97,132 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
assert audios[1].shape[0] == 50
def test_dac_decoder_uses_tensor_codes_from_runtime_info():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = _FakeCodec()
decoder._codec_decode_takes_lengths = True
decoder._num_codebooks = 2
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._ensure_codec_loaded = lambda: None
out = decoder.forward(
input_ids=torch.tensor([0], dtype=torch.long),
runtime_additional_information=[
{
"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)},
"meta": {"left_context_size": 1},
}
],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 1
# 2 total frames with 1 frame of left context => proportional trim removes half the samples.
assert audios[0].shape[0] == 50
def test_dac_decoder_supports_codec_decode_without_lengths():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = _FakeOneArgCodec()
decoder._codec_decode_takes_lengths = False
decoder._num_codebooks = 10
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._ensure_codec_loaded = lambda: None
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
torch.arange(10, dtype=torch.long),
torch.arange(20, dtype=torch.long),
]
out = decoder.forward(
input_ids=torch.arange(30, dtype=torch.long),
runtime_additional_information=[{}, {}],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 2
assert audios[0].shape[0] == 50
assert audios[1].shape[0] == 100
def test_dac_decoder_prefers_from_indices_for_one_arg_codec():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
codec = _FakeFromIndicesCodec()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = codec
decoder._codec_decode_takes_lengths = False
decoder._num_codebooks = 10
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._ensure_codec_loaded = lambda: None
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
torch.arange(10, dtype=torch.long),
torch.arange(20, dtype=torch.long),
]
out = decoder.forward(
input_ids=torch.arange(30, dtype=torch.long),
runtime_additional_information=[{}, {}],
)
audios = out.multimodal_outputs["model_outputs"]
assert codec.from_indices_calls == 1
assert len(audios) == 2
assert audios[0].shape[0] == 50
assert audios[1].shape[0] == 100
def test_dac_decoder_splits_groups_by_padded_frame_budget():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
codec = _RecordingCodec()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = codec
decoder._codec_decode_takes_lengths = True
decoder._num_codebooks = 2
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._decode_batch_max_padded_frames = 100
decoder._decode_batch_max_batch = 0
decoder._ensure_codec_loaded = lambda: None
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
torch.arange(2 * 46, dtype=torch.long),
torch.arange(2 * 45, dtype=torch.long),
torch.arange(2 * 47, dtype=torch.long),
]
decoder.forward(input_ids=torch.arange(2 * (46 + 45 + 47), dtype=torch.long))
assert codec.calls == [
((2, 2, 46), (45, 46)),
((1, 2, 47), (47,)),
]
def test_dac_decoder_dtype_can_be_configured():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
cfg = SimpleNamespace(
model_config=SimpleNamespace(
model="unused",
stage_connector_config={"extra": {"fish_speech_dac_dtype": "fp16"}},
)
)
decoder = FishSpeechDACDecoder(vllm_config=cfg)
assert decoder._dac_dtype == torch.float16
def test_structured_voice_clone_prefill_adds_full_codebooks_with_decode_scale(monkeypatch):
slow_ar_module, _, FishSpeechSlowARForConditionalGeneration = _fish_speech_regression_modules()
model = object.__new__(FishSpeechSlowARForConditionalGeneration)