[TTS][Perf] Optimize Fish Speech S2 Pro high-concurrency serving (#3773)
Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
37eebff674
commit
d37912b071
23 files changed
+3453
-92
No files matched your search
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.fish_speech.configuration_fish_speech import (
|
||||
FishSpeechFastARConfig,
|
||||
FishSpeechSlowARConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.fish_speech.fish_speech_fast_ar import FishSpeechFastAR
|
||||
from vllm_omni.model_executor.models.fish_speech.fish_speech_slow_ar import (
|
||||
FishSpeechSlowARForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
def test_fast_ar_reuses_dense_position_id_buffer(monkeypatch):
|
||||
fast_config = FishSpeechFastARConfig(
|
||||
vocab_size=16,
|
||||
num_codebooks=4,
|
||||
dim=8,
|
||||
n_head=2,
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
n_layer=1,
|
||||
intermediate_size=16,
|
||||
max_seq_len=5,
|
||||
)
|
||||
slow_config = FishSpeechSlowARConfig(
|
||||
vocab_size=32,
|
||||
dim=8,
|
||||
n_head=2,
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
n_layer=1,
|
||||
intermediate_size=16,
|
||||
codebook_size=16,
|
||||
num_codebooks=4,
|
||||
semantic_begin_id=4,
|
||||
semantic_end_id=19,
|
||||
)
|
||||
fast_ar = object.__new__(FishSpeechFastAR)
|
||||
nn.Module.__init__(fast_ar)
|
||||
fast_ar.config = fast_config
|
||||
fast_ar.slow_ar_config = slow_config
|
||||
fast_ar.fast_project_in = nn.Identity()
|
||||
fast_ar.fast_embeddings = nn.Embedding(fast_config.vocab_size, fast_config.hidden_size)
|
||||
fast_ar.fast_output = nn.Linear(fast_config.hidden_size, fast_config.vocab_size, bias=False)
|
||||
fast_ar.fast_norm = nn.Identity()
|
||||
fast_ar._num_codebooks = fast_config.num_codebooks
|
||||
fast_ar._fast_dim = fast_config.hidden_size
|
||||
fast_ar._embed_buf = None
|
||||
fast_ar._pos_ids = None
|
||||
fast_ar._k_cache = None
|
||||
fast_ar._v_cache = None
|
||||
fast_ar._compiled_model_fwd = None
|
||||
fast_ar._compile_attempted = True
|
||||
fast_ar._compile_failed = False
|
||||
fast_ar._disable_compile_for_graph = False
|
||||
|
||||
seen_position_ids: list[torch.Tensor] = []
|
||||
|
||||
def fake_run_model_one(step_input, step_pos_ids, cache_pos):
|
||||
seen_position_ids.append(step_pos_ids)
|
||||
return torch.zeros(
|
||||
step_input.shape[0],
|
||||
fast_config.hidden_size,
|
||||
dtype=step_input.dtype,
|
||||
device=step_input.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(fast_ar, "_run_model_one", fake_run_model_one)
|
||||
|
||||
hidden = torch.zeros(3, slow_config.hidden_size, dtype=torch.float32)
|
||||
semantic = torch.full((3,), slow_config.semantic_begin_id, dtype=torch.long)
|
||||
fast_ar(hidden, semantic, do_sample=False)
|
||||
|
||||
assert fast_ar._pos_ids is not None
|
||||
assert fast_ar._pos_ids.shape == (3, fast_config.num_codebooks + 1)
|
||||
assert len(seen_position_ids) == fast_config.num_codebooks
|
||||
for step, step_pos_ids in enumerate(seen_position_ids):
|
||||
assert step_pos_ids.shape == (3,)
|
||||
assert step_pos_ids.untyped_storage().data_ptr() == fast_ar._pos_ids.untyped_storage().data_ptr()
|
||||
assert step_pos_ids.stride() == (fast_ar._pos_ids.stride(0),)
|
||||
expected = torch.full((3,), step)
|
||||
assert torch.equal(step_pos_ids.cpu(), expected)
|
||||
|
||||
|
||||
def test_talker_mtp_does_not_mutate_input():
|
||||
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
|
||||
nn.Module.__init__(model)
|
||||
model._semantic_begin_id = 4
|
||||
model._semantic_end_id = 19
|
||||
model._codebook_size = 16
|
||||
model._num_codebooks = 4
|
||||
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
|
||||
model.fast_ar = lambda **_: torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3],
|
||||
[15, 14, 13, 12],
|
||||
],
|
||||
dtype=torch.long,
|
||||
)
|
||||
input_ids = torch.tensor([4, 2], dtype=torch.long)
|
||||
input_embeds = torch.randn(2, 8)
|
||||
input_embeds_before = input_embeds.clone()
|
||||
last_hidden = torch.randn(2, 8, dtype=torch.bfloat16)
|
||||
text_step = torch.zeros(2, 8, dtype=torch.bfloat16)
|
||||
|
||||
out1, _ = FishSpeechSlowARForConditionalGeneration.talker_mtp(
|
||||
model,
|
||||
input_ids,
|
||||
input_embeds,
|
||||
last_hidden,
|
||||
text_step,
|
||||
)
|
||||
|
||||
assert torch.equal(input_embeds, input_embeds_before)
|
||||
assert out1.untyped_storage().data_ptr() != input_embeds.untyped_storage().data_ptr()
|
||||
|
||||
|
||||
def test_talker_mtp_forwards_sampling_params():
|
||||
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
|
||||
nn.Module.__init__(model)
|
||||
model._semantic_begin_id = 4
|
||||
model._semantic_end_id = 19
|
||||
model._codebook_size = 16
|
||||
model._num_codebooks = 4
|
||||
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
|
||||
seen = {}
|
||||
|
||||
def fake_fast_ar(**kwargs):
|
||||
seen.update(kwargs)
|
||||
return torch.tensor([[0, 1, 2, 3]], dtype=torch.long)
|
||||
|
||||
model.fast_ar = fake_fast_ar
|
||||
input_ids = torch.tensor([4], dtype=torch.long)
|
||||
input_embeds = torch.randn(1, 8)
|
||||
last_hidden = torch.randn(1, 8, dtype=torch.bfloat16)
|
||||
text_step = torch.zeros(1, 8, dtype=torch.bfloat16)
|
||||
|
||||
FishSpeechSlowARForConditionalGeneration.talker_mtp(
|
||||
model,
|
||||
input_ids,
|
||||
input_embeds,
|
||||
last_hidden,
|
||||
text_step,
|
||||
do_sample=False,
|
||||
temperature=0.1,
|
||||
top_k=5,
|
||||
top_p=0.7,
|
||||
)
|
||||
|
||||
assert seen["do_sample"] is False
|
||||
assert seen["temperature"] == 0.1
|
||||
assert seen["top_k"] == 5
|
||||
assert seen["top_p"] == 0.7
|
||||
@@ -1,5 +1,6 @@
|
||||
import functools
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -26,6 +27,39 @@ class _FakeCodec:
|
||||
return wav, audio_lengths
|
||||
|
||||
|
||||
class _RecordingCodec:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def decode(self, codes_bqf: torch.Tensor, feature_lengths: torch.Tensor):
|
||||
self.calls.append((tuple(codes_bqf.shape), tuple(int(x) for x in feature_lengths.tolist())))
|
||||
batch = codes_bqf.shape[0]
|
||||
wav_len = int(codes_bqf.shape[-1]) * 10
|
||||
wav = torch.arange(batch * wav_len, dtype=torch.float32).view(batch, 1, wav_len)
|
||||
audio_lengths = feature_lengths.to(torch.long) * 10
|
||||
return wav, audio_lengths
|
||||
|
||||
|
||||
class _FakeOneArgCodec:
|
||||
def decode(self, codes_bqf: torch.Tensor):
|
||||
batch = codes_bqf.shape[0]
|
||||
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
|
||||
|
||||
|
||||
class _FakeFromIndicesCodec:
|
||||
def __init__(self):
|
||||
self.from_indices_calls = 0
|
||||
|
||||
def from_indices(self, codes_bqf: torch.Tensor):
|
||||
self.from_indices_calls += 1
|
||||
batch = codes_bqf.shape[0]
|
||||
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
|
||||
|
||||
def decode(self, codes_bqf: torch.Tensor):
|
||||
del codes_bqf
|
||||
raise AssertionError("decode() should not be used for index tensors when from_indices() exists")
|
||||
|
||||
|
||||
class _FakeTokenizer:
|
||||
def __init__(self, mapping, unk_token_id=-1):
|
||||
self._mapping = mapping
|
||||
@@ -40,6 +74,7 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = _FakeCodec()
|
||||
decoder._codec_decode_takes_lengths = True
|
||||
decoder._num_codebooks = 10
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
@@ -62,6 +97,132 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
|
||||
assert audios[1].shape[0] == 50
|
||||
|
||||
|
||||
def test_dac_decoder_uses_tensor_codes_from_runtime_info():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = _FakeCodec()
|
||||
decoder._codec_decode_takes_lengths = True
|
||||
decoder._num_codebooks = 2
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.tensor([0], dtype=torch.long),
|
||||
runtime_additional_information=[
|
||||
{
|
||||
"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)},
|
||||
"meta": {"left_context_size": 1},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audios) == 1
|
||||
# 2 total frames with 1 frame of left context => proportional trim removes half the samples.
|
||||
assert audios[0].shape[0] == 50
|
||||
|
||||
|
||||
def test_dac_decoder_supports_codec_decode_without_lengths():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = _FakeOneArgCodec()
|
||||
decoder._codec_decode_takes_lengths = False
|
||||
decoder._num_codebooks = 10
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
|
||||
torch.arange(10, dtype=torch.long),
|
||||
torch.arange(20, dtype=torch.long),
|
||||
]
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.arange(30, dtype=torch.long),
|
||||
runtime_additional_information=[{}, {}],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audios) == 2
|
||||
assert audios[0].shape[0] == 50
|
||||
assert audios[1].shape[0] == 100
|
||||
|
||||
|
||||
def test_dac_decoder_prefers_from_indices_for_one_arg_codec():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
codec = _FakeFromIndicesCodec()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = codec
|
||||
decoder._codec_decode_takes_lengths = False
|
||||
decoder._num_codebooks = 10
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
|
||||
torch.arange(10, dtype=torch.long),
|
||||
torch.arange(20, dtype=torch.long),
|
||||
]
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.arange(30, dtype=torch.long),
|
||||
runtime_additional_information=[{}, {}],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert codec.from_indices_calls == 1
|
||||
assert len(audios) == 2
|
||||
assert audios[0].shape[0] == 50
|
||||
assert audios[1].shape[0] == 100
|
||||
|
||||
|
||||
def test_dac_decoder_splits_groups_by_padded_frame_budget():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
codec = _RecordingCodec()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = codec
|
||||
decoder._codec_decode_takes_lengths = True
|
||||
decoder._num_codebooks = 2
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._decode_batch_max_padded_frames = 100
|
||||
decoder._decode_batch_max_batch = 0
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
|
||||
torch.arange(2 * 46, dtype=torch.long),
|
||||
torch.arange(2 * 45, dtype=torch.long),
|
||||
torch.arange(2 * 47, dtype=torch.long),
|
||||
]
|
||||
|
||||
decoder.forward(input_ids=torch.arange(2 * (46 + 45 + 47), dtype=torch.long))
|
||||
|
||||
assert codec.calls == [
|
||||
((2, 2, 46), (45, 46)),
|
||||
((1, 2, 47), (47,)),
|
||||
]
|
||||
|
||||
|
||||
def test_dac_decoder_dtype_can_be_configured():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
cfg = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
model="unused",
|
||||
stage_connector_config={"extra": {"fish_speech_dac_dtype": "fp16"}},
|
||||
)
|
||||
)
|
||||
|
||||
decoder = FishSpeechDACDecoder(vllm_config=cfg)
|
||||
|
||||
assert decoder._dac_dtype == torch.float16
|
||||
|
||||
|
||||
def test_structured_voice_clone_prefill_adds_full_codebooks_with_decode_scale(monkeypatch):
|
||||
slow_ar_module, _, FishSpeechSlowARForConditionalGeneration = _fish_speech_regression_modules()
|
||||
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
|
||||
|
||||
Reference in new issue
Block a user