[Refactor][Ming-TTS] Move speaker extraction and caching to Stage-0 (#5351)
Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com>
This commit is contained in:
1 parent
7762d664d8
commit
fb89ab43ea
9 files changed
+440
-123
No files matched your search
@@ -45,7 +45,13 @@ from vllm_omni.model_executor.models.fish_speech.prompt_utils import (
|
||||
FISH_TEXT_ONLY_SYSTEM_PROMPT,
|
||||
build_fish_voice_clone_prompt_ids,
|
||||
)
|
||||
from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM
|
||||
from vllm_omni.model_executor.models.ming_tts.constants import (
|
||||
KEY_SPEAKER_EMBEDDING,
|
||||
KEY_SPEAKER_SAMPLE_RATES,
|
||||
KEY_SPEAKER_WAVEFORM,
|
||||
KEY_SPEAKER_WAVEFORM_LENGTHS,
|
||||
SPEAKER_EMBEDDING_DIM,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -1014,7 +1020,6 @@ class TestTTSMethods:
|
||||
right = np.full(32000, -0.25, dtype=np.float32)
|
||||
samples = np.stack((left, right), axis=-1)
|
||||
mocker.patch("soundfile.read", return_value=(samples, 16000))
|
||||
mock_extract = mocker.patch.object(speech_server, "_extract_ming_speaker_embeddings_from_ref_audio")
|
||||
saved: dict[str, object] = {}
|
||||
|
||||
def fake_save_file(tensors, path, metadata=None):
|
||||
@@ -1038,7 +1043,6 @@ class TestTTSMethods:
|
||||
)
|
||||
)
|
||||
|
||||
mock_extract.assert_not_called()
|
||||
tensors = saved["tensors"]
|
||||
metadata = saved["metadata"]
|
||||
assert isinstance(tensors, dict)
|
||||
@@ -1669,16 +1673,20 @@ class TestTTSMethods:
|
||||
|
||||
mock_get_emb.assert_called_once_with("emb_voice")
|
||||
mock_get_audio.assert_not_called()
|
||||
mock_prompt.assert_called_once_with(req, ref_audio_data=None)
|
||||
mock_prompt.assert_called_once_with(
|
||||
req,
|
||||
ref_audio_data=None,
|
||||
voice_name=None,
|
||||
voice_created_at=0,
|
||||
)
|
||||
prompt_request = mock_prompt.call_args.args[0]
|
||||
assert prompt_request.speaker_embedding == fake_embedding
|
||||
assert len(prompt_request.speaker_embedding) == SPEAKER_EMBEDDING_DIM
|
||||
assert prepared.prompt["additional_information"]["speaker_count"] == 1
|
||||
assert prepared.model_type == "ming_tts"
|
||||
|
||||
def test_ming_adapter_uploaded_audio_uses_cached_embedding(self, speech_server, mocker: MockerFixture):
|
||||
"""Cached Ming audio voices skip repeated speaker extraction."""
|
||||
speech_server._speaker_cache.clear()
|
||||
def test_ming_adapter_uploaded_audio_forwards_worker_cache_metadata(self, speech_server, mocker: MockerFixture):
|
||||
"""Ming audio voices defer extraction and cache ownership to Stage-0."""
|
||||
speech_server.uploaded_speakers = {
|
||||
"audio_voice": {
|
||||
"name": "audio_voice",
|
||||
@@ -1691,26 +1699,12 @@ class TestTTSMethods:
|
||||
}
|
||||
ref_audio_source = "data:audio/wav;base64,ZmFrZWF1ZGlv"
|
||||
ref_audio_data = ([0.0, 0.1], 16000)
|
||||
fake_embedding = [0.2] * SPEAKER_EMBEDDING_DIM
|
||||
cache_key = speech_server._speaker_cache.make_cache_key(
|
||||
"audio_voice",
|
||||
model_type="ming_tts",
|
||||
created_at=123,
|
||||
)
|
||||
speech_server._speaker_cache.put(
|
||||
cache_key,
|
||||
{"speaker_embedding": torch.tensor(fake_embedding)},
|
||||
)
|
||||
mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source)
|
||||
mocker.patch.object(
|
||||
speech_server,
|
||||
"_resolve_ref_audio",
|
||||
new=mocker.AsyncMock(return_value=ref_audio_data),
|
||||
)
|
||||
mock_extract = mocker.patch.object(
|
||||
speech_server,
|
||||
"_extract_ming_speaker_embeddings_from_ref_audio",
|
||||
)
|
||||
mock_prompt = mocker.patch.object(
|
||||
speech_server,
|
||||
"_build_ming_dense_prompt",
|
||||
@@ -1721,13 +1715,82 @@ class TestTTSMethods:
|
||||
req = OpenAICreateSpeechRequest(input="Hello", voice="audio_voice")
|
||||
prepared = asyncio.run(adapter.build(req, [], False))
|
||||
|
||||
mock_extract.assert_not_called()
|
||||
mock_prompt.assert_called_once_with(req, ref_audio_data=ref_audio_data)
|
||||
mock_prompt.assert_called_once_with(
|
||||
req,
|
||||
ref_audio_data=ref_audio_data,
|
||||
voice_name="audio_voice",
|
||||
voice_created_at=123,
|
||||
)
|
||||
assert req.ref_text == "Reference transcript."
|
||||
assert req.speaker_embedding == pytest.approx(fake_embedding)
|
||||
assert req.speaker_embedding is None
|
||||
assert prepared.model_type == "ming_tts"
|
||||
|
||||
def test_ming_adapter_legacy_audio_only_voice_falls_back_and_caches(
|
||||
def test_build_ming_prompt_forwards_audio_segments_and_worker_cache_metadata(self, speech_server):
|
||||
class FakeTokenizer:
|
||||
unk_token_id = -1
|
||||
|
||||
def encode(self, text):
|
||||
return [777] if text == "<|vision_pad|>" else [1]
|
||||
|
||||
def convert_tokens_to_ids(self, token):
|
||||
return 888 if token == "<audioPatch>" else self.unk_token_id
|
||||
|
||||
speech_server._tts_tokenizer = FakeTokenizer()
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
ref_text="Two reference speakers.",
|
||||
)
|
||||
ref_audio_data = [
|
||||
([0.1, 0.2, 0.3], 16000),
|
||||
([0.4, 0.5], 24000),
|
||||
]
|
||||
|
||||
prompt = speech_server._build_ming_dense_prompt(
|
||||
request,
|
||||
ref_audio_data=ref_audio_data,
|
||||
voice_name="uploaded_voice",
|
||||
voice_created_at=123,
|
||||
)
|
||||
|
||||
info = prompt["additional_information"]
|
||||
assert KEY_SPEAKER_EMBEDDING not in info
|
||||
torch.testing.assert_close(
|
||||
info[KEY_SPEAKER_WAVEFORM],
|
||||
torch.tensor([[0.1, 0.2, 0.3, 0.4, 0.5]], dtype=torch.float32),
|
||||
)
|
||||
assert info[KEY_SPEAKER_WAVEFORM_LENGTHS].tolist() == [3, 2]
|
||||
assert info[KEY_SPEAKER_SAMPLE_RATES].tolist() == [16000, 24000]
|
||||
assert info["prompt_waveform"].shape[-1] > info[KEY_SPEAKER_WAVEFORM].shape[-1]
|
||||
assert info["voice_name"] == "uploaded_voice"
|
||||
assert info["voice_created_at"] == 123
|
||||
assert prompt["prompt_token_ids"].count(777) == 2
|
||||
|
||||
def test_build_ming_prompt_forwards_speaker_waveform_without_ref_text(self, speech_server):
|
||||
class FakeTokenizer:
|
||||
unk_token_id = -1
|
||||
|
||||
def encode(self, text):
|
||||
return [1]
|
||||
|
||||
def convert_tokens_to_ids(self, token):
|
||||
return 888 if token == "<audioPatch>" else self.unk_token_id
|
||||
|
||||
speech_server._tts_tokenizer = FakeTokenizer()
|
||||
request = OpenAICreateSpeechRequest(input="Hello")
|
||||
|
||||
prompt = speech_server._build_ming_dense_prompt(
|
||||
request,
|
||||
ref_audio_data=([0.1, 0.2, 0.3], 16000),
|
||||
)
|
||||
|
||||
info = prompt["additional_information"]
|
||||
assert info[KEY_SPEAKER_WAVEFORM].shape == (1, 3)
|
||||
assert info[KEY_SPEAKER_WAVEFORM_LENGTHS].tolist() == [3]
|
||||
assert info[KEY_SPEAKER_SAMPLE_RATES].tolist() == [16000]
|
||||
assert "prompt_waveform" not in info
|
||||
assert KEY_SPEAKER_EMBEDDING not in info
|
||||
|
||||
def test_ming_adapter_legacy_audio_only_voice_forwards_audio_to_worker(
|
||||
self,
|
||||
speech_server,
|
||||
mocker: MockerFixture,
|
||||
@@ -1753,20 +1816,13 @@ class TestTTSMethods:
|
||||
}
|
||||
ref_audio_source = "data:audio/wav;base64,ZmFrZWF1ZGlv"
|
||||
ref_audio_data = ([0.0, 0.1], 16000)
|
||||
fake_embedding = [0.4] * SPEAKER_EMBEDDING_DIM
|
||||
mocker.patch.object(speech_server, "_get_uploaded_audio_data", return_value=ref_audio_source)
|
||||
mocker.patch.object(
|
||||
speech_server,
|
||||
"_resolve_ref_audio",
|
||||
new=mocker.AsyncMock(return_value=ref_audio_data),
|
||||
)
|
||||
mock_extract = mocker.patch.object(
|
||||
speech_server,
|
||||
"_extract_ming_speaker_embeddings_from_ref_audio",
|
||||
return_value=[fake_embedding],
|
||||
)
|
||||
mock_to_thread = mocker.spy(asyncio, "to_thread")
|
||||
mocker.patch.object(
|
||||
mock_prompt = mocker.patch.object(
|
||||
speech_server,
|
||||
"_build_ming_dense_prompt",
|
||||
return_value={"additional_information": {"speaker_count": 1}},
|
||||
@@ -1778,22 +1834,25 @@ class TestTTSMethods:
|
||||
second_req = OpenAICreateSpeechRequest(input="Hello again", voice="legacy_audio_voice")
|
||||
second_prepared = asyncio.run(adapter.build(second_req, [], False))
|
||||
|
||||
mock_extract.assert_called_once_with([ref_audio_data])
|
||||
mock_to_thread.assert_called_once_with(mock_extract, [ref_audio_data])
|
||||
assert first_req.speaker_embedding == pytest.approx(fake_embedding)
|
||||
assert second_req.speaker_embedding == pytest.approx(fake_embedding)
|
||||
assert first_req.speaker_embedding is None
|
||||
assert second_req.speaker_embedding is None
|
||||
assert first_req.ref_text == "Reference transcript."
|
||||
assert second_req.ref_text == "Reference transcript."
|
||||
assert first_prepared.model_type == "ming_tts"
|
||||
assert second_prepared.model_type == "ming_tts"
|
||||
cache_key = speech_server._speaker_cache.make_cache_key(
|
||||
"legacy_audio_voice",
|
||||
model_type="ming_tts",
|
||||
created_at=456,
|
||||
assert mock_prompt.call_count == 2
|
||||
mock_prompt.assert_any_call(
|
||||
first_req,
|
||||
ref_audio_data=ref_audio_data,
|
||||
voice_name="legacy_audio_voice",
|
||||
voice_created_at=456,
|
||||
)
|
||||
mock_prompt.assert_any_call(
|
||||
second_req,
|
||||
ref_audio_data=ref_audio_data,
|
||||
voice_name="legacy_audio_voice",
|
||||
voice_created_at=456,
|
||||
)
|
||||
cached = speech_server._speaker_cache.get(cache_key)
|
||||
assert cached is not None
|
||||
torch.testing.assert_close(cached["speaker_embedding"], torch.tensor(fake_embedding))
|
||||
|
||||
# ── regression: full flow from issue #1603 ──
|
||||
|
||||
|
||||
@@ -12,17 +12,24 @@ from vllm_omni.model_executor.models.common.ming.fm import Solver
|
||||
from vllm_omni.model_executor.models.ming_tts.constants import (
|
||||
AGGREGATOR_HIDDEN_SIZE,
|
||||
HISTORY_PATCH_SIZE,
|
||||
KEY_SPEAKER_EMBEDDING,
|
||||
KEY_SPEAKER_SAMPLE_RATES,
|
||||
KEY_SPEAKER_WAVEFORM,
|
||||
KEY_SPEAKER_WAVEFORM_LENGTHS,
|
||||
LATENT_DIM,
|
||||
LLM_HIDDEN_SIZE,
|
||||
LLM_VOCAB_SIZE,
|
||||
PATCH_SIZE,
|
||||
SAMPLE_RATE,
|
||||
SPEAKER_EMBEDDING_DIM,
|
||||
VAE_PATCH_SIZE,
|
||||
)
|
||||
from vllm_omni.model_executor.models.ming_tts.flowloss_head import FlowLoss
|
||||
from vllm_omni.model_executor.models.ming_tts.ming_tts_llm import MingLLMModel
|
||||
from vllm_omni.model_executor.models.ming_tts.speaker_extractor import _resolve_speaker_embeddings
|
||||
from vllm_omni.model_executor.models.ming_tts.validation import validate_ming_tts_config
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.utils.speaker_cache import SpeakerEmbeddingCache
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.tts]
|
||||
|
||||
@@ -99,6 +106,122 @@ def test_ming_instruction_parser_preserves_dense_and_flash_defaults():
|
||||
assert flash_fields == {"风格": "calm"}
|
||||
|
||||
|
||||
class _FakeMingSpeakerExtractor:
|
||||
def __init__(self):
|
||||
self.calls: list[tuple[torch.Tensor, int]] = []
|
||||
|
||||
def extract_from_waveform(self, waveform, sample_rate):
|
||||
waveform = torch.as_tensor(waveform).detach().clone()
|
||||
self.calls.append((waveform, int(sample_rate)))
|
||||
return torch.full((SPEAKER_EMBEDDING_DIM,), float(len(self.calls)), dtype=torch.float32)
|
||||
|
||||
|
||||
def _make_ming_speaker_wrapper():
|
||||
return SimpleNamespace(
|
||||
_speaker_cache=SpeakerEmbeddingCache(max_bytes=1024 * 1024),
|
||||
_speaker_extractor=_FakeMingSpeakerExtractor(),
|
||||
ming_config=SimpleNamespace(sample_rate=SAMPLE_RATE),
|
||||
)
|
||||
|
||||
|
||||
def test_ming_stage0_speaker_cache_extracts_once_and_honors_created_at():
|
||||
wrapper = _make_ming_speaker_wrapper()
|
||||
waveform = torch.arange(12, dtype=torch.float32).reshape(1, -1)
|
||||
info = {
|
||||
KEY_SPEAKER_WAVEFORM: waveform,
|
||||
KEY_SPEAKER_WAVEFORM_LENGTHS: torch.tensor([12]),
|
||||
"voice_name": "UploadedVoice",
|
||||
"voice_created_at": 123,
|
||||
}
|
||||
|
||||
first = _resolve_speaker_embeddings(wrapper, info)
|
||||
cached = _resolve_speaker_embeddings(
|
||||
wrapper,
|
||||
{
|
||||
"voice_name": "uploadedvoice",
|
||||
"voice_created_at": 123,
|
||||
},
|
||||
)
|
||||
refreshed = _resolve_speaker_embeddings(
|
||||
wrapper,
|
||||
{
|
||||
**info,
|
||||
"voice_created_at": 124,
|
||||
},
|
||||
)
|
||||
|
||||
assert first is not None and cached is not None and refreshed is not None
|
||||
torch.testing.assert_close(first[0], cached[0])
|
||||
assert torch.all(first[0] == 1)
|
||||
assert torch.all(refreshed[0] == 2)
|
||||
assert len(wrapper._speaker_extractor.calls) == 2
|
||||
|
||||
|
||||
def test_ming_stage0_speaker_extraction_splits_multiple_waveforms():
|
||||
wrapper = _make_ming_speaker_wrapper()
|
||||
|
||||
embeddings = _resolve_speaker_embeddings(
|
||||
wrapper,
|
||||
{
|
||||
KEY_SPEAKER_WAVEFORM: torch.arange(10, dtype=torch.float32).reshape(1, -1),
|
||||
KEY_SPEAKER_WAVEFORM_LENGTHS: torch.tensor([4, 6]),
|
||||
KEY_SPEAKER_SAMPLE_RATES: torch.tensor([16000, 24000]),
|
||||
},
|
||||
)
|
||||
|
||||
assert embeddings is not None and len(embeddings) == 2
|
||||
assert [call[0].shape[-1] for call in wrapper._speaker_extractor.calls] == [4, 6]
|
||||
assert [call[1] for call in wrapper._speaker_extractor.calls] == [16000, 24000]
|
||||
|
||||
|
||||
def test_ming_stage0_speaker_extraction_rejects_mismatched_sample_rates():
|
||||
wrapper = _make_ming_speaker_wrapper()
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid Ming speaker sample rates"):
|
||||
_resolve_speaker_embeddings(
|
||||
wrapper,
|
||||
{
|
||||
KEY_SPEAKER_WAVEFORM: torch.arange(10, dtype=torch.float32).reshape(1, -1),
|
||||
KEY_SPEAKER_WAVEFORM_LENGTHS: torch.tensor([4, 6]),
|
||||
KEY_SPEAKER_SAMPLE_RATES: torch.tensor([16000]),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_ming_stage0_prompt_waveform_does_not_trigger_speaker_extraction():
|
||||
wrapper = _make_ming_speaker_wrapper()
|
||||
|
||||
embeddings = _resolve_speaker_embeddings(
|
||||
wrapper,
|
||||
{
|
||||
"prompt_waveform": torch.ones((1, 10)),
|
||||
"prompt_text": "Reference transcript.",
|
||||
},
|
||||
)
|
||||
|
||||
assert embeddings is None
|
||||
assert not wrapper._speaker_extractor.calls
|
||||
|
||||
|
||||
def test_ming_stage0_direct_speaker_embedding_bypasses_extractor():
|
||||
wrapper = _make_ming_speaker_wrapper()
|
||||
direct = torch.full((SPEAKER_EMBEDDING_DIM,), 0.25)
|
||||
|
||||
embeddings = _resolve_speaker_embeddings(
|
||||
wrapper,
|
||||
{
|
||||
KEY_SPEAKER_EMBEDDING: direct,
|
||||
KEY_SPEAKER_WAVEFORM: torch.ones((1, 10)),
|
||||
"voice_name": "direct",
|
||||
"voice_created_at": 1,
|
||||
},
|
||||
)
|
||||
|
||||
assert embeddings is not None
|
||||
torch.testing.assert_close(embeddings[0], direct)
|
||||
assert not wrapper._speaker_extractor.calls
|
||||
|
||||
|
||||
def _make_ming_logits_model(vocab_size=8):
|
||||
model = object.__new__(MingLLMModel)
|
||||
model.ming_config = SimpleNamespace(
|
||||
|
||||
Reference in new issue
Block a user