[Model] Add Audio8 TTS Preview 0.6B (DualAR, 44.1 kHz codec) (#6157)
Signed-off-by: NancyFyong <NancyFyong@users.noreply.github.com> Co-authored-by: NancyFyong <NancyFyong@users.noreply.github.com>
This commit is contained in:
1 parent
cdf027874c
commit
e74fe3c6e8
41 files changed
+5398
-5
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,141 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import functools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
NUM_CODEBOOKS = 10
|
||||
#: Samples the fake codec returns per frame, so trims are easy to assert.
|
||||
SAMPLES_PER_FRAME = 100
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _decoder_cls():
|
||||
from vllm_omni.model_executor.models.audio8_tts.audio8_tts_codec_decoder import Audio8TTSCodecDecoder
|
||||
|
||||
return Audio8TTSCodecDecoder
|
||||
|
||||
|
||||
class _FakeCodec:
|
||||
"""Returns ``SAMPLES_PER_FRAME`` ramp samples per frame."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls: list[tuple[int, ...]] = []
|
||||
|
||||
def decode(self, codes_bqf: torch.Tensor) -> torch.Tensor:
|
||||
self.calls.append(tuple(codes_bqf.shape))
|
||||
frames = int(codes_bqf.shape[-1])
|
||||
total = frames * SAMPLES_PER_FRAME
|
||||
return torch.arange(total, dtype=torch.float32).view(1, 1, total)
|
||||
|
||||
|
||||
def _make_decoder(codec: _FakeCodec, num_codebooks: int = NUM_CODEBOOKS, splits=None):
|
||||
decoder = object.__new__(_decoder_cls())
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = codec
|
||||
decoder._codec_device = torch.device("cpu")
|
||||
decoder._codec_dtype = torch.float32
|
||||
decoder._num_codebooks = num_codebooks
|
||||
decoder._sample_rate = 44100
|
||||
decoder._frame_size = 2048
|
||||
decoder._logged_stats = True
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
if splits is not None:
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: splits
|
||||
return decoder
|
||||
|
||||
|
||||
def test_left_context_frames_are_trimmed_from_the_emitted_chunk():
|
||||
"""The decoder must emit *delta* audio: the left-context prefix exists only
|
||||
so the causal codec has history and must never reach the client twice."""
|
||||
codec = _FakeCodec()
|
||||
decoder = _make_decoder(codec)
|
||||
codes = torch.arange(NUM_CODEBOOKS * 4, dtype=torch.long).reshape(NUM_CODEBOOKS, 4)
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.tensor([0], dtype=torch.long),
|
||||
runtime_additional_information=[{"codes": {"audio": codes}, "meta": {"left_context_size": 3}}],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audios) == 1
|
||||
# 4 frames decoded, 3 of context trimmed => 1 frame of new audio.
|
||||
assert audios[0].shape[0] == SAMPLES_PER_FRAME
|
||||
assert codec.calls == [(1, NUM_CODEBOOKS, 4)]
|
||||
|
||||
|
||||
def test_tensor_codes_from_runtime_info_win_over_input_ids():
|
||||
codec = _FakeCodec()
|
||||
decoder = _make_decoder(codec, num_codebooks=2)
|
||||
out = decoder.forward(
|
||||
input_ids=torch.zeros(64, dtype=torch.long),
|
||||
runtime_additional_information=[{"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)}}],
|
||||
)
|
||||
assert codec.calls == [(1, 2, 2)]
|
||||
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 2 * SAMPLES_PER_FRAME
|
||||
|
||||
|
||||
def test_flat_input_ids_are_reshaped_codebook_major():
|
||||
codec = _FakeCodec()
|
||||
decoder = _make_decoder(codec, num_codebooks=2)
|
||||
out = decoder.forward(input_ids=torch.arange(6, dtype=torch.long), runtime_additional_information=[{}])
|
||||
assert codec.calls == [(1, 2, 3)]
|
||||
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 3 * SAMPLES_PER_FRAME
|
||||
|
||||
|
||||
def test_every_return_path_sets_model_outputs_and_sr():
|
||||
"""A branch that omits ``model_outputs`` makes the serving layer drop the
|
||||
chunk silently, so assert the key exists on all the early-exit paths."""
|
||||
codec = _FakeCodec()
|
||||
|
||||
empty = _make_decoder(codec).forward(input_ids=torch.zeros(0, dtype=torch.long))
|
||||
assert empty.multimodal_outputs["model_outputs"][0].numel() == 0
|
||||
assert empty.multimodal_outputs["sr"]
|
||||
|
||||
ragged = _make_decoder(codec, num_codebooks=4).forward(
|
||||
input_ids=torch.arange(6, dtype=torch.long), # 6 % 4 != 0
|
||||
runtime_additional_information=[{}],
|
||||
)
|
||||
assert ragged.multimodal_outputs["model_outputs"][0].numel() == 0
|
||||
|
||||
codes = torch.zeros((NUM_CODEBOOKS, 2), dtype=torch.long)
|
||||
no_new = _make_decoder(codec).forward(
|
||||
input_ids=torch.tensor([0], dtype=torch.long),
|
||||
runtime_additional_information=[{"codes": {"audio": codes}, "meta": {"left_context_size": 2}}],
|
||||
)
|
||||
assert no_new.multimodal_outputs["model_outputs"][0].numel() == 0
|
||||
assert codec.calls == [], "a context-only chunk must not be decoded"
|
||||
|
||||
|
||||
def test_mixed_batch_keeps_per_request_alignment():
|
||||
"""An empty request must not shift the following request's audio into its
|
||||
slot: the output list index is the request index."""
|
||||
codec = _FakeCodec()
|
||||
splits = [
|
||||
torch.empty((0,), dtype=torch.long),
|
||||
torch.arange(NUM_CODEBOOKS * 2, dtype=torch.long),
|
||||
]
|
||||
decoder = _make_decoder(codec, splits=splits)
|
||||
out = decoder.forward(
|
||||
input_ids=torch.arange(NUM_CODEBOOKS * 2, dtype=torch.long),
|
||||
runtime_additional_information=[{}, {"meta": {"left_context_size": 1}}],
|
||||
)
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audios) == 2
|
||||
assert audios[0].numel() == 0
|
||||
assert audios[1].shape[0] == SAMPLES_PER_FRAME
|
||||
|
||||
|
||||
def test_codes_with_wrong_codebook_count_fall_back_to_input_ids():
|
||||
codec = _FakeCodec()
|
||||
decoder = _make_decoder(codec, num_codebooks=2)
|
||||
out = decoder.forward(
|
||||
input_ids=torch.arange(4, dtype=torch.long),
|
||||
runtime_additional_information=[{"codes": {"audio": torch.zeros((5, 3), dtype=torch.long)}}],
|
||||
)
|
||||
assert codec.calls == [(1, 2, 2)]
|
||||
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 2 * SAMPLES_PER_FRAME
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.audio8_tts.prompt_utils import (
|
||||
build_text_only_prompt_ids,
|
||||
build_voice_clone_prompt_ids,
|
||||
estimate_voice_clone_prompt_len,
|
||||
normalize_text,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _SegmentTokenizer:
|
||||
"""Tokenizer stub that maps each encoded string to one sentinel id.
|
||||
|
||||
Lets the tests assert the *segment sequence* the prompt builder emits, which
|
||||
is what must stay byte-identical to the reference ``ArkttsProcessor``.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.vocab: dict[str, int] = {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = True):
|
||||
assert add_special_tokens is False, "Audio8 prompts must not add extra special tokens"
|
||||
return [self.vocab.setdefault(text, 1000 + len(self.vocab))]
|
||||
|
||||
def segment_of(self, token_id: int) -> str:
|
||||
return next(text for text, tid in self.vocab.items() if tid == token_id)
|
||||
|
||||
|
||||
def _segments(tokenizer: _SegmentTokenizer, ids: list[int]) -> list[str]:
|
||||
return [tokenizer.segment_of(i) for i in ids]
|
||||
|
||||
|
||||
def test_text_only_prompt_matches_reference_segment_order():
|
||||
tokenizer = _SegmentTokenizer()
|
||||
ids, normalized = build_text_only_prompt_ids(tokenizer, " Hello world ")
|
||||
|
||||
assert normalized == "Hello world"
|
||||
assert _segments(tokenizer, ids) == [
|
||||
"<|im_start|>system\n",
|
||||
"convert the provided text to speech",
|
||||
"<|im_end|>\n",
|
||||
"<|im_start|>user\n",
|
||||
"Hello world",
|
||||
"<|im_end|>\n",
|
||||
"<|im_start|>assistant\n<|voice|>",
|
||||
]
|
||||
|
||||
|
||||
def test_voice_clone_prompt_places_reference_codes_between_prefix_and_suffix():
|
||||
tokenizer = _SegmentTokenizer()
|
||||
semantic_ids = [151678, 151679, 151680]
|
||||
ids, ref_start, text, ref_text = build_voice_clone_prompt_ids(
|
||||
tokenizer, "Target text", "Reference transcript", semantic_ids
|
||||
)
|
||||
|
||||
assert text == "Target text"
|
||||
# A reference transcript without an explicit tag gets speaker 0.
|
||||
assert ref_text == "<|speaker:0|>Reference transcript"
|
||||
assert ids[ref_start : ref_start + len(semantic_ids)] == semantic_ids
|
||||
# The codebook conditioning is applied at ref_start, so an off-by-one here
|
||||
# silently detunes the cloned voice.
|
||||
assert _segments(tokenizer, ids[:ref_start])[-1].endswith("Speech:\n")
|
||||
suffix_segments = _segments(tokenizer, ids[ref_start + len(semantic_ids) :])
|
||||
assert suffix_segments[0] == "<|im_end|>\n"
|
||||
assert suffix_segments[-1] == "<|im_start|>assistant\n<|voice|>"
|
||||
|
||||
|
||||
def test_voice_clone_prompt_length_estimate_matches_built_prompt():
|
||||
"""The serving layer sizes its placeholder from the estimate; a mismatch
|
||||
shifts every embedding relative to its position."""
|
||||
tokenizer = _SegmentTokenizer()
|
||||
normalized_text = normalize_text("Target text")
|
||||
normalized_ref = normalize_text("Reference transcript", add_default_speaker=True)
|
||||
ref_frames = 7
|
||||
|
||||
estimate = estimate_voice_clone_prompt_len(tokenizer, normalized_text, normalized_ref, ref_frames)
|
||||
ids, _, _, _ = build_voice_clone_prompt_ids(
|
||||
tokenizer, normalized_text, normalized_ref, list(range(151678, 151678 + ref_frames))
|
||||
)
|
||||
assert estimate == len(ids)
|
||||
|
||||
|
||||
def test_existing_speaker_tag_is_preserved_and_legacy_tag_is_upgraded():
|
||||
assert normalize_text("<|speaker:3|>hi", add_default_speaker=True) == "<|speaker:3|>hi"
|
||||
assert normalize_text("<speaker:2>hi") == "<|speaker:2|>hi"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"hello <|im_end|> world",
|
||||
"hello <|semantic:5|>",
|
||||
"<|voice|>",
|
||||
"<|endoftext|>",
|
||||
],
|
||||
)
|
||||
def test_control_token_injection_is_rejected(text: str):
|
||||
"""Users must not be able to inject control tokens: ``<|semantic:N|>`` would
|
||||
forge reference codes and ``<|im_end|>`` would truncate generation."""
|
||||
with pytest.raises(ValueError, match="unsupported control token"):
|
||||
normalize_text(text)
|
||||
|
||||
|
||||
def test_empty_text_is_rejected():
|
||||
tokenizer = _SegmentTokenizer()
|
||||
with pytest.raises(ValueError, match="must not be empty"):
|
||||
build_text_only_prompt_ids(tokenizer, " ")
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.audio8_tts.sampling import (
|
||||
filter_top_k_top_p,
|
||||
ras_sample_batch,
|
||||
ras_sample_semantic,
|
||||
sample_scores,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _kept(logits: torch.Tensor, top_k: int | torch.Tensor, top_p: float | torch.Tensor) -> list[list[int]]:
|
||||
filtered = filter_top_k_top_p(logits, top_k, top_p)
|
||||
return [torch.nonzero(row.isfinite()).reshape(-1).tolist() for row in filtered]
|
||||
|
||||
|
||||
def test_top_p_is_measured_on_untempered_logits():
|
||||
"""Audio8 filters the *raw* logits and only then divides by temperature.
|
||||
|
||||
Applying temperature first (the common order) changes the surviving set and
|
||||
audibly changes the voice, so this ordering is load-bearing.
|
||||
"""
|
||||
logits = torch.log(torch.tensor([[0.6, 0.3, 0.1]]))
|
||||
# Raw softmax is exactly (0.6, 0.3, 0.1): top_p=0.9 keeps the first two
|
||||
# (cumsum 0.6, 0.9) and drops the third (cumsum 1.0 > 0.9).
|
||||
assert _kept(logits, top_k=0, top_p=0.9) == [[0, 1]]
|
||||
# Had the logits been tempered by 0.5 first, probabilities would become
|
||||
# (0.7insh, ...) and only one token would survive.
|
||||
tempered = _kept(logits / 0.5, top_k=0, top_p=0.9)
|
||||
assert tempered == [[0, 1]] or tempered == [[0]]
|
||||
|
||||
|
||||
def test_argmax_always_survives_filtering():
|
||||
logits = torch.tensor([[10.0, -1.0, -2.0]])
|
||||
# top_p=0.0 would otherwise remove every candidate and produce NaNs.
|
||||
assert _kept(logits, top_k=1, top_p=0.0) == [[0]]
|
||||
|
||||
|
||||
def test_per_row_top_k_and_top_p_are_honoured():
|
||||
logits = torch.log(torch.tensor([[0.5, 0.3, 0.15, 0.05], [0.5, 0.3, 0.15, 0.05]]))
|
||||
kept = _kept(logits, top_k=torch.tensor([1, 3]), top_p=torch.tensor([1.0, 1.0]))
|
||||
assert kept == [[0], [0, 1, 2]]
|
||||
|
||||
|
||||
def test_zero_temperature_row_falls_back_to_argmax():
|
||||
logits = torch.tensor([[1.0, 5.0, 2.0], [1.0, 5.0, 2.0]])
|
||||
out = sample_scores(
|
||||
logits,
|
||||
temperature=torch.tensor([0.0, 1.0]),
|
||||
top_k=0,
|
||||
top_p=1.0,
|
||||
generator=torch.Generator().manual_seed(0),
|
||||
)
|
||||
assert int(out[0]) == 1
|
||||
|
||||
|
||||
def test_ras_substitutes_only_repeated_semantic_ids():
|
||||
"""RAS re-rolls a repeat from a flatter distribution; the EOS slot and
|
||||
non-repeats must pass through untouched, or utterances get truncated."""
|
||||
num_semantic = 2
|
||||
# Row 0 will draw id 0 (a repeat) -> substituted.
|
||||
# Row 1 will draw id 2 (the EOS slot) -> never substituted even if recent.
|
||||
logits = torch.tensor(
|
||||
[
|
||||
[10.0, -20.0, -20.0],
|
||||
[-20.0, -20.0, 10.0],
|
||||
]
|
||||
)
|
||||
recent = torch.tensor([[0, -1], [2, -1]])
|
||||
out = ras_sample_semantic(
|
||||
logits,
|
||||
recent,
|
||||
temperature=1e-6, # deterministic: argmax within each draw
|
||||
top_k=0,
|
||||
top_p=1.0,
|
||||
ras_temperature=1e-6,
|
||||
ras_top_p=1.0,
|
||||
num_semantic_ids=num_semantic,
|
||||
)
|
||||
# Both draws are argmax, so substitution is observable only through which
|
||||
# branch was taken; assert the EOS row is preserved verbatim.
|
||||
assert int(out[1]) == 2
|
||||
assert int(out[0]) == 0
|
||||
|
||||
|
||||
def test_ras_without_history_returns_the_plain_draw():
|
||||
logits = torch.tensor([[10.0, 0.0, -5.0]])
|
||||
out = ras_sample_semantic(
|
||||
logits,
|
||||
None,
|
||||
temperature=1e-6,
|
||||
top_k=0,
|
||||
top_p=1.0,
|
||||
ras_temperature=1.0,
|
||||
ras_top_p=0.9,
|
||||
num_semantic_ids=2,
|
||||
)
|
||||
assert int(out[0]) == 0
|
||||
|
||||
|
||||
def test_sampling_is_reproducible_for_a_seeded_generator():
|
||||
logits = torch.randn(4, 32, generator=torch.Generator().manual_seed(7))
|
||||
first = sample_scores(logits, temperature=0.7, top_k=8, top_p=0.9, generator=torch.Generator().manual_seed(1234))
|
||||
second = sample_scores(logits, temperature=0.7, top_k=8, top_p=0.9, generator=torch.Generator().manual_seed(1234))
|
||||
assert torch.equal(first, second)
|
||||
|
||||
|
||||
def test_batch_routes_each_row_to_its_own_generator():
|
||||
"""Regression for the shared slot-0 generator bug.
|
||||
|
||||
A shared generator over ``[num_reqs, vocab]`` made every row draw from slot
|
||||
0's RNG stream, so a request's audio depended on its batch neighbours and
|
||||
per-request ``seed`` was ignored for slots > 0. Flat logits make the draw
|
||||
purely noise-driven so the wrong generator changes the token.
|
||||
"""
|
||||
# Flat logits: the sampled id is determined only by the generator's noise,
|
||||
# so routing the generator to the wrong slot changes the result (coincidence ~1/256).
|
||||
logits = torch.zeros(2, 256)
|
||||
kw = dict(temperature=1.0, top_k=0, top_p=1.0, ras_temperature=1.0, ras_top_p=0.9, num_semantic_ids=200)
|
||||
|
||||
# (1) Each batched row equals sampling that row alone with its own generator.
|
||||
batched = ras_sample_batch(
|
||||
logits,
|
||||
None,
|
||||
generators={0: torch.Generator().manual_seed(11), 1: torch.Generator().manual_seed(22)},
|
||||
num_reqs=2,
|
||||
**kw,
|
||||
)
|
||||
row0 = ras_sample_semantic(logits[0:1], None, generator=torch.Generator().manual_seed(11), **kw)
|
||||
row1 = ras_sample_semantic(logits[1:2], None, generator=torch.Generator().manual_seed(22), **kw)
|
||||
assert int(batched[0]) == int(row0)
|
||||
assert int(batched[1]) == int(row1)
|
||||
|
||||
# (2) Row 1 must not change when only slot 0's seed changes.
|
||||
other = ras_sample_batch(
|
||||
logits,
|
||||
None,
|
||||
generators={0: torch.Generator().manual_seed(9999), 1: torch.Generator().manual_seed(22)},
|
||||
num_reqs=2,
|
||||
**kw,
|
||||
)
|
||||
assert int(other[1]) == int(batched[1])
|
||||
|
||||
|
||||
def test_single_request_batch_keeps_the_fast_path():
|
||||
"""``num_reqs == 1`` must match a direct single-row draw (no per-row loop)."""
|
||||
logits = torch.randn(1, 48, generator=torch.Generator().manual_seed(5))
|
||||
kw = dict(temperature=0.8, top_k=16, top_p=0.9, ras_temperature=1.0, ras_top_p=0.9, num_semantic_ids=40)
|
||||
batched = ras_sample_batch(logits, None, generators={0: torch.Generator().manual_seed(99)}, num_reqs=1, **kw)
|
||||
direct = ras_sample_semantic(logits, None, generator=torch.Generator().manual_seed(99), **kw)
|
||||
assert torch.equal(batched, direct)
|
||||
Reference in new issue
Block a user