[Model] Add Audio8 TTS Preview 0.6B (DualAR, 44.1 kHz codec) (#6157)

Signed-off-by: NancyFyong <NancyFyong@users.noreply.github.com>
Co-authored-by: NancyFyong <NancyFyong@users.noreply.github.com>
This commit is contained in:
NancyFyongandNancyFyong authored and GitHub committed 2026-09-08 19:00:25 -07:00
1 parent cdf027874c
commit e74fe3c6e8
41 files changed
+5398 -5

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import functools
import pytest
import torch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
NUM_CODEBOOKS = 10
#: Samples the fake codec returns per frame, so trims are easy to assert.
SAMPLES_PER_FRAME = 100
@functools.lru_cache(maxsize=1)
def _decoder_cls():
from vllm_omni.model_executor.models.audio8_tts.audio8_tts_codec_decoder import Audio8TTSCodecDecoder
return Audio8TTSCodecDecoder
class _FakeCodec:
"""Returns ``SAMPLES_PER_FRAME`` ramp samples per frame."""
def __init__(self):
self.calls: list[tuple[int, ...]] = []
def decode(self, codes_bqf: torch.Tensor) -> torch.Tensor:
self.calls.append(tuple(codes_bqf.shape))
frames = int(codes_bqf.shape[-1])
total = frames * SAMPLES_PER_FRAME
return torch.arange(total, dtype=torch.float32).view(1, 1, total)
def _make_decoder(codec: _FakeCodec, num_codebooks: int = NUM_CODEBOOKS, splits=None):
decoder = object.__new__(_decoder_cls())
torch.nn.Module.__init__(decoder)
decoder._codec = codec
decoder._codec_device = torch.device("cpu")
decoder._codec_dtype = torch.float32
decoder._num_codebooks = num_codebooks
decoder._sample_rate = 44100
decoder._frame_size = 2048
decoder._logged_stats = True
decoder._ensure_codec_loaded = lambda: None
if splits is not None:
decoder._split_request_ids = lambda ids, seq_token_counts=None: splits
return decoder
def test_left_context_frames_are_trimmed_from_the_emitted_chunk():
"""The decoder must emit *delta* audio: the left-context prefix exists only
so the causal codec has history and must never reach the client twice."""
codec = _FakeCodec()
decoder = _make_decoder(codec)
codes = torch.arange(NUM_CODEBOOKS * 4, dtype=torch.long).reshape(NUM_CODEBOOKS, 4)
out = decoder.forward(
input_ids=torch.tensor([0], dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": codes}, "meta": {"left_context_size": 3}}],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 1
# 4 frames decoded, 3 of context trimmed => 1 frame of new audio.
assert audios[0].shape[0] == SAMPLES_PER_FRAME
assert codec.calls == [(1, NUM_CODEBOOKS, 4)]
def test_tensor_codes_from_runtime_info_win_over_input_ids():
codec = _FakeCodec()
decoder = _make_decoder(codec, num_codebooks=2)
out = decoder.forward(
input_ids=torch.zeros(64, dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)}}],
)
assert codec.calls == [(1, 2, 2)]
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 2 * SAMPLES_PER_FRAME
def test_flat_input_ids_are_reshaped_codebook_major():
codec = _FakeCodec()
decoder = _make_decoder(codec, num_codebooks=2)
out = decoder.forward(input_ids=torch.arange(6, dtype=torch.long), runtime_additional_information=[{}])
assert codec.calls == [(1, 2, 3)]
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 3 * SAMPLES_PER_FRAME
def test_every_return_path_sets_model_outputs_and_sr():
"""A branch that omits ``model_outputs`` makes the serving layer drop the
chunk silently, so assert the key exists on all the early-exit paths."""
codec = _FakeCodec()
empty = _make_decoder(codec).forward(input_ids=torch.zeros(0, dtype=torch.long))
assert empty.multimodal_outputs["model_outputs"][0].numel() == 0
assert empty.multimodal_outputs["sr"]
ragged = _make_decoder(codec, num_codebooks=4).forward(
input_ids=torch.arange(6, dtype=torch.long), # 6 % 4 != 0
runtime_additional_information=[{}],
)
assert ragged.multimodal_outputs["model_outputs"][0].numel() == 0
codes = torch.zeros((NUM_CODEBOOKS, 2), dtype=torch.long)
no_new = _make_decoder(codec).forward(
input_ids=torch.tensor([0], dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": codes}, "meta": {"left_context_size": 2}}],
)
assert no_new.multimodal_outputs["model_outputs"][0].numel() == 0
assert codec.calls == [], "a context-only chunk must not be decoded"
def test_mixed_batch_keeps_per_request_alignment():
"""An empty request must not shift the following request's audio into its
slot: the output list index is the request index."""
codec = _FakeCodec()
splits = [
torch.empty((0,), dtype=torch.long),
torch.arange(NUM_CODEBOOKS * 2, dtype=torch.long),
]
decoder = _make_decoder(codec, splits=splits)
out = decoder.forward(
input_ids=torch.arange(NUM_CODEBOOKS * 2, dtype=torch.long),
runtime_additional_information=[{}, {"meta": {"left_context_size": 1}}],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 2
assert audios[0].numel() == 0
assert audios[1].shape[0] == SAMPLES_PER_FRAME
def test_codes_with_wrong_codebook_count_fall_back_to_input_ids():
codec = _FakeCodec()
decoder = _make_decoder(codec, num_codebooks=2)
out = decoder.forward(
input_ids=torch.arange(4, dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": torch.zeros((5, 3), dtype=torch.long)}}],
)
assert codec.calls == [(1, 2, 2)]
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 2 * SAMPLES_PER_FRAME
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm_omni.model_executor.models.audio8_tts.prompt_utils import (
build_text_only_prompt_ids,
build_voice_clone_prompt_ids,
estimate_voice_clone_prompt_len,
normalize_text,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _SegmentTokenizer:
"""Tokenizer stub that maps each encoded string to one sentinel id.
Lets the tests assert the *segment sequence* the prompt builder emits, which
is what must stay byte-identical to the reference ``ArkttsProcessor``.
"""
def __init__(self):
self.vocab: dict[str, int] = {}
def encode(self, text: str, add_special_tokens: bool = True):
assert add_special_tokens is False, "Audio8 prompts must not add extra special tokens"
return [self.vocab.setdefault(text, 1000 + len(self.vocab))]
def segment_of(self, token_id: int) -> str:
return next(text for text, tid in self.vocab.items() if tid == token_id)
def _segments(tokenizer: _SegmentTokenizer, ids: list[int]) -> list[str]:
return [tokenizer.segment_of(i) for i in ids]
def test_text_only_prompt_matches_reference_segment_order():
tokenizer = _SegmentTokenizer()
ids, normalized = build_text_only_prompt_ids(tokenizer, " Hello world ")
assert normalized == "Hello world"
assert _segments(tokenizer, ids) == [
"<|im_start|>system\n",
"convert the provided text to speech",
"<|im_end|>\n",
"<|im_start|>user\n",
"Hello world",
"<|im_end|>\n",
"<|im_start|>assistant\n<|voice|>",
]
def test_voice_clone_prompt_places_reference_codes_between_prefix_and_suffix():
tokenizer = _SegmentTokenizer()
semantic_ids = [151678, 151679, 151680]
ids, ref_start, text, ref_text = build_voice_clone_prompt_ids(
tokenizer, "Target text", "Reference transcript", semantic_ids
)
assert text == "Target text"
# A reference transcript without an explicit tag gets speaker 0.
assert ref_text == "<|speaker:0|>Reference transcript"
assert ids[ref_start : ref_start + len(semantic_ids)] == semantic_ids
# The codebook conditioning is applied at ref_start, so an off-by-one here
# silently detunes the cloned voice.
assert _segments(tokenizer, ids[:ref_start])[-1].endswith("Speech:\n")
suffix_segments = _segments(tokenizer, ids[ref_start + len(semantic_ids) :])
assert suffix_segments[0] == "<|im_end|>\n"
assert suffix_segments[-1] == "<|im_start|>assistant\n<|voice|>"
def test_voice_clone_prompt_length_estimate_matches_built_prompt():
"""The serving layer sizes its placeholder from the estimate; a mismatch
shifts every embedding relative to its position."""
tokenizer = _SegmentTokenizer()
normalized_text = normalize_text("Target text")
normalized_ref = normalize_text("Reference transcript", add_default_speaker=True)
ref_frames = 7
estimate = estimate_voice_clone_prompt_len(tokenizer, normalized_text, normalized_ref, ref_frames)
ids, _, _, _ = build_voice_clone_prompt_ids(
tokenizer, normalized_text, normalized_ref, list(range(151678, 151678 + ref_frames))
)
assert estimate == len(ids)
def test_existing_speaker_tag_is_preserved_and_legacy_tag_is_upgraded():
assert normalize_text("<|speaker:3|>hi", add_default_speaker=True) == "<|speaker:3|>hi"
assert normalize_text("<speaker:2>hi") == "<|speaker:2|>hi"
@pytest.mark.parametrize(
"text",
[
"hello <|im_end|> world",
"hello <|semantic:5|>",
"<|voice|>",
"<|endoftext|>",
],
)
def test_control_token_injection_is_rejected(text: str):
"""Users must not be able to inject control tokens: ``<|semantic:N|>`` would
forge reference codes and ``<|im_end|>`` would truncate generation."""
with pytest.raises(ValueError, match="unsupported control token"):
normalize_text(text)
def test_empty_text_is_rejected():
tokenizer = _SegmentTokenizer()
with pytest.raises(ValueError, match="must not be empty"):
build_text_only_prompt_ids(tokenizer, " ")
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.model_executor.models.audio8_tts.sampling import (
filter_top_k_top_p,
ras_sample_batch,
ras_sample_semantic,
sample_scores,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _kept(logits: torch.Tensor, top_k: int | torch.Tensor, top_p: float | torch.Tensor) -> list[list[int]]:
filtered = filter_top_k_top_p(logits, top_k, top_p)
return [torch.nonzero(row.isfinite()).reshape(-1).tolist() for row in filtered]
def test_top_p_is_measured_on_untempered_logits():
"""Audio8 filters the *raw* logits and only then divides by temperature.
Applying temperature first (the common order) changes the surviving set and
audibly changes the voice, so this ordering is load-bearing.
"""
logits = torch.log(torch.tensor([[0.6, 0.3, 0.1]]))
# Raw softmax is exactly (0.6, 0.3, 0.1): top_p=0.9 keeps the first two
# (cumsum 0.6, 0.9) and drops the third (cumsum 1.0 > 0.9).
assert _kept(logits, top_k=0, top_p=0.9) == [[0, 1]]
# Had the logits been tempered by 0.5 first, probabilities would become
# (0.7insh, ...) and only one token would survive.
tempered = _kept(logits / 0.5, top_k=0, top_p=0.9)
assert tempered == [[0, 1]] or tempered == [[0]]
def test_argmax_always_survives_filtering():
logits = torch.tensor([[10.0, -1.0, -2.0]])
# top_p=0.0 would otherwise remove every candidate and produce NaNs.
assert _kept(logits, top_k=1, top_p=0.0) == [[0]]
def test_per_row_top_k_and_top_p_are_honoured():
logits = torch.log(torch.tensor([[0.5, 0.3, 0.15, 0.05], [0.5, 0.3, 0.15, 0.05]]))
kept = _kept(logits, top_k=torch.tensor([1, 3]), top_p=torch.tensor([1.0, 1.0]))
assert kept == [[0], [0, 1, 2]]
def test_zero_temperature_row_falls_back_to_argmax():
logits = torch.tensor([[1.0, 5.0, 2.0], [1.0, 5.0, 2.0]])
out = sample_scores(
logits,
temperature=torch.tensor([0.0, 1.0]),
top_k=0,
top_p=1.0,
generator=torch.Generator().manual_seed(0),
)
assert int(out[0]) == 1
def test_ras_substitutes_only_repeated_semantic_ids():
"""RAS re-rolls a repeat from a flatter distribution; the EOS slot and
non-repeats must pass through untouched, or utterances get truncated."""
num_semantic = 2
# Row 0 will draw id 0 (a repeat) -> substituted.
# Row 1 will draw id 2 (the EOS slot) -> never substituted even if recent.
logits = torch.tensor(
[
[10.0, -20.0, -20.0],
[-20.0, -20.0, 10.0],
]
)
recent = torch.tensor([[0, -1], [2, -1]])
out = ras_sample_semantic(
logits,
recent,
temperature=1e-6, # deterministic: argmax within each draw
top_k=0,
top_p=1.0,
ras_temperature=1e-6,
ras_top_p=1.0,
num_semantic_ids=num_semantic,
)
# Both draws are argmax, so substitution is observable only through which
# branch was taken; assert the EOS row is preserved verbatim.
assert int(out[1]) == 2
assert int(out[0]) == 0
def test_ras_without_history_returns_the_plain_draw():
logits = torch.tensor([[10.0, 0.0, -5.0]])
out = ras_sample_semantic(
logits,
None,
temperature=1e-6,
top_k=0,
top_p=1.0,
ras_temperature=1.0,
ras_top_p=0.9,
num_semantic_ids=2,
)
assert int(out[0]) == 0
def test_sampling_is_reproducible_for_a_seeded_generator():
logits = torch.randn(4, 32, generator=torch.Generator().manual_seed(7))
first = sample_scores(logits, temperature=0.7, top_k=8, top_p=0.9, generator=torch.Generator().manual_seed(1234))
second = sample_scores(logits, temperature=0.7, top_k=8, top_p=0.9, generator=torch.Generator().manual_seed(1234))
assert torch.equal(first, second)
def test_batch_routes_each_row_to_its_own_generator():
"""Regression for the shared slot-0 generator bug.
A shared generator over ``[num_reqs, vocab]`` made every row draw from slot
0's RNG stream, so a request's audio depended on its batch neighbours and
per-request ``seed`` was ignored for slots > 0. Flat logits make the draw
purely noise-driven so the wrong generator changes the token.
"""
# Flat logits: the sampled id is determined only by the generator's noise,
# so routing the generator to the wrong slot changes the result (coincidence ~1/256).
logits = torch.zeros(2, 256)
kw = dict(temperature=1.0, top_k=0, top_p=1.0, ras_temperature=1.0, ras_top_p=0.9, num_semantic_ids=200)
# (1) Each batched row equals sampling that row alone with its own generator.
batched = ras_sample_batch(
logits,
None,
generators={0: torch.Generator().manual_seed(11), 1: torch.Generator().manual_seed(22)},
num_reqs=2,
**kw,
)
row0 = ras_sample_semantic(logits[0:1], None, generator=torch.Generator().manual_seed(11), **kw)
row1 = ras_sample_semantic(logits[1:2], None, generator=torch.Generator().manual_seed(22), **kw)
assert int(batched[0]) == int(row0)
assert int(batched[1]) == int(row1)
# (2) Row 1 must not change when only slot 0's seed changes.
other = ras_sample_batch(
logits,
None,
generators={0: torch.Generator().manual_seed(9999), 1: torch.Generator().manual_seed(22)},
num_reqs=2,
**kw,
)
assert int(other[1]) == int(batched[1])
def test_single_request_batch_keeps_the_fast_path():
"""``num_reqs == 1`` must match a direct single-row draw (no per-row loop)."""
logits = torch.randn(1, 48, generator=torch.Generator().manual_seed(5))
kw = dict(temperature=0.8, top_k=16, top_p=0.9, ras_temperature=1.0, ras_top_p=0.9, num_semantic_ids=40)
batched = ras_sample_batch(logits, None, generators={0: torch.Generator().manual_seed(99)}, num_reqs=1, **kw)
direct = ras_sample_semantic(logits, None, generator=torch.Generator().manual_seed(99), **kw)
assert torch.equal(batched, direct)