[Model] Add Audio8 TTS Preview 0.6B (DualAR, 44.1 kHz codec) (#6157)

Signed-off-by: NancyFyong <NancyFyong@users.noreply.github.com>
Co-authored-by: NancyFyong <NancyFyong@users.noreply.github.com>
This commit is contained in:
NancyFyongandNancyFyong authored and GitHub committed 2026-09-08 19:00:25 -07:00
1 parent cdf027874c
commit e74fe3c6e8
41 files changed
+5398 -5

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import functools
import pytest
import torch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
NUM_CODEBOOKS = 10
#: Samples the fake codec returns per frame, so trims are easy to assert.
SAMPLES_PER_FRAME = 100
@functools.lru_cache(maxsize=1)
def _decoder_cls():
from vllm_omni.model_executor.models.audio8_tts.audio8_tts_codec_decoder import Audio8TTSCodecDecoder
return Audio8TTSCodecDecoder
class _FakeCodec:
"""Returns ``SAMPLES_PER_FRAME`` ramp samples per frame."""
def __init__(self):
self.calls: list[tuple[int, ...]] = []
def decode(self, codes_bqf: torch.Tensor) -> torch.Tensor:
self.calls.append(tuple(codes_bqf.shape))
frames = int(codes_bqf.shape[-1])
total = frames * SAMPLES_PER_FRAME
return torch.arange(total, dtype=torch.float32).view(1, 1, total)
def _make_decoder(codec: _FakeCodec, num_codebooks: int = NUM_CODEBOOKS, splits=None):
decoder = object.__new__(_decoder_cls())
torch.nn.Module.__init__(decoder)
decoder._codec = codec
decoder._codec_device = torch.device("cpu")
decoder._codec_dtype = torch.float32
decoder._num_codebooks = num_codebooks
decoder._sample_rate = 44100
decoder._frame_size = 2048
decoder._logged_stats = True
decoder._ensure_codec_loaded = lambda: None
if splits is not None:
decoder._split_request_ids = lambda ids, seq_token_counts=None: splits
return decoder
def test_left_context_frames_are_trimmed_from_the_emitted_chunk():
"""The decoder must emit *delta* audio: the left-context prefix exists only
so the causal codec has history and must never reach the client twice."""
codec = _FakeCodec()
decoder = _make_decoder(codec)
codes = torch.arange(NUM_CODEBOOKS * 4, dtype=torch.long).reshape(NUM_CODEBOOKS, 4)
out = decoder.forward(
input_ids=torch.tensor([0], dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": codes}, "meta": {"left_context_size": 3}}],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 1
# 4 frames decoded, 3 of context trimmed => 1 frame of new audio.
assert audios[0].shape[0] == SAMPLES_PER_FRAME
assert codec.calls == [(1, NUM_CODEBOOKS, 4)]
def test_tensor_codes_from_runtime_info_win_over_input_ids():
codec = _FakeCodec()
decoder = _make_decoder(codec, num_codebooks=2)
out = decoder.forward(
input_ids=torch.zeros(64, dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)}}],
)
assert codec.calls == [(1, 2, 2)]
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 2 * SAMPLES_PER_FRAME
def test_flat_input_ids_are_reshaped_codebook_major():
codec = _FakeCodec()
decoder = _make_decoder(codec, num_codebooks=2)
out = decoder.forward(input_ids=torch.arange(6, dtype=torch.long), runtime_additional_information=[{}])
assert codec.calls == [(1, 2, 3)]
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 3 * SAMPLES_PER_FRAME
def test_every_return_path_sets_model_outputs_and_sr():
"""A branch that omits ``model_outputs`` makes the serving layer drop the
chunk silently, so assert the key exists on all the early-exit paths."""
codec = _FakeCodec()
empty = _make_decoder(codec).forward(input_ids=torch.zeros(0, dtype=torch.long))
assert empty.multimodal_outputs["model_outputs"][0].numel() == 0
assert empty.multimodal_outputs["sr"]
ragged = _make_decoder(codec, num_codebooks=4).forward(
input_ids=torch.arange(6, dtype=torch.long), # 6 % 4 != 0
runtime_additional_information=[{}],
)
assert ragged.multimodal_outputs["model_outputs"][0].numel() == 0
codes = torch.zeros((NUM_CODEBOOKS, 2), dtype=torch.long)
no_new = _make_decoder(codec).forward(
input_ids=torch.tensor([0], dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": codes}, "meta": {"left_context_size": 2}}],
)
assert no_new.multimodal_outputs["model_outputs"][0].numel() == 0
assert codec.calls == [], "a context-only chunk must not be decoded"
def test_mixed_batch_keeps_per_request_alignment():
"""An empty request must not shift the following request's audio into its
slot: the output list index is the request index."""
codec = _FakeCodec()
splits = [
torch.empty((0,), dtype=torch.long),
torch.arange(NUM_CODEBOOKS * 2, dtype=torch.long),
]
decoder = _make_decoder(codec, splits=splits)
out = decoder.forward(
input_ids=torch.arange(NUM_CODEBOOKS * 2, dtype=torch.long),
runtime_additional_information=[{}, {"meta": {"left_context_size": 1}}],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 2
assert audios[0].numel() == 0
assert audios[1].shape[0] == SAMPLES_PER_FRAME
def test_codes_with_wrong_codebook_count_fall_back_to_input_ids():
codec = _FakeCodec()
decoder = _make_decoder(codec, num_codebooks=2)
out = decoder.forward(
input_ids=torch.arange(4, dtype=torch.long),
runtime_additional_information=[{"codes": {"audio": torch.zeros((5, 3), dtype=torch.long)}}],
)
assert codec.calls == [(1, 2, 2)]
assert out.multimodal_outputs["model_outputs"][0].shape[0] == 2 * SAMPLES_PER_FRAME
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from vllm_omni.model_executor.models.audio8_tts.prompt_utils import (
build_text_only_prompt_ids,
build_voice_clone_prompt_ids,
estimate_voice_clone_prompt_len,
normalize_text,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _SegmentTokenizer:
"""Tokenizer stub that maps each encoded string to one sentinel id.
Lets the tests assert the *segment sequence* the prompt builder emits, which
is what must stay byte-identical to the reference ``ArkttsProcessor``.
"""
def __init__(self):
self.vocab: dict[str, int] = {}
def encode(self, text: str, add_special_tokens: bool = True):
assert add_special_tokens is False, "Audio8 prompts must not add extra special tokens"
return [self.vocab.setdefault(text, 1000 + len(self.vocab))]
def segment_of(self, token_id: int) -> str:
return next(text for text, tid in self.vocab.items() if tid == token_id)
def _segments(tokenizer: _SegmentTokenizer, ids: list[int]) -> list[str]:
return [tokenizer.segment_of(i) for i in ids]
def test_text_only_prompt_matches_reference_segment_order():
tokenizer = _SegmentTokenizer()
ids, normalized = build_text_only_prompt_ids(tokenizer, " Hello world ")
assert normalized == "Hello world"
assert _segments(tokenizer, ids) == [
"<|im_start|>system\n",
"convert the provided text to speech",
"<|im_end|>\n",
"<|im_start|>user\n",
"Hello world",
"<|im_end|>\n",
"<|im_start|>assistant\n<|voice|>",
]
def test_voice_clone_prompt_places_reference_codes_between_prefix_and_suffix():
tokenizer = _SegmentTokenizer()
semantic_ids = [151678, 151679, 151680]
ids, ref_start, text, ref_text = build_voice_clone_prompt_ids(
tokenizer, "Target text", "Reference transcript", semantic_ids
)
assert text == "Target text"
# A reference transcript without an explicit tag gets speaker 0.
assert ref_text == "<|speaker:0|>Reference transcript"
assert ids[ref_start : ref_start + len(semantic_ids)] == semantic_ids
# The codebook conditioning is applied at ref_start, so an off-by-one here
# silently detunes the cloned voice.
assert _segments(tokenizer, ids[:ref_start])[-1].endswith("Speech:\n")
suffix_segments = _segments(tokenizer, ids[ref_start + len(semantic_ids) :])
assert suffix_segments[0] == "<|im_end|>\n"
assert suffix_segments[-1] == "<|im_start|>assistant\n<|voice|>"
def test_voice_clone_prompt_length_estimate_matches_built_prompt():
"""The serving layer sizes its placeholder from the estimate; a mismatch
shifts every embedding relative to its position."""
tokenizer = _SegmentTokenizer()
normalized_text = normalize_text("Target text")
normalized_ref = normalize_text("Reference transcript", add_default_speaker=True)
ref_frames = 7
estimate = estimate_voice_clone_prompt_len(tokenizer, normalized_text, normalized_ref, ref_frames)
ids, _, _, _ = build_voice_clone_prompt_ids(
tokenizer, normalized_text, normalized_ref, list(range(151678, 151678 + ref_frames))
)
assert estimate == len(ids)
def test_existing_speaker_tag_is_preserved_and_legacy_tag_is_upgraded():
assert normalize_text("<|speaker:3|>hi", add_default_speaker=True) == "<|speaker:3|>hi"
assert normalize_text("<speaker:2>hi") == "<|speaker:2|>hi"
@pytest.mark.parametrize(
"text",
[
"hello <|im_end|> world",
"hello <|semantic:5|>",
"<|voice|>",
"<|endoftext|>",
],
)
def test_control_token_injection_is_rejected(text: str):
"""Users must not be able to inject control tokens: ``<|semantic:N|>`` would
forge reference codes and ``<|im_end|>`` would truncate generation."""
with pytest.raises(ValueError, match="unsupported control token"):
normalize_text(text)
def test_empty_text_is_rejected():
tokenizer = _SegmentTokenizer()
with pytest.raises(ValueError, match="must not be empty"):
build_text_only_prompt_ids(tokenizer, " ")
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
import torch
from vllm_omni.model_executor.models.audio8_tts.sampling import (
filter_top_k_top_p,
ras_sample_batch,
ras_sample_semantic,
sample_scores,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _kept(logits: torch.Tensor, top_k: int | torch.Tensor, top_p: float | torch.Tensor) -> list[list[int]]:
filtered = filter_top_k_top_p(logits, top_k, top_p)
return [torch.nonzero(row.isfinite()).reshape(-1).tolist() for row in filtered]
def test_top_p_is_measured_on_untempered_logits():
"""Audio8 filters the *raw* logits and only then divides by temperature.
Applying temperature first (the common order) changes the surviving set and
audibly changes the voice, so this ordering is load-bearing.
"""
logits = torch.log(torch.tensor([[0.6, 0.3, 0.1]]))
# Raw softmax is exactly (0.6, 0.3, 0.1): top_p=0.9 keeps the first two
# (cumsum 0.6, 0.9) and drops the third (cumsum 1.0 > 0.9).
assert _kept(logits, top_k=0, top_p=0.9) == [[0, 1]]
# Had the logits been tempered by 0.5 first, probabilities would become
# (0.7insh, ...) and only one token would survive.
tempered = _kept(logits / 0.5, top_k=0, top_p=0.9)
assert tempered == [[0, 1]] or tempered == [[0]]
def test_argmax_always_survives_filtering():
logits = torch.tensor([[10.0, -1.0, -2.0]])
# top_p=0.0 would otherwise remove every candidate and produce NaNs.
assert _kept(logits, top_k=1, top_p=0.0) == [[0]]
def test_per_row_top_k_and_top_p_are_honoured():
logits = torch.log(torch.tensor([[0.5, 0.3, 0.15, 0.05], [0.5, 0.3, 0.15, 0.05]]))
kept = _kept(logits, top_k=torch.tensor([1, 3]), top_p=torch.tensor([1.0, 1.0]))
assert kept == [[0], [0, 1, 2]]
def test_zero_temperature_row_falls_back_to_argmax():
logits = torch.tensor([[1.0, 5.0, 2.0], [1.0, 5.0, 2.0]])
out = sample_scores(
logits,
temperature=torch.tensor([0.0, 1.0]),
top_k=0,
top_p=1.0,
generator=torch.Generator().manual_seed(0),
)
assert int(out[0]) == 1
def test_ras_substitutes_only_repeated_semantic_ids():
"""RAS re-rolls a repeat from a flatter distribution; the EOS slot and
non-repeats must pass through untouched, or utterances get truncated."""
num_semantic = 2
# Row 0 will draw id 0 (a repeat) -> substituted.
# Row 1 will draw id 2 (the EOS slot) -> never substituted even if recent.
logits = torch.tensor(
[
[10.0, -20.0, -20.0],
[-20.0, -20.0, 10.0],
]
)
recent = torch.tensor([[0, -1], [2, -1]])
out = ras_sample_semantic(
logits,
recent,
temperature=1e-6, # deterministic: argmax within each draw
top_k=0,
top_p=1.0,
ras_temperature=1e-6,
ras_top_p=1.0,
num_semantic_ids=num_semantic,
)
# Both draws are argmax, so substitution is observable only through which
# branch was taken; assert the EOS row is preserved verbatim.
assert int(out[1]) == 2
assert int(out[0]) == 0
def test_ras_without_history_returns_the_plain_draw():
logits = torch.tensor([[10.0, 0.0, -5.0]])
out = ras_sample_semantic(
logits,
None,
temperature=1e-6,
top_k=0,
top_p=1.0,
ras_temperature=1.0,
ras_top_p=0.9,
num_semantic_ids=2,
)
assert int(out[0]) == 0
def test_sampling_is_reproducible_for_a_seeded_generator():
logits = torch.randn(4, 32, generator=torch.Generator().manual_seed(7))
first = sample_scores(logits, temperature=0.7, top_k=8, top_p=0.9, generator=torch.Generator().manual_seed(1234))
second = sample_scores(logits, temperature=0.7, top_k=8, top_p=0.9, generator=torch.Generator().manual_seed(1234))
assert torch.equal(first, second)
def test_batch_routes_each_row_to_its_own_generator():
"""Regression for the shared slot-0 generator bug.
A shared generator over ``[num_reqs, vocab]`` made every row draw from slot
0's RNG stream, so a request's audio depended on its batch neighbours and
per-request ``seed`` was ignored for slots > 0. Flat logits make the draw
purely noise-driven so the wrong generator changes the token.
"""
# Flat logits: the sampled id is determined only by the generator's noise,
# so routing the generator to the wrong slot changes the result (coincidence ~1/256).
logits = torch.zeros(2, 256)
kw = dict(temperature=1.0, top_k=0, top_p=1.0, ras_temperature=1.0, ras_top_p=0.9, num_semantic_ids=200)
# (1) Each batched row equals sampling that row alone with its own generator.
batched = ras_sample_batch(
logits,
None,
generators={0: torch.Generator().manual_seed(11), 1: torch.Generator().manual_seed(22)},
num_reqs=2,
**kw,
)
row0 = ras_sample_semantic(logits[0:1], None, generator=torch.Generator().manual_seed(11), **kw)
row1 = ras_sample_semantic(logits[1:2], None, generator=torch.Generator().manual_seed(22), **kw)
assert int(batched[0]) == int(row0)
assert int(batched[1]) == int(row1)
# (2) Row 1 must not change when only slot 0's seed changes.
other = ras_sample_batch(
logits,
None,
generators={0: torch.Generator().manual_seed(9999), 1: torch.Generator().manual_seed(22)},
num_reqs=2,
**kw,
)
assert int(other[1]) == int(batched[1])
def test_single_request_batch_keeps_the_fast_path():
"""``num_reqs == 1`` must match a direct single-row draw (no per-row loop)."""
logits = torch.randn(1, 48, generator=torch.Generator().manual_seed(5))
kw = dict(temperature=0.8, top_k=16, top_p=0.9, ras_temperature=1.0, ras_top_p=0.9, num_semantic_ids=40)
batched = ras_sample_batch(logits, None, generators={0: torch.Generator().manual_seed(99)}, num_reqs=1, **kw)
direct = ras_sample_semantic(logits, None, generator=torch.Generator().manual_seed(99), **kw)
assert torch.equal(batched, direct)
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from collections import defaultdict
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.stage_input_processors.audio8_tts import (
extract_last_frame,
slow_ar_to_codec_decoder_async_chunk,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
NUM_CODEBOOKS = 3
def _make_transfer_manager(*, chunk_frames=2, left_context=1, initial_chunk_frames=0):
return SimpleNamespace(
code_prompt_token_ids=defaultdict(list),
put_req_chunk=defaultdict(int),
connector=SimpleNamespace(
config={
"extra": {
"codec_chunk_frames": chunk_frames,
"codec_left_context_frames": left_context,
"initial_codec_chunk_frames": initial_chunk_frames,
}
}
),
)
def _make_request(req_id="req", *, finished=False):
return SimpleNamespace(
external_req_id=req_id,
additional_information=None,
is_finished=lambda: finished,
)
def _frame(value: int) -> torch.Tensor:
return torch.full((NUM_CODEBOOKS,), value, dtype=torch.long)
def _step(transfer_manager, request, frame_value: int | None, *, finished: bool = False):
multimodal_output = {} if frame_value is None else {"audio_codes": _frame(frame_value).unsqueeze(0)}
return slow_ar_to_codec_decoder_async_chunk(transfer_manager, multimodal_output, request, is_finished=finished)
def test_codes_accumulate_across_steps_and_chunk_is_withheld_until_full():
"""Codes must accumulate; emitting per-step would starve the codec of the
context it needs and resetting would truncate the utterance."""
transfer_manager = _make_transfer_manager(chunk_frames=2, left_context=1)
request = _make_request()
assert _step(transfer_manager, request, 1) is None
payload = _step(transfer_manager, request, 2)
assert payload is not None
assert len(transfer_manager.code_prompt_token_ids["req"]) == 2
# [num_codebooks, frames], codebook-major.
assert tuple(payload.codes.audio.shape) == (NUM_CODEBOOKS, 2)
assert payload.codes.audio[0].tolist() == [1, 2]
# Nothing precedes the first chunk, so there is no left context yet.
assert payload.meta.left_context_size == 0
def test_second_chunk_carries_left_context_and_reports_its_size():
transfer_manager = _make_transfer_manager(chunk_frames=2, left_context=1)
request = _make_request()
for value in (1, 2, 3):
_step(transfer_manager, request, value)
payload = _step(transfer_manager, request, 4)
assert payload is not None
# 1 context frame + 2 new frames.
assert tuple(payload.codes.audio.shape) == (NUM_CODEBOOKS, 3)
assert payload.codes.audio[0].tolist() == [2, 3, 4]
assert payload.meta.left_context_size == 1
def test_partial_tail_is_flushed_when_the_request_finishes():
transfer_manager = _make_transfer_manager(chunk_frames=4, left_context=2)
request = _make_request()
_step(transfer_manager, request, 1)
_step(transfer_manager, request, 2)
payload = _step(transfer_manager, request, 3, finished=True)
assert payload is not None
assert payload.codes.audio[0].tolist() == [1, 2, 3]
assert bool(payload.meta.finished) is True
def test_finished_request_without_any_frame_emits_a_terminal_payload():
transfer_manager = _make_transfer_manager()
payload = slow_ar_to_codec_decoder_async_chunk(
transfer_manager, None, _make_request(finished=True), is_finished=True
)
assert payload is not None
assert payload.codes.audio.numel() == 0
assert bool(payload.meta.finished) is True
def test_terminal_call_on_a_boundary_does_not_re_emit_the_last_chunk():
"""A finished call carrying no new frame, arriving when the stream sits
exactly on a chunk boundary, must close the stream rather than re-ship the
last window. Before the guard, the final chunk_frames frames shipped twice."""
transfer_manager = _make_transfer_manager(chunk_frames=2, left_context=1)
request = _make_request()
for value in (1, 2, 3, 4): # length ends exactly on a chunk boundary
_step(transfer_manager, request, value)
# Separate terminal call, no new frame: the boundary chunk is already sent.
payload = _step(transfer_manager, request, None, finished=True)
assert payload is not None
assert payload.codes.audio.numel() == 0
assert bool(payload.meta.finished) is True
def test_final_frame_on_a_boundary_still_emits_when_it_arrives_finished():
"""If the last frame both lands on a boundary and carries finished, its chunk
has not been shipped yet, so it must still be emitted (the guard must key on
'no new frame this call', not merely on being finished at a boundary)."""
transfer_manager = _make_transfer_manager(chunk_frames=2, left_context=1)
request = _make_request()
_step(transfer_manager, request, 1)
payload = _step(transfer_manager, request, 2, finished=True)
assert payload is not None
assert payload.codes.audio[0].tolist() == [1, 2]
assert bool(payload.meta.finished) is True
def test_initial_chunk_shortens_time_to_first_audio():
transfer_manager = _make_transfer_manager(chunk_frames=4, left_context=2, initial_chunk_frames=1)
request = _make_request()
payload = _step(transfer_manager, request, 1)
assert payload is not None
assert payload.codes.audio[0].tolist() == [1]
assert payload.meta.left_context_size == 0
def test_prefill_placeholder_frames_are_not_forwarded():
"""Prefill emits an all-zero placeholder frame per position; forwarding it
would prepend silence and desynchronise the chunk boundaries."""
assert extract_last_frame({"audio_codes": torch.zeros((3, NUM_CODEBOOKS), dtype=torch.long)}) is None
assert extract_last_frame({}) is None
assert extract_last_frame({"audio_codes": torch.zeros((0, NUM_CODEBOOKS), dtype=torch.long)}) is None
def test_explicit_validity_flag_overrides_the_all_zero_heuristic():
frame = torch.zeros((1, NUM_CODEBOOKS), dtype=torch.long)
out = extract_last_frame({"audio_codes": frame, "audio_code_valid": torch.tensor([True])})
assert out is not None and out.tolist() == [0] * NUM_CODEBOOKS
def test_invalid_chunk_config_fails_loudly():
transfer_manager = _make_transfer_manager(chunk_frames=0)
request = _make_request()
with pytest.raises(ValueError, match="codec chunk config"):
_step(transfer_manager, request, 1)