[TTS][New Model] support bosonai/higgs-audio-v3-tts-4b (#4169)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yuekai ZhangandYueqian Lin authored and GitHub committed 2026-06-06 15:59:12 +00:00
1 parent 963ba1ab7a
commit 82e74ce972
22 files changed
+4295

No files matched your search

@@ -0,0 +1,388 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end online tests for higgs-audio v3 against /v1/audio/speech.
Mirrors the higgs_audio_v2 test layout. Covers the plain-text-in / audio-out
happy path, a small concurrent burst (Stage 0 prefix caching has to coexist
with batching), the voice-clone path via ``ref_audio`` + ``ref_text``, the
upstream ``references[]`` cookbook alias added by
``normalize_references_alias`` in ``protocol/audio.py``, and the few
validator rejections that should remain 4xx.
"""
from __future__ import annotations
import os
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
os.environ.setdefault("VLLM_USE_DEEP_GEMM", "0")
os.environ.setdefault("VLLM_MOE_USE_DEEP_GEMM", "0")
import pytest
from tests.helpers.mark import hardware_test
from tests.helpers.media import load_test_audio_data_url
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = "bosonai/higgs-audio-v3-tts-4b"
STAGE_CONFIG = get_deploy_config_path("higgs_multimodal_qwen3.yaml")
SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
SERVER_ENV = {"VLLM_USE_DEEP_GEMM": "0", "VLLM_MOE_USE_DEEP_GEMM": "0"}
TEST_PARAMS = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=STAGE_CONFIG,
server_args=SERVER_ARGS,
env_dict=SERVER_ENV,
),
id="higgs_audio_v3_plain_text",
)
]
DEFAULT_SPEECH_TIMEOUT_S = 180.0
# Floor for ~0.5 s of 24 kHz mono PCM_16: 24000 * 0.5 * 2 bytes ~= 24 KiB.
# A WAV header adds 44 bytes; conservative floor catches truncated /
# silence-only outputs without flagging short legitimate clips.
_MIN_AUDIO_BYTES = 20_000
# Reuse the shared TTS reference clip (clean ~5 s 24 kHz mono human speech)
# vendored under tests/assets/qwen3_tts/. Keeps a single WAV across TTS
# tests rather than duplicating asset bytes.
_REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
_REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestHiggsAudioV3OnlineHappyPath:
"""Plain-text -> audio happy paths against the live HTTP server."""
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_plain_text_wav(self, omni_server, openai_client) -> None:
"""Single non-streaming WAV request - canonical TTS happy path."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Hello world.",
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None:
"""``max_new_tokens`` is one of the few extra fields the v3 validator accepts."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Innovation distinguishes between a leader and a follower.",
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"max_new_tokens": 500,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_concurrent_plain_text(self, omni_server, openai_client) -> None:
"""Three concurrent non-streaming requests - guards per-slot audio state and Stage-0 PC under batching."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "It was the night before my birthday.",
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
},
request_num=3,
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None:
"""Streaming PCM via talker2code2wav_async_chunk + the connector's codec_streaming path.
The deploy YAML pins ``async_chunk: true`` and ``codec_streaming: true`` so chunks
flow Stage 0 -> Stage 1 per AR loop. The Stage-1 codec honors
``meta.left_context_size`` and ``meta.right_holdback_size`` so per-chunk codec
windows stitch into a coherent PCM stream. The byte-count gate is the same as
the sync paths; per-chunk audio content is verified offline against Whisper.
NOTE: ``min_hnr_db=0.0`` sits below the typical speech-noise floor so the
check still catches catastrophic codec failure (silence, white noise, sample
scramble all give HNR << 0) while allowing for the sliding-window codec's
slightly-noisier-than-sync output. The default 1.0 dB threshold has only
~0.16 dB margin over measured single-request output (1.16 dB on L4), so
flake risk is too high there.
"""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "The quick brown fox jumps over the lazy dog.",
"stream": True,
"response_format": "pcm",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
"min_hnr_db": 0.0,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None:
"""Three concurrent streaming requests - guards per-request frame cursors
in ``talker2code2wav_async_chunk`` and per-slot delay-pattern state under batched AR.
NOTE: ``min_hnr_db=-2.0`` is looser than the single-request streaming test
because batched AR adds per-request codec quality variance (DAC boundary
artifacts compounded across the 3-way batch); -2.0 dB gives ~1.6 dB margin
over the measured worst-of-3 on L4 (-0.42 dB) while still well above the
catastrophic-failure region.
"""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "She sells seashells by the seashore.",
"stream": True,
"response_format": "pcm",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
"min_hnr_db": -2.0,
},
request_num=3,
)
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestHiggsAudioV3OnlineInlineControlTokens:
"""Inline control-token surface from the bosonai cookbook.
The model exposes four categories of ``<|category:value|>`` tokens
(emotion, style, prosody, sfx). Delivery tokens (emotion / style /
prosody speed-pitch-expressive) go at the START of input; positional
tokens (pause, sfx) go inline. SFX tokens must pair with their written
onomatopoeia.
These tests check the *serving surface* - the validator accepts the
payload, the engine produces audio, and the WAV has non-trivial size.
They do not assert audio quality (out of scope for CI).
"""
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None:
"""Delivery tokens (emotion + expressive_high) at the start of input."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": ("<|emotion:amusement|><|prosody:expressive_high|>Wait, that was actually hilarious."),
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_inline_style_whispering(self, omni_server, openai_client) -> None:
"""Style token at the start - ``whispering``."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "<|style:whispering|>It is just between you and me, alright?",
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None:
"""Two prosody tokens at the start - slow speed plus low pitch."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": (
"<|prosody:speed_slow|><|prosody:pitch_low|>The radar shows a storm approaching from the east."
),
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_inline_pause_mid_text(self, omni_server, openai_client) -> None:
"""Positional pause token placed inline between two clauses."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Hold on a moment <|prosody:pause|> let me think about it.",
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None:
"""SFX token paired with its written onomatopoeia (``<|sfx:laughter|>Hehe``)."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": (
"<|emotion:amusement|>I cannot believe that just happened. "
"<|sfx:laughter|>Hehe, I am still recovering from it."
),
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestHiggsAudioV3OnlineVoiceClone:
"""Voice clone via the two payload shapes the model serves."""
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None:
"""Canonical vllm-omni voice clone via ``ref_audio`` + ``ref_text``."""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Hello world.",
"ref_audio": _REF_AUDIO_URL,
"ref_text": _REF_TEXT,
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_voice_clone_references_alias(self, omni_server, openai_client) -> None:
"""BosonAI cookbook payload: ``references=[{audio_path, text}]``.
``normalize_references_alias`` (``protocol/audio.py``) is supposed to
translate the cookbook field into ``ref_audio`` / ``ref_text``; without
it the request would silently fall through to zero-shot synthesis.
"""
openai_client.send_audio_speech_request(
{
"model": omni_server.model,
"input": "Hello world.",
"references": [{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT}],
"stream": False,
"response_format": "wav",
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"min_audio_bytes": _MIN_AUDIO_BYTES,
}
)
@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
class TestHiggsAudioV3OnlineValidatorRejections:
"""Out-of-scope shapes must come back as 4xx.
These cases use ``send_audio_speech_http_request`` (raw HTTP POST against
``/v1/audio/speech``) rather than the OpenAI SDK helper because:
- We need the failure-path matcher; the SDK helper asserts success.
- The cookbook ``references`` field is not in the OpenAI SDK schema, so
the SDK strips it before the request leaves the client and the server
sees a plain TTS request that returns 200. Raw HTTP keeps the JSON body
intact so the validator actually fires.
"""
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_rejects_empty_input(self, omni_server, openai_client) -> None:
"""Empty text input - validator should reject before reaching the engine."""
openai_client.send_audio_speech_http_request(
{
"json": {
"model": omni_server.model,
"input": "",
"response_format": "wav",
},
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"err_code": (400, 422),
"err_message": ("input", "empty"),
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_rejects_multi_reference_payload(self, omni_server, openai_client) -> None:
"""``references[]`` with more than one entry - multi-shot voice clone is not supported."""
openai_client.send_audio_speech_http_request(
{
"json": {
"model": omni_server.model,
"input": "Hello world.",
"references": [
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
{"audio_path": _REF_AUDIO_URL, "text": _REF_TEXT},
],
"response_format": "wav",
},
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"err_code": (400, 422),
"err_message": "references",
}
)
@pytest.mark.core_model
@pytest.mark.tts
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_rejects_conflicting_ref_audio_and_references(self, omni_server, openai_client) -> None:
"""``ref_audio`` and ``references`` cannot both be set to different values."""
openai_client.send_audio_speech_http_request(
{
"json": {
"model": omni_server.model,
"input": "Hello world.",
"ref_audio": _REF_AUDIO_URL,
"references": [{"audio_path": "https://example.com/other.wav"}],
"response_format": "wav",
},
"timeout": DEFAULT_SPEECH_TIMEOUT_S,
"err_code": (400, 422),
"err_message": "mutually exclusive",
}
)
Whitespace-only changes.
@@ -0,0 +1,910 @@
# SPDX-License-Identifier: Apache-2.0
"""Deterministic unit tests for higgs-audio v3.
These tests verify AC-1 (config), AC-3 (prompt), AC-4 (fused modules),
AC-5 (delay pattern), AC-7 (stage processor), and AC-10 (registry)
without requiring the actual checkpoint or GPU.
"""
import pytest
import torch
# ---- AC-1: Configuration ----
class TestHiggsAudioV3Config:
def test_default_config_loads(self):
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
config = HiggsAudioV3Config()
assert config.num_codebooks == 8
assert config.codebook_size == 1026
assert config.audio_stream_bos_id == 1024
assert config.audio_stream_eos_id == 1025
assert config.sample_rate == 24000
assert config.frame_rate == 25
assert config.num_real_codes == 1024
assert config.tie_modality_embeddings is True
def test_custom_audio_encoder_config(self):
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
config = HiggsAudioV3Config(
audio_encoder_config={
"encoder_type": "discrete",
"num_codebooks": 4,
"vocab_size": 512,
"tie_word_embeddings": False,
}
)
assert config.num_codebooks == 4
assert config.codebook_size == 512
assert config.tie_modality_embeddings is False
def test_invalid_num_codebooks_rejected(self):
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": 0, "vocab_size": 1026})
def test_negative_num_codebooks_rejected(self):
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": -1, "vocab_size": 1026})
def test_special_token_ids_initially_none(self):
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
config = HiggsAudioV3Config()
assert config.tts_token_id is None
assert config.text_token_id is None
assert config.audio_continuation_id is None
def test_hidden_size_from_text_config(self):
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
config = HiggsAudioV3Config(text_config={"model_type": "qwen3", "hidden_size": 2560})
assert config.hidden_size == 2560
assert config.audio_hidden_size == 2560
# ---- AC-4: Fused Multi-Codebook Modules ----
class TestFusedModules:
def test_embedding_shape(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsFusedMultiTextEmbedding,
)
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=256)
assert embed.weight.shape == (8 * 1026, 256)
codes = torch.randint(0, 1024, (5, 8))
out = embed(codes)
assert out.shape == (5, 256)
def test_embedding_offset_indexing(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsFusedMultiTextEmbedding,
)
embed = HiggsFusedMultiTextEmbedding(num_codebooks=2, vocab_size=4, hidden_size=3)
# Set weights to identity-like pattern so we can verify offsets
with torch.no_grad():
embed.weight.zero_()
# Codebook 0, vocab [0..3] -> row 0..3
# Codebook 1, vocab [0..3] -> row 4..7
for i in range(8):
embed.weight[i, i % 3] = float(i)
# codes[0] = [1, 2] -> embed[1] + embed[4+2=6]
codes = torch.tensor([[1, 2]])
out = embed(codes)
expected = embed.weight[1] + embed.weight[6]
assert torch.allclose(out[0], expected)
def test_head_shape(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsFusedMultiTextHead,
)
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=256)
assert head.weight.shape == (8 * 1026, 256)
hidden = torch.randn(3, 256)
out = head.generate(hidden)
assert out.shape == (3, 8, 1026)
def test_tying(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsFusedMultiTextEmbedding,
HiggsFusedMultiTextHead,
)
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=64)
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=64)
head.weight = embed.weight
assert head.weight is embed.weight
# Modification to one should reflect in the other
with torch.no_grad():
embed.weight[0, 0] = 42.0
assert head.weight[0, 0] == 42.0
def test_codes_out_of_range(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsFusedMultiTextEmbedding,
)
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
# Code 1026 + offset for codebook 7 = 1026 + 7*1026 = 8208
# But weight only has 8*1026 = 8208 rows, so index 8208 is out of bounds
codes = torch.tensor([[0, 0, 0, 0, 0, 0, 0, 1026]]) # Last cb: 1026 is OOB
with pytest.raises(IndexError):
embed(codes)
# ---- AC-5: Delay Pattern Behavior ----
class TestDelayPatternBehavior:
"""Test the delay pattern masking logic extracted from the talker."""
def test_boc_eoc_ids(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
BOC_ID,
EOC_ID,
)
assert BOC_ID == 1024
assert EOC_ID == 1025
def test_delay_phase_boc_masking(self):
"""During delay phase, codebooks beyond delay_count are forced to BOC."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
num_codebooks = 8
# Simulate delay phase at step 3 (delay_count=2, so CBs 3-7 should be BOC)
codes = torch.randint(0, 1024, (num_codebooks,))
delay_count = 2
next_cb = delay_count + 1
if next_cb < num_codebooks:
codes[next_cb:] = BOC_ID
# CBs 0-2 should have original codes, CBs 3-7 should be BOC
assert all(codes[i] != BOC_ID or i >= 3 for i in range(num_codebooks))
assert all(codes[i] == BOC_ID for i in range(3, num_codebooks))
def test_cb0_eoc_triggers_rampdown(self):
"""EOC on codebook 0 starts ramp-down; EOC on other codebooks does not."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
EOC_ID,
)
num_codebooks = 8
codes = torch.randint(0, 1024, (num_codebooks,))
# Simulate: cb0 emits EOC
codes[0] = EOC_ID
assert int(codes[0].item()) == EOC_ID
# Ramp-down should start with countdown = N-2 = 6
eoc_countdown = num_codebooks - 2
assert eoc_countdown == 6
# If cb3 emits EOC but cb0 doesn't, no ramp-down
codes2 = torch.randint(0, 1024, (num_codebooks,))
codes2[3] = EOC_ID
assert int(codes2[0].item()) != EOC_ID # cb0 is not EOC
def test_rampdown_termination(self):
"""After N-2 steps of ramp-down, generation_done becomes True."""
num_codebooks = 8
eoc_countdown = num_codebooks - 2 # = 6
generation_done = False
for _ in range(6):
eoc_countdown -= 1
if eoc_countdown <= 0:
generation_done = True
assert generation_done is True
def test_all_boc_seed(self):
"""First audio step should seed last_codes with all-BOC."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
seeded = torch.full((8,), BOC_ID, dtype=torch.long)
assert seeded.shape == (8,)
assert all(seeded[i] == BOC_ID for i in range(8))
# ---- AC-5: Real Sampler Method Tests ----
class TestSamplerMethods:
"""Test the actual _sample_audio_codes and _apply_delay_pattern_masking methods."""
def _make_minimal_talker(self):
"""Create a minimal talker-like object with sampler/masking methods."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
class FakeTalker:
num_codebooks = 8
codebook_size = 1026
_audio_state = {}
t = FakeTalker()
t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
t._apply_delay_pattern_masking = (
mod.HiggsAudioV3TalkerForConditionalGeneration._apply_delay_pattern_masking.__get__(t)
)
return t
def test_sample_respects_mask(self):
"""Tokens masked to -inf must never be sampled."""
t = self._make_minimal_talker()
logits = torch.full((10, 1026), float("-inf"))
# Only token 500 is allowed for each row
logits[:, 500] = 0.0
result = t._sample_audio_codes(logits)
assert result.shape == (10,)
assert (result == 500).all(), f"Expected all 500, got {result.tolist()}"
def test_sample_all_masked_falls_back_to_argmax(self):
"""All-masked row should fall back to argmax (least-negative logit)."""
t = self._make_minimal_talker()
logits = torch.full((2, 1026), float("-inf"))
# Row 0: all masked
# Row 1: only token 42 allowed
logits[1, 42] = 0.0
result = t._sample_audio_codes(logits)
assert result.shape == (2,)
assert result[1].item() == 42
def test_delay_masking_forces_boc_during_delay(self):
"""During delay phase, codebooks beyond delay_count must have only BOC allowed."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
t = self._make_minimal_talker()
# Simulate delay_count=2 for batch row 0
t._audio_state[0] = {"num_delay": 2, "num_remaining_delays": None}
cb_logits = torch.zeros(1, 8, 1026) # [1 audio row, 8 codebooks, 1026 vocab]
t._apply_delay_pattern_masking(cb_logits, [0])
# CBs 3-7 should have only BOC allowed (everything else -inf)
for q in range(3, 8):
row = cb_logits[0, q]
assert row[BOC_ID].item() == 0.0 # BOC kept
# All non-BOC should be -inf
non_boc = torch.cat([row[:BOC_ID], row[BOC_ID + 1 :]])
assert (non_boc == float("-inf")).all(), f"CB{q} has non-inf non-BOC values"
def test_delay_masking_disallows_boc_for_active_codebooks(self):
"""Active codebooks during delay should have BOC disallowed."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
t = self._make_minimal_talker()
t._audio_state[0] = {"num_delay": 3, "num_remaining_delays": None}
cb_logits = torch.zeros(1, 8, 1026)
t._apply_delay_pattern_masking(cb_logits, [0])
# CBs 0-3 are active: BOC should be -inf
for q in range(4):
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
def test_delay_masking_only_cb0_allows_eoc(self):
"""Only codebook 0 should allow EOC during normal generation."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
t = self._make_minimal_talker()
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": None}
cb_logits = torch.zeros(1, 8, 1026)
t._apply_delay_pattern_masking(cb_logits, [0])
# CB0 should keep EOC
assert cb_logits[0, 0, EOC_ID].item() != float("-inf")
# CB1-7 should have EOC masked
for q in range(1, 8):
assert cb_logits[0, q, EOC_ID].item() == float("-inf"), f"CB{q} EOC not masked"
def test_rampdown_masking_locks_to_eoc(self):
"""During ramp-down, locked codebooks should only allow EOC."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
BOC_ID,
EOC_ID,
)
t = self._make_minimal_talker()
# Ramp-down with 4 remaining delays: lock CBs 0-3 to EOC
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": 4}
cb_logits = torch.zeros(1, 8, 1026)
t._apply_delay_pattern_masking(cb_logits, [0])
# CBs 0-3 locked: only EOC allowed
for q in range(4):
row = cb_logits[0, q]
assert row[EOC_ID].item() == 0.0
non_eoc = torch.cat([row[:EOC_ID], row[EOC_ID + 1 :]])
assert (non_eoc == float("-inf")).all(), f"Locked CB{q} has non-inf non-EOC"
# CBs 4-7 active: BOC and EOC disallowed
for q in range(4, 8):
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
assert cb_logits[0, q, EOC_ID].item() == float("-inf")
# ---- AC-6: Feedback Method Tests ----
class TestFeedbackMethods:
def test_postprocess_emits_audio_codes(self):
"""postprocess() should return codes from _last_audio_codes."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
class FakeTalker:
_last_audio_codes = torch.tensor([[100, 200, 300, 400, 500, 600, 700, 800]])
_postprocess_cursor = 0
t = FakeTalker()
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
result = t.postprocess(torch.zeros(1, 64))
assert "codes" in result
assert "audio" in result["codes"]
assert result["codes"]["audio"].shape == (1, 8)
def test_postprocess_skips_negative_rows(self):
"""postprocess() should skip rows with -1 (no audio)."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
class FakeTalker:
_last_audio_codes = torch.tensor([[-1, -1, -1, -1, -1, -1, -1, -1]])
_postprocess_cursor = 0
t = FakeTalker()
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
result = t.postprocess(torch.zeros(1, 64))
assert result == {}
def test_postprocess_advances_cursor(self):
"""postprocess() should advance cursor by 1 per call."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
class FakeTalker:
_last_audio_codes = torch.tensor(
[[100, 200, 300, 400, 500, 600, 700, 800], [-1, -1, -1, -1, -1, -1, -1, -1]]
)
_postprocess_cursor = 0
t = FakeTalker()
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
r1 = t.postprocess(torch.zeros(1, 64))
assert "codes" in r1
r2 = t.postprocess(torch.zeros(1, 64))
assert r2 == {} # Second row is all -1
# ---- AC-6: Audio Feedback Method Tests ----
class TestAudioFeedback:
"""Test _apply_audio_feedback with minimal fake talker."""
def _make_feedback_talker(self):
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
embed = mod.HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
torch.nn.init.ones_(embed.weight)
class FakeTalker:
num_codebooks = 8
codebook_size = 1026
_audio_continuation_id = 99999 # fake audio token
_last_step_query_start_loc = None
_audio_state = {}
multimodal_embedding = embed
model = type("M", (), {"embed_tokens": lambda self, ids: torch.zeros(ids.shape[0], 16)})()
t = FakeTalker()
t._apply_audio_feedback = mod.HiggsAudioV3TalkerForConditionalGeneration._apply_audio_feedback.__get__(t)
return t
def test_text_positions_unchanged(self):
"""Non-audio positions should not be modified."""
t = self._make_feedback_talker()
# No audio state → all positions unchanged
input_ids = torch.tensor([1, 2, 3, 4])
hidden = torch.randn(4, 16)
result = t._apply_audio_feedback(hidden, input_ids)
assert torch.equal(result, hidden)
def test_audio_position_replaced_with_state(self):
"""Audio position with state should have its embedding replaced."""
t = self._make_feedback_talker()
audio_id = t._audio_continuation_id
t._audio_state[1] = {"last_codes": torch.zeros(8, dtype=torch.long)}
input_ids = torch.tensor([1, audio_id, 3])
hidden = torch.zeros(3, 16)
result = t._apply_audio_feedback(hidden, input_ids)
# Position 1 (audio) should be non-zero (embedding of all-0 codes)
assert result[1].abs().sum() > 0
# Positions 0 and 2 should remain zero
assert result[0].abs().sum() == 0
assert result[2].abs().sum() == 0
def test_no_state_no_replacement(self):
"""Audio position without state should not be replaced."""
t = self._make_feedback_talker()
audio_id = t._audio_continuation_id
# No state for row 0
input_ids = torch.tensor([audio_id])
hidden = torch.zeros(1, 16)
result = t._apply_audio_feedback(hidden, input_ids)
assert torch.equal(result, hidden)
# ---- AC-8: Codec Strictness ----
class TestCodecStrictness:
def test_bundled_missing_quantizer_key_raises(self):
"""Bundled codec load must fail when a quantizer codebook key is missing."""
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
HiggsAudioV3Code2Wav,
)
config = HiggsAudioV3Config()
c2w = HiggsAudioV3Code2Wav(config=config)
# State with only 7 quantizers (missing quantizer.quantizers.7.*)
codec_state = {}
for i in range(7):
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
codec_state["fc2.weight"] = torch.randn(256, 1024)
codec_state["fc2.bias"] = torch.randn(256)
with pytest.raises(KeyError, match="quantizer 7"):
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
def test_bundled_missing_fc2_raises(self):
"""Bundled codec load must fail when fc2 keys are missing."""
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
HiggsAudioV3Code2Wav,
)
config = HiggsAudioV3Config()
c2w = HiggsAudioV3Code2Wav(config=config)
codec_state = {}
for i in range(8):
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
# No fc2 keys
with pytest.raises(KeyError, match="fc2"):
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
def test_bundled_unknown_decoder_key_rejected(self, monkeypatch):
"""Unknown decoder-side key should be rejected as unconsumed."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
HiggsAudioV3Code2Wav,
)
class FakeDAC(torch.nn.Module):
def __init__(self):
super().__init__()
self.dummy = torch.nn.Parameter(torch.zeros(1))
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
config = HiggsAudioV3Config()
c2w = HiggsAudioV3Code2Wav(config=config)
codec_state = _build_full_codec_state()
codec_state["unknown_decoder_module.weight"] = torch.randn(10)
with pytest.raises(RuntimeError, match="unexpected decoder-side keys"):
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
def test_bundled_encoder_side_keys_accepted(self, monkeypatch):
"""Known encoder-side keys should not trigger unconsumed-key rejection."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
HiggsAudioV3Code2Wav,
)
class FakeDAC(torch.nn.Module):
def __init__(self):
super().__init__()
self.dummy = torch.nn.Parameter(torch.zeros(1))
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
config = HiggsAudioV3Config()
c2w = HiggsAudioV3Code2Wav(config=config)
codec_state = _build_full_codec_state()
codec_state["acoustic_encoder.block.0.conv_t1.weight"] = torch.randn(10)
codec_state["semantic_model.encoder.layers.0.weight"] = torch.randn(10)
codec_state["fc.weight"] = torch.randn(10)
# Should NOT raise — encoder-side keys are allowed
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
assert c2w._loaded
def _build_minimal_codec_state() -> dict[str, torch.Tensor]:
"""Build a minimal codec state dict (quantizer + fc2, no DAC)."""
state: dict[str, torch.Tensor] = {}
for i in range(8):
state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
state["fc2.weight"] = torch.randn(256, 1024)
state["fc2.bias"] = torch.randn(256)
return state
def _build_full_codec_state() -> dict[str, torch.Tensor]:
"""Build a full codec state dict including acoustic_decoder keys.
The acoustic_decoder keys here match what a FakeDAC module expects
(just 'dummy'). Real DAC has many more keys.
"""
state = _build_minimal_codec_state()
state["acoustic_decoder.dummy"] = torch.zeros(1)
return state
# ---- AC-2: Loader Required-Key Validation ----
class TestLoaderRequiredKeys:
"""Test exact required-key validation in load_weights."""
def test_build_required_keys_count(self):
"""Required key set should have 1 text_emb + 36*11 layers + 1 norm + 1 modality = 399."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsAudioV3TalkerForConditionalGeneration,
)
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(36)
assert len(keys) == 1 + 36 * 11 + 1 + 1 # text + layers + norm + modality = 399
def test_build_required_keys_contains_norm(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsAudioV3TalkerForConditionalGeneration,
)
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
assert "body.norm.weight" in keys
def test_build_required_keys_contains_layer_subkeys(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsAudioV3TalkerForConditionalGeneration,
)
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
assert "body.layers.0.self_attn.k_proj.weight" in keys
assert "body.layers.1.mlp.gate_proj.weight" in keys
def test_build_required_keys_contains_modality_embedding(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsAudioV3TalkerForConditionalGeneration,
)
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
assert "tied.embedding.modality_embeddings.0.embedding.weight" in keys
def test_build_required_keys_contains_text_embedding(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
HiggsAudioV3TalkerForConditionalGeneration,
)
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
assert "tied.embedding.text_embedding.weight" in keys
class TestLoadWeightsEnforcement:
"""Test load_weights() itself with synthetic key sets via a minimal fake talker."""
def _make_fake_talker_and_keys(self):
"""Build a minimal talker-like object and the full 1-layer key set."""
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
# Minimal fake talker with just enough to run load_weights
talker = object.__new__(mod.HiggsAudioV3TalkerForConditionalGeneration)
torch.nn.Module.__init__(talker)
# Attach minimal attributes load_weights needs
talker.multimodal_embedding = mod.HiggsFusedMultiTextEmbedding(8, 1026, 16)
talker.modality_head = mod.HiggsFusedMultiTextHead(8, 1026, 16)
talker.tie_modality = True
talker.modality_head.weight = talker.multimodal_embedding.weight
# Fake backbone config with 1 layer
class FakeConfig:
num_hidden_layers = 1
tie_word_embeddings = True
talker._backbone_config = FakeConfig()
# Minimal model and lm_head so _BackboneWrapper can be constructed
talker.model = torch.nn.Module()
talker.lm_head = torch.nn.Module()
# Fake vllm_config for resolve_special_tokens (no-op)
class FakeModelConfig:
model = None
class FakeVllmConfig:
model_config = FakeModelConfig()
talker.vllm_config = FakeVllmConfig()
talker.config = type("C", (), {"resolve_special_tokens": lambda self, p: None})()
talker._resolved_tokens = True
talker._audio_continuation_id = None
talker._eos_token_id = None
# Build the full required key set for 1 layer
required = mod.HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
# Build synthetic weights: tiny tensors for every required key
weights = {}
for key in required:
if "modality_embeddings.0.embedding" in key:
weights[key] = torch.randn(8 * 1026, 16)
else:
weights[key] = torch.randn(4) # tiny placeholder
# Monkeypatch _BackboneWrapper.load_weights to accept and return keys
def fake_backbone_load(self_wrapper, ws):
consumed = set()
for name, _ in ws:
consumed.add(name)
return consumed
talker._fake_backbone_load = fake_backbone_load
return talker, weights, mod, fake_backbone_load
def test_load_weights_succeeds_with_all_keys(self, monkeypatch):
"""load_weights succeeds when all required keys are present."""
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
result = talker.load_weights(iter(weights.items()))
assert isinstance(result, set)
assert "multimodal_embedding.weight" in result
def test_load_weights_fails_missing_norm(self, monkeypatch):
"""load_weights raises when body.norm.weight is missing."""
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
del weights["body.norm.weight"]
with pytest.raises(RuntimeError, match="body.norm.weight"):
talker.load_weights(iter(weights.items()))
def test_load_weights_fails_missing_layer_subkey(self, monkeypatch):
"""load_weights raises when a specific layer subkey is missing."""
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
del weights["body.layers.0.self_attn.k_proj.weight"]
with pytest.raises(RuntimeError, match="body.layers.0.self_attn.k_proj.weight"):
talker.load_weights(iter(weights.items()))
def test_load_weights_fails_missing_modality_embedding(self, monkeypatch):
"""load_weights raises when modality embedding is missing."""
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
del weights["tied.embedding.modality_embeddings.0.embedding.weight"]
with pytest.raises(RuntimeError, match="modality_embeddings"):
talker.load_weights(iter(weights.items()))
def test_load_weights_fails_unknown_prefix(self, monkeypatch):
"""load_weights raises on unknown non-codec checkpoint prefix."""
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
weights["unknown.prefix.weight"] = torch.randn(4)
with pytest.raises(ValueError, match="Unexpected checkpoint key"):
talker.load_weights(iter(weights.items()))
# ---- AC-7: Stage Input Processor ----
class TestStageInputProcessor:
def test_revert_delay_pattern(self):
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
_revert_delay_pattern,
)
# 8 codebooks, 3 real frames -> delayed shape [8, 3+8-1=10]
Q, T = 8, 3
delayed = torch.full((Q, T + Q - 1), 1024) # Fill with BOC
for i in range(Q):
for t in range(T):
delayed[i, i + t] = i * 100 + t # Real codes at shifted positions
result = _revert_delay_pattern(delayed)
assert result.shape == (Q, T)
for i in range(Q):
for t in range(T):
assert result[i, t].item() == i * 100 + t
def test_revert_delay_pattern_rejects_wrong_codebooks(self):
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
_revert_delay_pattern,
)
# 7 codebooks should be rejected
codes = torch.zeros(7, 20)
with pytest.raises(ValueError, match="Expected exactly 8 codebook rows"):
_revert_delay_pattern(codes)
def test_revert_delay_pattern_rejects_too_few_frames(self):
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
_revert_delay_pattern,
)
# 8 codebooks but only 5 frames (need at least 8)
codes = torch.zeros(8, 5)
with pytest.raises(ValueError, match="Not enough frames"):
_revert_delay_pattern(codes)
def test_filter_real_code_frames(self):
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
_filter_real_code_frames,
)
# 8 codebooks, 4 frames
codes = torch.tensor(
[
[100, 200, 1024, 300], # cb0: frame 2 has BOC
[101, 201, 1024, 301],
[102, 202, 1024, 302],
[103, 203, 1024, 303],
[104, 204, 1024, 304],
[105, 205, 1024, 305],
[106, 206, 1024, 306],
[107, 207, 1024, 307],
]
)
result = _filter_real_code_frames(codes)
# Frame 2 (column 2) has BOC in all codebooks -> filtered out
assert result.shape == (8, 3)
assert result[0, 0].item() == 100
assert result[0, 1].item() == 200
assert result[0, 2].item() == 300
# ---- AC-10: Registry ----
class TestRegistry:
def test_talker_registered(self):
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
assert "HiggsMultimodalQwen3ForConditionalGeneration" in _OMNI_MODELS
assert "HiggsAudioV3TalkerForConditionalGeneration" in _OMNI_MODELS
def test_code2wav_registered(self):
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
assert "HiggsAudioV3Code2WavForConditionalGeneration" in _OMNI_MODELS
def test_pipeline_registered(self):
from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
assert "higgs_multimodal_qwen3" in _OMNI_PIPELINES
def test_deploy_yaml_exists(self):
import os
# __file__ = tests/unit/higgs_audio_v3/test_*.py → 4 dirnames to repo root
repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
yaml_path = os.path.join(repo_root, "vllm_omni", "deploy", "higgs_multimodal_qwen3.yaml")
assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}"
# ---- AC-3: Prompt Builder ----
class TestPromptBuilder:
def _make_mock_tokenizer(self):
"""Create a mock tokenizer with the required special tokens."""
class MockTokenizer:
def __init__(self):
self._added_vocab = {
"<|tts|>": 151700,
"<|text|>": 151701,
"<|audio|>": 151702,
"<|ref_audio|>": 151703,
"<|ref_text|>": 151704,
}
def get_added_vocab(self):
return self._added_vocab
def encode(self, text, add_special_tokens=True):
# Simple word-level tokenization for testing
return list(range(100, 100 + len(text.split())))
return MockTokenizer()
def test_plain_tts_prompt(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
HiggsAudioV3TokenizerAdapter,
)
tok = self._make_mock_tokenizer()
adapter = HiggsAudioV3TokenizerAdapter(tok)
ids = adapter.build_prompt("Hello world")
assert ids[0] == 151700 # <|tts|>
assert ids[1] == 151701 # <|text|>
assert ids[-1] == 151702 # <|audio|>
assert len(ids) == 2 + 2 + 1 # tts + text + 2 word tokens + audio
def test_empty_text_rejected(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
HiggsAudioV3TokenizerAdapter,
)
tok = self._make_mock_tokenizer()
adapter = HiggsAudioV3TokenizerAdapter(tok)
with pytest.raises(ValueError, match="non-empty"):
adapter.build_prompt("")
def test_whitespace_only_rejected(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
HiggsAudioV3TokenizerAdapter,
)
tok = self._make_mock_tokenizer()
adapter = HiggsAudioV3TokenizerAdapter(tok)
with pytest.raises(ValueError, match="non-empty"):
adapter.build_prompt(" ")
def test_missing_specials_rejected(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
HiggsAudioV3TokenizerAdapter,
)
class BadTokenizer:
def get_added_vocab(self):
return {"<|tts|>": 1} # Missing <|text|> and <|audio|>
with pytest.raises(ValueError, match="missing"):
HiggsAudioV3TokenizerAdapter(BadTokenizer())
def test_no_voice_clone_tokens_in_plain_tts(self):
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
HiggsAudioV3TokenizerAdapter,
)
tok = self._make_mock_tokenizer()
adapter = HiggsAudioV3TokenizerAdapter(tok)
ids = adapter.build_prompt("Hello")
# Should not contain ref_audio or ref_text token IDs
assert 151703 not in ids # <|ref_audio|>
assert 151704 not in ids # <|ref_text|>