[TTS][New Model] support bosonai/higgs-audio-v3-tts-4b (#4169)
Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
1 parent
963ba1ab7a
commit
82e74ce972
22 files changed
+4295
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,910 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Deterministic unit tests for higgs-audio v3.
|
||||
|
||||
These tests verify AC-1 (config), AC-3 (prompt), AC-4 (fused modules),
|
||||
AC-5 (delay pattern), AC-7 (stage processor), and AC-10 (registry)
|
||||
without requiring the actual checkpoint or GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# ---- AC-1: Configuration ----
|
||||
|
||||
|
||||
class TestHiggsAudioV3Config:
|
||||
def test_default_config_loads(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
config = HiggsAudioV3Config()
|
||||
assert config.num_codebooks == 8
|
||||
assert config.codebook_size == 1026
|
||||
assert config.audio_stream_bos_id == 1024
|
||||
assert config.audio_stream_eos_id == 1025
|
||||
assert config.sample_rate == 24000
|
||||
assert config.frame_rate == 25
|
||||
assert config.num_real_codes == 1024
|
||||
assert config.tie_modality_embeddings is True
|
||||
|
||||
def test_custom_audio_encoder_config(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
config = HiggsAudioV3Config(
|
||||
audio_encoder_config={
|
||||
"encoder_type": "discrete",
|
||||
"num_codebooks": 4,
|
||||
"vocab_size": 512,
|
||||
"tie_word_embeddings": False,
|
||||
}
|
||||
)
|
||||
assert config.num_codebooks == 4
|
||||
assert config.codebook_size == 512
|
||||
assert config.tie_modality_embeddings is False
|
||||
|
||||
def test_invalid_num_codebooks_rejected(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
|
||||
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": 0, "vocab_size": 1026})
|
||||
|
||||
def test_negative_num_codebooks_rejected(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="num_codebooks must be > 0"):
|
||||
HiggsAudioV3Config(audio_encoder_config={"num_codebooks": -1, "vocab_size": 1026})
|
||||
|
||||
def test_special_token_ids_initially_none(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
config = HiggsAudioV3Config()
|
||||
assert config.tts_token_id is None
|
||||
assert config.text_token_id is None
|
||||
assert config.audio_continuation_id is None
|
||||
|
||||
def test_hidden_size_from_text_config(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
|
||||
config = HiggsAudioV3Config(text_config={"model_type": "qwen3", "hidden_size": 2560})
|
||||
assert config.hidden_size == 2560
|
||||
assert config.audio_hidden_size == 2560
|
||||
|
||||
|
||||
# ---- AC-4: Fused Multi-Codebook Modules ----
|
||||
|
||||
|
||||
class TestFusedModules:
|
||||
def test_embedding_shape(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsFusedMultiTextEmbedding,
|
||||
)
|
||||
|
||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=256)
|
||||
assert embed.weight.shape == (8 * 1026, 256)
|
||||
|
||||
codes = torch.randint(0, 1024, (5, 8))
|
||||
out = embed(codes)
|
||||
assert out.shape == (5, 256)
|
||||
|
||||
def test_embedding_offset_indexing(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsFusedMultiTextEmbedding,
|
||||
)
|
||||
|
||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=2, vocab_size=4, hidden_size=3)
|
||||
# Set weights to identity-like pattern so we can verify offsets
|
||||
with torch.no_grad():
|
||||
embed.weight.zero_()
|
||||
# Codebook 0, vocab [0..3] -> row 0..3
|
||||
# Codebook 1, vocab [0..3] -> row 4..7
|
||||
for i in range(8):
|
||||
embed.weight[i, i % 3] = float(i)
|
||||
|
||||
# codes[0] = [1, 2] -> embed[1] + embed[4+2=6]
|
||||
codes = torch.tensor([[1, 2]])
|
||||
out = embed(codes)
|
||||
expected = embed.weight[1] + embed.weight[6]
|
||||
assert torch.allclose(out[0], expected)
|
||||
|
||||
def test_head_shape(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsFusedMultiTextHead,
|
||||
)
|
||||
|
||||
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=256)
|
||||
assert head.weight.shape == (8 * 1026, 256)
|
||||
|
||||
hidden = torch.randn(3, 256)
|
||||
out = head.generate(hidden)
|
||||
assert out.shape == (3, 8, 1026)
|
||||
|
||||
def test_tying(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsFusedMultiTextEmbedding,
|
||||
HiggsFusedMultiTextHead,
|
||||
)
|
||||
|
||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=64)
|
||||
head = HiggsFusedMultiTextHead(num_codebooks=8, vocab_size=1026, hidden_size=64)
|
||||
head.weight = embed.weight
|
||||
assert head.weight is embed.weight
|
||||
# Modification to one should reflect in the other
|
||||
with torch.no_grad():
|
||||
embed.weight[0, 0] = 42.0
|
||||
assert head.weight[0, 0] == 42.0
|
||||
|
||||
def test_codes_out_of_range(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsFusedMultiTextEmbedding,
|
||||
)
|
||||
|
||||
embed = HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
|
||||
# Code 1026 + offset for codebook 7 = 1026 + 7*1026 = 8208
|
||||
# But weight only has 8*1026 = 8208 rows, so index 8208 is out of bounds
|
||||
codes = torch.tensor([[0, 0, 0, 0, 0, 0, 0, 1026]]) # Last cb: 1026 is OOB
|
||||
with pytest.raises(IndexError):
|
||||
embed(codes)
|
||||
|
||||
|
||||
# ---- AC-5: Delay Pattern Behavior ----
|
||||
|
||||
|
||||
class TestDelayPatternBehavior:
|
||||
"""Test the delay pattern masking logic extracted from the talker."""
|
||||
|
||||
def test_boc_eoc_ids(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
BOC_ID,
|
||||
EOC_ID,
|
||||
)
|
||||
|
||||
assert BOC_ID == 1024
|
||||
assert EOC_ID == 1025
|
||||
|
||||
def test_delay_phase_boc_masking(self):
|
||||
"""During delay phase, codebooks beyond delay_count are forced to BOC."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||
|
||||
num_codebooks = 8
|
||||
# Simulate delay phase at step 3 (delay_count=2, so CBs 3-7 should be BOC)
|
||||
codes = torch.randint(0, 1024, (num_codebooks,))
|
||||
delay_count = 2
|
||||
next_cb = delay_count + 1
|
||||
if next_cb < num_codebooks:
|
||||
codes[next_cb:] = BOC_ID
|
||||
# CBs 0-2 should have original codes, CBs 3-7 should be BOC
|
||||
assert all(codes[i] != BOC_ID or i >= 3 for i in range(num_codebooks))
|
||||
assert all(codes[i] == BOC_ID for i in range(3, num_codebooks))
|
||||
|
||||
def test_cb0_eoc_triggers_rampdown(self):
|
||||
"""EOC on codebook 0 starts ramp-down; EOC on other codebooks does not."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
EOC_ID,
|
||||
)
|
||||
|
||||
num_codebooks = 8
|
||||
codes = torch.randint(0, 1024, (num_codebooks,))
|
||||
|
||||
# Simulate: cb0 emits EOC
|
||||
codes[0] = EOC_ID
|
||||
assert int(codes[0].item()) == EOC_ID
|
||||
# Ramp-down should start with countdown = N-2 = 6
|
||||
eoc_countdown = num_codebooks - 2
|
||||
assert eoc_countdown == 6
|
||||
|
||||
# If cb3 emits EOC but cb0 doesn't, no ramp-down
|
||||
codes2 = torch.randint(0, 1024, (num_codebooks,))
|
||||
codes2[3] = EOC_ID
|
||||
assert int(codes2[0].item()) != EOC_ID # cb0 is not EOC
|
||||
|
||||
def test_rampdown_termination(self):
|
||||
"""After N-2 steps of ramp-down, generation_done becomes True."""
|
||||
num_codebooks = 8
|
||||
eoc_countdown = num_codebooks - 2 # = 6
|
||||
generation_done = False
|
||||
for _ in range(6):
|
||||
eoc_countdown -= 1
|
||||
if eoc_countdown <= 0:
|
||||
generation_done = True
|
||||
assert generation_done is True
|
||||
|
||||
def test_all_boc_seed(self):
|
||||
"""First audio step should seed last_codes with all-BOC."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||
|
||||
seeded = torch.full((8,), BOC_ID, dtype=torch.long)
|
||||
assert seeded.shape == (8,)
|
||||
assert all(seeded[i] == BOC_ID for i in range(8))
|
||||
|
||||
|
||||
# ---- AC-5: Real Sampler Method Tests ----
|
||||
|
||||
|
||||
class TestSamplerMethods:
|
||||
"""Test the actual _sample_audio_codes and _apply_delay_pattern_masking methods."""
|
||||
|
||||
def _make_minimal_talker(self):
|
||||
"""Create a minimal talker-like object with sampler/masking methods."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||
|
||||
class FakeTalker:
|
||||
num_codebooks = 8
|
||||
codebook_size = 1026
|
||||
_audio_state = {}
|
||||
|
||||
t = FakeTalker()
|
||||
t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
|
||||
t._apply_delay_pattern_masking = (
|
||||
mod.HiggsAudioV3TalkerForConditionalGeneration._apply_delay_pattern_masking.__get__(t)
|
||||
)
|
||||
return t
|
||||
|
||||
def test_sample_respects_mask(self):
|
||||
"""Tokens masked to -inf must never be sampled."""
|
||||
t = self._make_minimal_talker()
|
||||
logits = torch.full((10, 1026), float("-inf"))
|
||||
# Only token 500 is allowed for each row
|
||||
logits[:, 500] = 0.0
|
||||
result = t._sample_audio_codes(logits)
|
||||
assert result.shape == (10,)
|
||||
assert (result == 500).all(), f"Expected all 500, got {result.tolist()}"
|
||||
|
||||
def test_sample_all_masked_falls_back_to_argmax(self):
|
||||
"""All-masked row should fall back to argmax (least-negative logit)."""
|
||||
t = self._make_minimal_talker()
|
||||
logits = torch.full((2, 1026), float("-inf"))
|
||||
# Row 0: all masked
|
||||
# Row 1: only token 42 allowed
|
||||
logits[1, 42] = 0.0
|
||||
result = t._sample_audio_codes(logits)
|
||||
assert result.shape == (2,)
|
||||
assert result[1].item() == 42
|
||||
|
||||
def test_delay_masking_forces_boc_during_delay(self):
|
||||
"""During delay phase, codebooks beyond delay_count must have only BOC allowed."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||
|
||||
t = self._make_minimal_talker()
|
||||
# Simulate delay_count=2 for batch row 0
|
||||
t._audio_state[0] = {"num_delay": 2, "num_remaining_delays": None}
|
||||
cb_logits = torch.zeros(1, 8, 1026) # [1 audio row, 8 codebooks, 1026 vocab]
|
||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||
# CBs 3-7 should have only BOC allowed (everything else -inf)
|
||||
for q in range(3, 8):
|
||||
row = cb_logits[0, q]
|
||||
assert row[BOC_ID].item() == 0.0 # BOC kept
|
||||
# All non-BOC should be -inf
|
||||
non_boc = torch.cat([row[:BOC_ID], row[BOC_ID + 1 :]])
|
||||
assert (non_boc == float("-inf")).all(), f"CB{q} has non-inf non-BOC values"
|
||||
|
||||
def test_delay_masking_disallows_boc_for_active_codebooks(self):
|
||||
"""Active codebooks during delay should have BOC disallowed."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||
|
||||
t = self._make_minimal_talker()
|
||||
t._audio_state[0] = {"num_delay": 3, "num_remaining_delays": None}
|
||||
cb_logits = torch.zeros(1, 8, 1026)
|
||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||
# CBs 0-3 are active: BOC should be -inf
|
||||
for q in range(4):
|
||||
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
|
||||
|
||||
def test_delay_masking_only_cb0_allows_eoc(self):
|
||||
"""Only codebook 0 should allow EOC during normal generation."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
|
||||
|
||||
t = self._make_minimal_talker()
|
||||
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": None}
|
||||
cb_logits = torch.zeros(1, 8, 1026)
|
||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||
# CB0 should keep EOC
|
||||
assert cb_logits[0, 0, EOC_ID].item() != float("-inf")
|
||||
# CB1-7 should have EOC masked
|
||||
for q in range(1, 8):
|
||||
assert cb_logits[0, q, EOC_ID].item() == float("-inf"), f"CB{q} EOC not masked"
|
||||
|
||||
def test_rampdown_masking_locks_to_eoc(self):
|
||||
"""During ramp-down, locked codebooks should only allow EOC."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
BOC_ID,
|
||||
EOC_ID,
|
||||
)
|
||||
|
||||
t = self._make_minimal_talker()
|
||||
# Ramp-down with 4 remaining delays: lock CBs 0-3 to EOC
|
||||
t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": 4}
|
||||
cb_logits = torch.zeros(1, 8, 1026)
|
||||
t._apply_delay_pattern_masking(cb_logits, [0])
|
||||
# CBs 0-3 locked: only EOC allowed
|
||||
for q in range(4):
|
||||
row = cb_logits[0, q]
|
||||
assert row[EOC_ID].item() == 0.0
|
||||
non_eoc = torch.cat([row[:EOC_ID], row[EOC_ID + 1 :]])
|
||||
assert (non_eoc == float("-inf")).all(), f"Locked CB{q} has non-inf non-EOC"
|
||||
# CBs 4-7 active: BOC and EOC disallowed
|
||||
for q in range(4, 8):
|
||||
assert cb_logits[0, q, BOC_ID].item() == float("-inf")
|
||||
assert cb_logits[0, q, EOC_ID].item() == float("-inf")
|
||||
|
||||
|
||||
# ---- AC-6: Feedback Method Tests ----
|
||||
|
||||
|
||||
class TestFeedbackMethods:
|
||||
def test_postprocess_emits_audio_codes(self):
|
||||
"""postprocess() should return codes from _last_audio_codes."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||
|
||||
class FakeTalker:
|
||||
_last_audio_codes = torch.tensor([[100, 200, 300, 400, 500, 600, 700, 800]])
|
||||
_postprocess_cursor = 0
|
||||
|
||||
t = FakeTalker()
|
||||
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
||||
result = t.postprocess(torch.zeros(1, 64))
|
||||
assert "codes" in result
|
||||
assert "audio" in result["codes"]
|
||||
assert result["codes"]["audio"].shape == (1, 8)
|
||||
|
||||
def test_postprocess_skips_negative_rows(self):
|
||||
"""postprocess() should skip rows with -1 (no audio)."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||
|
||||
class FakeTalker:
|
||||
_last_audio_codes = torch.tensor([[-1, -1, -1, -1, -1, -1, -1, -1]])
|
||||
_postprocess_cursor = 0
|
||||
|
||||
t = FakeTalker()
|
||||
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
||||
result = t.postprocess(torch.zeros(1, 64))
|
||||
assert result == {}
|
||||
|
||||
def test_postprocess_advances_cursor(self):
|
||||
"""postprocess() should advance cursor by 1 per call."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||
|
||||
class FakeTalker:
|
||||
_last_audio_codes = torch.tensor(
|
||||
[[100, 200, 300, 400, 500, 600, 700, 800], [-1, -1, -1, -1, -1, -1, -1, -1]]
|
||||
)
|
||||
_postprocess_cursor = 0
|
||||
|
||||
t = FakeTalker()
|
||||
t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
|
||||
r1 = t.postprocess(torch.zeros(1, 64))
|
||||
assert "codes" in r1
|
||||
r2 = t.postprocess(torch.zeros(1, 64))
|
||||
assert r2 == {} # Second row is all -1
|
||||
|
||||
|
||||
# ---- AC-6: Audio Feedback Method Tests ----
|
||||
|
||||
|
||||
class TestAudioFeedback:
|
||||
"""Test _apply_audio_feedback with minimal fake talker."""
|
||||
|
||||
def _make_feedback_talker(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||
|
||||
embed = mod.HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
|
||||
torch.nn.init.ones_(embed.weight)
|
||||
|
||||
class FakeTalker:
|
||||
num_codebooks = 8
|
||||
codebook_size = 1026
|
||||
_audio_continuation_id = 99999 # fake audio token
|
||||
_last_step_query_start_loc = None
|
||||
_audio_state = {}
|
||||
multimodal_embedding = embed
|
||||
model = type("M", (), {"embed_tokens": lambda self, ids: torch.zeros(ids.shape[0], 16)})()
|
||||
|
||||
t = FakeTalker()
|
||||
t._apply_audio_feedback = mod.HiggsAudioV3TalkerForConditionalGeneration._apply_audio_feedback.__get__(t)
|
||||
return t
|
||||
|
||||
def test_text_positions_unchanged(self):
|
||||
"""Non-audio positions should not be modified."""
|
||||
t = self._make_feedback_talker()
|
||||
# No audio state → all positions unchanged
|
||||
input_ids = torch.tensor([1, 2, 3, 4])
|
||||
hidden = torch.randn(4, 16)
|
||||
result = t._apply_audio_feedback(hidden, input_ids)
|
||||
assert torch.equal(result, hidden)
|
||||
|
||||
def test_audio_position_replaced_with_state(self):
|
||||
"""Audio position with state should have its embedding replaced."""
|
||||
t = self._make_feedback_talker()
|
||||
audio_id = t._audio_continuation_id
|
||||
t._audio_state[1] = {"last_codes": torch.zeros(8, dtype=torch.long)}
|
||||
input_ids = torch.tensor([1, audio_id, 3])
|
||||
hidden = torch.zeros(3, 16)
|
||||
result = t._apply_audio_feedback(hidden, input_ids)
|
||||
# Position 1 (audio) should be non-zero (embedding of all-0 codes)
|
||||
assert result[1].abs().sum() > 0
|
||||
# Positions 0 and 2 should remain zero
|
||||
assert result[0].abs().sum() == 0
|
||||
assert result[2].abs().sum() == 0
|
||||
|
||||
def test_no_state_no_replacement(self):
|
||||
"""Audio position without state should not be replaced."""
|
||||
t = self._make_feedback_talker()
|
||||
audio_id = t._audio_continuation_id
|
||||
# No state for row 0
|
||||
input_ids = torch.tensor([audio_id])
|
||||
hidden = torch.zeros(1, 16)
|
||||
result = t._apply_audio_feedback(hidden, input_ids)
|
||||
assert torch.equal(result, hidden)
|
||||
|
||||
|
||||
# ---- AC-8: Codec Strictness ----
|
||||
|
||||
|
||||
class TestCodecStrictness:
|
||||
def test_bundled_missing_quantizer_key_raises(self):
|
||||
"""Bundled codec load must fail when a quantizer codebook key is missing."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||
HiggsAudioV3Code2Wav,
|
||||
)
|
||||
|
||||
config = HiggsAudioV3Config()
|
||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||
|
||||
# State with only 7 quantizers (missing quantizer.quantizers.7.*)
|
||||
codec_state = {}
|
||||
for i in range(7):
|
||||
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
||||
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
||||
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
||||
codec_state["fc2.weight"] = torch.randn(256, 1024)
|
||||
codec_state["fc2.bias"] = torch.randn(256)
|
||||
|
||||
with pytest.raises(KeyError, match="quantizer 7"):
|
||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||
|
||||
def test_bundled_missing_fc2_raises(self):
|
||||
"""Bundled codec load must fail when fc2 keys are missing."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||
HiggsAudioV3Code2Wav,
|
||||
)
|
||||
|
||||
config = HiggsAudioV3Config()
|
||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||
|
||||
codec_state = {}
|
||||
for i in range(8):
|
||||
codec_state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
||||
codec_state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
||||
codec_state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
||||
# No fc2 keys
|
||||
|
||||
with pytest.raises(KeyError, match="fc2"):
|
||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||
|
||||
def test_bundled_unknown_decoder_key_rejected(self, monkeypatch):
|
||||
"""Unknown decoder-side key should be rejected as unconsumed."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||
HiggsAudioV3Code2Wav,
|
||||
)
|
||||
|
||||
class FakeDAC(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.dummy = torch.nn.Parameter(torch.zeros(1))
|
||||
|
||||
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
|
||||
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
|
||||
|
||||
config = HiggsAudioV3Config()
|
||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||
|
||||
codec_state = _build_full_codec_state()
|
||||
codec_state["unknown_decoder_module.weight"] = torch.randn(10)
|
||||
|
||||
with pytest.raises(RuntimeError, match="unexpected decoder-side keys"):
|
||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||
|
||||
def test_bundled_encoder_side_keys_accepted(self, monkeypatch):
|
||||
"""Known encoder-side keys should not trigger unconsumed-key rejection."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_code2wav as c2w_mod
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
|
||||
HiggsAudioV3Config,
|
||||
)
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_code2wav import (
|
||||
HiggsAudioV3Code2Wav,
|
||||
)
|
||||
|
||||
class FakeDAC(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.dummy = torch.nn.Parameter(torch.zeros(1))
|
||||
|
||||
monkeypatch.setattr(c2w_mod, "build_boson_dac_decoder", lambda device: FakeDAC().to(device))
|
||||
monkeypatch.setattr(c2w_mod, "build_higgs_audio_acoustic_decoder", lambda cfg, device: FakeDAC().to(device))
|
||||
|
||||
config = HiggsAudioV3Config()
|
||||
c2w = HiggsAudioV3Code2Wav(config=config)
|
||||
|
||||
codec_state = _build_full_codec_state()
|
||||
codec_state["acoustic_encoder.block.0.conv_t1.weight"] = torch.randn(10)
|
||||
codec_state["semantic_model.encoder.layers.0.weight"] = torch.randn(10)
|
||||
codec_state["fc.weight"] = torch.randn(10)
|
||||
|
||||
# Should NOT raise — encoder-side keys are allowed
|
||||
c2w._load_from_bundled_state(codec_state, device=torch.device("cpu"))
|
||||
assert c2w._loaded
|
||||
|
||||
|
||||
def _build_minimal_codec_state() -> dict[str, torch.Tensor]:
|
||||
"""Build a minimal codec state dict (quantizer + fc2, no DAC)."""
|
||||
state: dict[str, torch.Tensor] = {}
|
||||
for i in range(8):
|
||||
state[f"quantizer.quantizers.{i}.codebook.embed"] = torch.randn(1024, 64)
|
||||
state[f"quantizer.quantizers.{i}.project_out.weight"] = torch.randn(1024, 64)
|
||||
state[f"quantizer.quantizers.{i}.project_out.bias"] = torch.randn(1024)
|
||||
state["fc2.weight"] = torch.randn(256, 1024)
|
||||
state["fc2.bias"] = torch.randn(256)
|
||||
return state
|
||||
|
||||
|
||||
def _build_full_codec_state() -> dict[str, torch.Tensor]:
|
||||
"""Build a full codec state dict including acoustic_decoder keys.
|
||||
|
||||
The acoustic_decoder keys here match what a FakeDAC module expects
|
||||
(just 'dummy'). Real DAC has many more keys.
|
||||
"""
|
||||
state = _build_minimal_codec_state()
|
||||
state["acoustic_decoder.dummy"] = torch.zeros(1)
|
||||
return state
|
||||
|
||||
|
||||
# ---- AC-2: Loader Required-Key Validation ----
|
||||
|
||||
|
||||
class TestLoaderRequiredKeys:
|
||||
"""Test exact required-key validation in load_weights."""
|
||||
|
||||
def test_build_required_keys_count(self):
|
||||
"""Required key set should have 1 text_emb + 36*11 layers + 1 norm + 1 modality = 399."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsAudioV3TalkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(36)
|
||||
assert len(keys) == 1 + 36 * 11 + 1 + 1 # text + layers + norm + modality = 399
|
||||
|
||||
def test_build_required_keys_contains_norm(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsAudioV3TalkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
|
||||
assert "body.norm.weight" in keys
|
||||
|
||||
def test_build_required_keys_contains_layer_subkeys(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsAudioV3TalkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(2)
|
||||
assert "body.layers.0.self_attn.k_proj.weight" in keys
|
||||
assert "body.layers.1.mlp.gate_proj.weight" in keys
|
||||
|
||||
def test_build_required_keys_contains_modality_embedding(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsAudioV3TalkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
||||
assert "tied.embedding.modality_embeddings.0.embedding.weight" in keys
|
||||
|
||||
def test_build_required_keys_contains_text_embedding(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
|
||||
HiggsAudioV3TalkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
keys = HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
||||
assert "tied.embedding.text_embedding.weight" in keys
|
||||
|
||||
|
||||
class TestLoadWeightsEnforcement:
|
||||
"""Test load_weights() itself with synthetic key sets via a minimal fake talker."""
|
||||
|
||||
def _make_fake_talker_and_keys(self):
|
||||
"""Build a minimal talker-like object and the full 1-layer key set."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
|
||||
|
||||
# Minimal fake talker with just enough to run load_weights
|
||||
talker = object.__new__(mod.HiggsAudioV3TalkerForConditionalGeneration)
|
||||
torch.nn.Module.__init__(talker)
|
||||
|
||||
# Attach minimal attributes load_weights needs
|
||||
talker.multimodal_embedding = mod.HiggsFusedMultiTextEmbedding(8, 1026, 16)
|
||||
talker.modality_head = mod.HiggsFusedMultiTextHead(8, 1026, 16)
|
||||
talker.tie_modality = True
|
||||
talker.modality_head.weight = talker.multimodal_embedding.weight
|
||||
|
||||
# Fake backbone config with 1 layer
|
||||
class FakeConfig:
|
||||
num_hidden_layers = 1
|
||||
tie_word_embeddings = True
|
||||
|
||||
talker._backbone_config = FakeConfig()
|
||||
|
||||
# Minimal model and lm_head so _BackboneWrapper can be constructed
|
||||
talker.model = torch.nn.Module()
|
||||
talker.lm_head = torch.nn.Module()
|
||||
|
||||
# Fake vllm_config for resolve_special_tokens (no-op)
|
||||
class FakeModelConfig:
|
||||
model = None
|
||||
|
||||
class FakeVllmConfig:
|
||||
model_config = FakeModelConfig()
|
||||
|
||||
talker.vllm_config = FakeVllmConfig()
|
||||
talker.config = type("C", (), {"resolve_special_tokens": lambda self, p: None})()
|
||||
talker._resolved_tokens = True
|
||||
talker._audio_continuation_id = None
|
||||
talker._eos_token_id = None
|
||||
|
||||
# Build the full required key set for 1 layer
|
||||
required = mod.HiggsAudioV3TalkerForConditionalGeneration._build_required_keys(1)
|
||||
|
||||
# Build synthetic weights: tiny tensors for every required key
|
||||
weights = {}
|
||||
for key in required:
|
||||
if "modality_embeddings.0.embedding" in key:
|
||||
weights[key] = torch.randn(8 * 1026, 16)
|
||||
else:
|
||||
weights[key] = torch.randn(4) # tiny placeholder
|
||||
|
||||
# Monkeypatch _BackboneWrapper.load_weights to accept and return keys
|
||||
def fake_backbone_load(self_wrapper, ws):
|
||||
consumed = set()
|
||||
for name, _ in ws:
|
||||
consumed.add(name)
|
||||
return consumed
|
||||
|
||||
talker._fake_backbone_load = fake_backbone_load
|
||||
return talker, weights, mod, fake_backbone_load
|
||||
|
||||
def test_load_weights_succeeds_with_all_keys(self, monkeypatch):
|
||||
"""load_weights succeeds when all required keys are present."""
|
||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||
result = talker.load_weights(iter(weights.items()))
|
||||
assert isinstance(result, set)
|
||||
assert "multimodal_embedding.weight" in result
|
||||
|
||||
def test_load_weights_fails_missing_norm(self, monkeypatch):
|
||||
"""load_weights raises when body.norm.weight is missing."""
|
||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||
del weights["body.norm.weight"]
|
||||
with pytest.raises(RuntimeError, match="body.norm.weight"):
|
||||
talker.load_weights(iter(weights.items()))
|
||||
|
||||
def test_load_weights_fails_missing_layer_subkey(self, monkeypatch):
|
||||
"""load_weights raises when a specific layer subkey is missing."""
|
||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||
del weights["body.layers.0.self_attn.k_proj.weight"]
|
||||
with pytest.raises(RuntimeError, match="body.layers.0.self_attn.k_proj.weight"):
|
||||
talker.load_weights(iter(weights.items()))
|
||||
|
||||
def test_load_weights_fails_missing_modality_embedding(self, monkeypatch):
|
||||
"""load_weights raises when modality embedding is missing."""
|
||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||
del weights["tied.embedding.modality_embeddings.0.embedding.weight"]
|
||||
with pytest.raises(RuntimeError, match="modality_embeddings"):
|
||||
talker.load_weights(iter(weights.items()))
|
||||
|
||||
def test_load_weights_fails_unknown_prefix(self, monkeypatch):
|
||||
"""load_weights raises on unknown non-codec checkpoint prefix."""
|
||||
talker, weights, mod, fake_load = self._make_fake_talker_and_keys()
|
||||
monkeypatch.setattr(mod._BackboneWrapper, "load_weights", fake_load)
|
||||
weights["unknown.prefix.weight"] = torch.randn(4)
|
||||
with pytest.raises(ValueError, match="Unexpected checkpoint key"):
|
||||
talker.load_weights(iter(weights.items()))
|
||||
|
||||
|
||||
# ---- AC-7: Stage Input Processor ----
|
||||
|
||||
|
||||
class TestStageInputProcessor:
|
||||
def test_revert_delay_pattern(self):
|
||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||
_revert_delay_pattern,
|
||||
)
|
||||
|
||||
# 8 codebooks, 3 real frames -> delayed shape [8, 3+8-1=10]
|
||||
Q, T = 8, 3
|
||||
delayed = torch.full((Q, T + Q - 1), 1024) # Fill with BOC
|
||||
for i in range(Q):
|
||||
for t in range(T):
|
||||
delayed[i, i + t] = i * 100 + t # Real codes at shifted positions
|
||||
result = _revert_delay_pattern(delayed)
|
||||
assert result.shape == (Q, T)
|
||||
for i in range(Q):
|
||||
for t in range(T):
|
||||
assert result[i, t].item() == i * 100 + t
|
||||
|
||||
def test_revert_delay_pattern_rejects_wrong_codebooks(self):
|
||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||
_revert_delay_pattern,
|
||||
)
|
||||
|
||||
# 7 codebooks should be rejected
|
||||
codes = torch.zeros(7, 20)
|
||||
with pytest.raises(ValueError, match="Expected exactly 8 codebook rows"):
|
||||
_revert_delay_pattern(codes)
|
||||
|
||||
def test_revert_delay_pattern_rejects_too_few_frames(self):
|
||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||
_revert_delay_pattern,
|
||||
)
|
||||
|
||||
# 8 codebooks but only 5 frames (need at least 8)
|
||||
codes = torch.zeros(8, 5)
|
||||
with pytest.raises(ValueError, match="Not enough frames"):
|
||||
_revert_delay_pattern(codes)
|
||||
|
||||
def test_filter_real_code_frames(self):
|
||||
from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
|
||||
_filter_real_code_frames,
|
||||
)
|
||||
|
||||
# 8 codebooks, 4 frames
|
||||
codes = torch.tensor(
|
||||
[
|
||||
[100, 200, 1024, 300], # cb0: frame 2 has BOC
|
||||
[101, 201, 1024, 301],
|
||||
[102, 202, 1024, 302],
|
||||
[103, 203, 1024, 303],
|
||||
[104, 204, 1024, 304],
|
||||
[105, 205, 1024, 305],
|
||||
[106, 206, 1024, 306],
|
||||
[107, 207, 1024, 307],
|
||||
]
|
||||
)
|
||||
result = _filter_real_code_frames(codes)
|
||||
# Frame 2 (column 2) has BOC in all codebooks -> filtered out
|
||||
assert result.shape == (8, 3)
|
||||
assert result[0, 0].item() == 100
|
||||
assert result[0, 1].item() == 200
|
||||
assert result[0, 2].item() == 300
|
||||
|
||||
|
||||
# ---- AC-10: Registry ----
|
||||
|
||||
|
||||
class TestRegistry:
|
||||
def test_talker_registered(self):
|
||||
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
||||
|
||||
assert "HiggsMultimodalQwen3ForConditionalGeneration" in _OMNI_MODELS
|
||||
assert "HiggsAudioV3TalkerForConditionalGeneration" in _OMNI_MODELS
|
||||
|
||||
def test_code2wav_registered(self):
|
||||
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
||||
|
||||
assert "HiggsAudioV3Code2WavForConditionalGeneration" in _OMNI_MODELS
|
||||
|
||||
def test_pipeline_registered(self):
|
||||
from vllm_omni.config.pipeline_registry import _OMNI_PIPELINES
|
||||
|
||||
assert "higgs_multimodal_qwen3" in _OMNI_PIPELINES
|
||||
|
||||
def test_deploy_yaml_exists(self):
|
||||
import os
|
||||
|
||||
# __file__ = tests/unit/higgs_audio_v3/test_*.py → 4 dirnames to repo root
|
||||
repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
yaml_path = os.path.join(repo_root, "vllm_omni", "deploy", "higgs_multimodal_qwen3.yaml")
|
||||
assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}"
|
||||
|
||||
|
||||
# ---- AC-3: Prompt Builder ----
|
||||
|
||||
|
||||
class TestPromptBuilder:
|
||||
def _make_mock_tokenizer(self):
|
||||
"""Create a mock tokenizer with the required special tokens."""
|
||||
|
||||
class MockTokenizer:
|
||||
def __init__(self):
|
||||
self._added_vocab = {
|
||||
"<|tts|>": 151700,
|
||||
"<|text|>": 151701,
|
||||
"<|audio|>": 151702,
|
||||
"<|ref_audio|>": 151703,
|
||||
"<|ref_text|>": 151704,
|
||||
}
|
||||
|
||||
def get_added_vocab(self):
|
||||
return self._added_vocab
|
||||
|
||||
def encode(self, text, add_special_tokens=True):
|
||||
# Simple word-level tokenization for testing
|
||||
return list(range(100, 100 + len(text.split())))
|
||||
|
||||
return MockTokenizer()
|
||||
|
||||
def test_plain_tts_prompt(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||
HiggsAudioV3TokenizerAdapter,
|
||||
)
|
||||
|
||||
tok = self._make_mock_tokenizer()
|
||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||
ids = adapter.build_prompt("Hello world")
|
||||
assert ids[0] == 151700 # <|tts|>
|
||||
assert ids[1] == 151701 # <|text|>
|
||||
assert ids[-1] == 151702 # <|audio|>
|
||||
assert len(ids) == 2 + 2 + 1 # tts + text + 2 word tokens + audio
|
||||
|
||||
def test_empty_text_rejected(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||
HiggsAudioV3TokenizerAdapter,
|
||||
)
|
||||
|
||||
tok = self._make_mock_tokenizer()
|
||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||
with pytest.raises(ValueError, match="non-empty"):
|
||||
adapter.build_prompt("")
|
||||
|
||||
def test_whitespace_only_rejected(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||
HiggsAudioV3TokenizerAdapter,
|
||||
)
|
||||
|
||||
tok = self._make_mock_tokenizer()
|
||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||
with pytest.raises(ValueError, match="non-empty"):
|
||||
adapter.build_prompt(" ")
|
||||
|
||||
def test_missing_specials_rejected(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||
HiggsAudioV3TokenizerAdapter,
|
||||
)
|
||||
|
||||
class BadTokenizer:
|
||||
def get_added_vocab(self):
|
||||
return {"<|tts|>": 1} # Missing <|text|> and <|audio|>
|
||||
|
||||
with pytest.raises(ValueError, match="missing"):
|
||||
HiggsAudioV3TokenizerAdapter(BadTokenizer())
|
||||
|
||||
def test_no_voice_clone_tokens_in_plain_tts(self):
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_tokenizer import (
|
||||
HiggsAudioV3TokenizerAdapter,
|
||||
)
|
||||
|
||||
tok = self._make_mock_tokenizer()
|
||||
adapter = HiggsAudioV3TokenizerAdapter(tok)
|
||||
ids = adapter.build_prompt("Hello")
|
||||
# Should not contain ref_audio or ref_text token IDs
|
||||
assert 151703 not in ids # <|ref_audio|>
|
||||
assert 151704 not in ids # <|ref_text|>
|
||||
Reference in new issue
Block a user