[Bugfix][Higgs-Audio-V3] Propagate Sampling Parameters to Sampler (#5406)

Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
This commit is contained in:
boatman authored and GitHub committed 2026-07-25 15:25:35 +00:00
1 parent af2f84582b
commit 76e4a78afc
4 files changed
+215 -21

No files matched your search

@@ -250,6 +250,7 @@ class TestSamplerMethods:
t = FakeTalker()
t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
t._expand_audio_sampling_params = mod.HiggsAudioV3TalkerForConditionalGeneration._expand_audio_sampling_params
return t
def _make_batched_sampler_talker(self, num_rows=4):
@@ -300,16 +301,30 @@ class TestSamplerMethods:
"_audio_seed_mask_from_step_input",
):
setattr(t, name, getattr(cls, name).__get__(t))
t._expand_audio_sampling_params = cls._expand_audio_sampling_params
t._device_cache_key = cls._device_cache_key
return t
@staticmethod
def _sampling_metadata(temperature=1.0, top_k=50, top_p=0.95):
return type(
"SamplingMetadata",
(),
{
"temperature": temperature,
"top_k": top_k,
"top_p": top_p,
"all_greedy": temperature is None,
},
)()
def test_sample_respects_mask(self):
"""Tokens masked to -inf must never be sampled."""
t = self._make_batched_sampler_talker(num_rows=1)
logits = torch.full((10, 1026), float("-inf"))
# Only token 500 is allowed for each row
logits[:, 500] = 0.0
result = t._sample_audio_codes(logits)
result = t._sample_audio_codes(logits, self._sampling_metadata(), num_codebooks=1)
assert result.shape == (10,)
assert (result == 500).all(), f"Expected all 500, got {result.tolist()}"
@@ -320,10 +335,76 @@ class TestSamplerMethods:
# Row 0: all masked
# Row 1: only token 42 allowed
logits[1, 42] = 0.0
result = t._sample_audio_codes(logits)
result = t._sample_audio_codes(logits, self._sampling_metadata(), num_codebooks=1)
assert result.shape == (2,)
assert result[1].item() == 42
def test_top_k_minus_one_disables_top_k_filtering(self, monkeypatch):
t = self._make_minimal_talker()
seen_probs = []
def fake_multinomial(probs, num_samples):
seen_probs.append(probs.clone())
return probs.argmax(dim=-1, keepdim=True)
monkeypatch.setattr(torch, "multinomial", fake_multinomial)
logits = torch.tensor([[4.0, 3.0, 2.0, 1.0]])
metadata = self._sampling_metadata(
temperature=1.0,
top_k=-1,
top_p=1.0,
)
t._sample_audio_codes(logits, metadata, num_codebooks=1)
assert len(seen_probs) == 1
assert torch.count_nonzero(seen_probs[0][0]).item() == 4
def test_sampling_metadata_invalid_temperature_is_sanitized(self, monkeypatch):
t = self._make_minimal_talker()
seen = []
monkeypatch.setattr(
torch, "multinomial", lambda probs, num_samples: seen.append(probs) or probs.argmax(-1, True)
)
logits = torch.tensor([[0.0, 2.0], [3.0, 0.0]])
result = t._sample_audio_codes(
logits,
self._sampling_metadata(torch.tensor([float("nan"), float("inf")])),
num_codebooks=1,
)
assert len(seen) == 1
assert torch.isfinite(seen[0]).all()
assert result.tolist() == [1, 0]
def test_sampling_metadata_is_expanded_per_request_and_vq(self, monkeypatch):
t = self._make_minimal_talker()
seen = []
monkeypatch.setattr(
torch, "multinomial", lambda probs, num_samples: seen.append(probs) or probs.argmax(-1, True)
)
cb_logits = torch.tensor([[[4.0, 3.0, 2.0, 1.0]] * 2] * 2) # [2 requests, 2 VQs, 4 vocab]
logits = cb_logits.reshape(-1, cb_logits.shape[-1])
metadata = self._sampling_metadata(
temperature=torch.tensor([0.5, 1.0]),
top_k=torch.tensor([1, 4]),
top_p=torch.tensor([1.0, 0.8]),
)
temperatures, top_ks, top_ps = t._expand_audio_sampling_params(metadata, logits, num_codebooks=2)
t._sample_audio_codes(logits, metadata, num_codebooks=2)
torch.testing.assert_close(temperatures, torch.tensor([0.5, 0.5, 1.0, 1.0]))
assert top_ks.tolist() == [1, 1, 4, 4]
torch.testing.assert_close(top_ps, torch.tensor([1.0, 1.0, 0.8, 0.8]))
assert len(seen) == 1
torch.testing.assert_close(seen[0][0], seen[0][1])
torch.testing.assert_close(seen[0][2], seen[0][3])
assert torch.count_nonzero(seen[0][0]).item() == 1
assert torch.count_nonzero(seen[0][2]).item() > 1
def test_delay_masking_forces_boc_during_delay(self):
"""During delay phase, codebooks beyond delay_count must have only BOC allowed."""
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
@@ -468,7 +549,9 @@ class TestSamplerMethods:
t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None
t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026)
t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None
t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long)
t._sample_audio_codes = lambda logits_2d, *args, **kwargs: torch.zeros(
int(logits_2d.shape[0]), dtype=torch.long
)
t._update_delay_state_batched = lambda *args, **kwargs: None
t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t)
@@ -499,7 +582,9 @@ class TestSamplerMethods:
t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None
t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026)
t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None
t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long)
t._sample_audio_codes = lambda logits_2d, *args, **kwargs: torch.zeros(
int(logits_2d.shape[0]), dtype=torch.long
)
t._update_delay_state_batched = lambda *args, **kwargs: None
t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t)
+80
View File
@@ -1,10 +1,17 @@
import asyncio
from collections.abc import Sequence
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from vllm.sampling_params import SamplingParams
from vllm.v1.worker import gpu_input_batch
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest
from vllm_omni.outputs import OmniModelRunnerOutput
from vllm_omni.worker.gpu_ar_model_runner import (
ExecuteModelState,
@@ -25,6 +32,79 @@ def _make_runner(engine_output_type: str | None, downstream_req_ids: set[str]) -
return runner
def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatch):
requested = {"temperature": 0.7, "top_p": 0.8, "top_k": 17}
request = OpenAICreateSpeechRequest.model_validate(
{"input": "Hello", "extra_params": requested},
)
class Adapter:
def validate(self, request):
return None
async def build(self, request, sampling_params_list, has_inline_ref_audio):
return PreparedRequest(prompt={"prompt": request.input}, tts_params={}, model_type="higgs_audio_v3")
engine = SimpleNamespace(
errored=False,
default_sampling_params_list=[SamplingParams(temperature=1.0, top_p=0.95, top_k=50)],
generate=lambda **kwargs: kwargs["sampling_params_list"],
)
serving = object.__new__(OmniOpenAIServingSpeech)
serving.engine_client = engine
serving.model_config = SimpleNamespace(async_chunk=True)
serving._tts_model_type = "higgs_audio_v3"
serving._get_tts_adapter = lambda: Adapter()
serving._track_ref_audio_artifact_warmup = lambda *args, **kwargs: None
_, stage_sampling_params, _ = asyncio.run(serving._prepare_speech_generation(request, request_id="speech-test"))
stage0_params = stage_sampling_params[0]
monkeypatch.setattr(gpu_input_batch, "PIN_MEMORY", False)
input_batch = InputBatch(
max_num_reqs=1,
max_model_len=8,
max_num_batched_tokens=8,
device=torch.device("cpu"),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
)
input_batch.add_request(
CachedRequestState(
req_id="speech-test",
prompt_token_ids=[1],
mm_features=[],
sampling_params=stage0_params,
generator=None,
block_ids=([],),
num_computed_tokens=0,
output_token_ids=[],
)
)
input_batch.sampling_metadata = input_batch._make_sampling_metadata()
received = []
runner = object.__new__(GPUARModelRunner)
runner.input_batch = input_batch
runner.model = SimpleNamespace(
prefer_model_sampler=True,
skips_model_sampler_output_token_history=True,
sample=lambda logits, metadata: received.append(metadata) or "model-sampler",
)
runner.sampler = SimpleNamespace()
logits = torch.zeros((1, 4))
output = runner._sample(logits, spec_decode_metadata=None)
assert output == "model-sampler"
assert len(received) == 1
sampling_metadata = received[0]
torch.testing.assert_close(sampling_metadata.temperature, torch.tensor([requested["temperature"]]))
torch.testing.assert_close(sampling_metadata.top_p, torch.tensor([requested["top_p"]]))
assert sampling_metadata.top_k.tolist() == [requested["top_k"]]
def test_resolve_pooler_payload_req_ids_audio_terminal_stage_keeps_payload():
runner = _make_runner(engine_output_type="audio", downstream_req_ids=set())