[Bugfix][Higgs-Audio-V3] Propagate Sampling Parameters to Sampler (#5406)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
This commit is contained in:
1 parent
af2f84582b
commit
76e4a78afc
4 files changed
+215
-21
No files matched your search
@@ -250,6 +250,7 @@ class TestSamplerMethods:
|
||||
|
||||
t = FakeTalker()
|
||||
t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
|
||||
t._expand_audio_sampling_params = mod.HiggsAudioV3TalkerForConditionalGeneration._expand_audio_sampling_params
|
||||
return t
|
||||
|
||||
def _make_batched_sampler_talker(self, num_rows=4):
|
||||
@@ -300,16 +301,30 @@ class TestSamplerMethods:
|
||||
"_audio_seed_mask_from_step_input",
|
||||
):
|
||||
setattr(t, name, getattr(cls, name).__get__(t))
|
||||
t._expand_audio_sampling_params = cls._expand_audio_sampling_params
|
||||
t._device_cache_key = cls._device_cache_key
|
||||
return t
|
||||
|
||||
@staticmethod
|
||||
def _sampling_metadata(temperature=1.0, top_k=50, top_p=0.95):
|
||||
return type(
|
||||
"SamplingMetadata",
|
||||
(),
|
||||
{
|
||||
"temperature": temperature,
|
||||
"top_k": top_k,
|
||||
"top_p": top_p,
|
||||
"all_greedy": temperature is None,
|
||||
},
|
||||
)()
|
||||
|
||||
def test_sample_respects_mask(self):
|
||||
"""Tokens masked to -inf must never be sampled."""
|
||||
t = self._make_batched_sampler_talker(num_rows=1)
|
||||
logits = torch.full((10, 1026), float("-inf"))
|
||||
# Only token 500 is allowed for each row
|
||||
logits[:, 500] = 0.0
|
||||
result = t._sample_audio_codes(logits)
|
||||
result = t._sample_audio_codes(logits, self._sampling_metadata(), num_codebooks=1)
|
||||
assert result.shape == (10,)
|
||||
assert (result == 500).all(), f"Expected all 500, got {result.tolist()}"
|
||||
|
||||
@@ -320,10 +335,76 @@ class TestSamplerMethods:
|
||||
# Row 0: all masked
|
||||
# Row 1: only token 42 allowed
|
||||
logits[1, 42] = 0.0
|
||||
result = t._sample_audio_codes(logits)
|
||||
result = t._sample_audio_codes(logits, self._sampling_metadata(), num_codebooks=1)
|
||||
assert result.shape == (2,)
|
||||
assert result[1].item() == 42
|
||||
|
||||
def test_top_k_minus_one_disables_top_k_filtering(self, monkeypatch):
|
||||
t = self._make_minimal_talker()
|
||||
seen_probs = []
|
||||
|
||||
def fake_multinomial(probs, num_samples):
|
||||
seen_probs.append(probs.clone())
|
||||
return probs.argmax(dim=-1, keepdim=True)
|
||||
|
||||
monkeypatch.setattr(torch, "multinomial", fake_multinomial)
|
||||
|
||||
logits = torch.tensor([[4.0, 3.0, 2.0, 1.0]])
|
||||
metadata = self._sampling_metadata(
|
||||
temperature=1.0,
|
||||
top_k=-1,
|
||||
top_p=1.0,
|
||||
)
|
||||
|
||||
t._sample_audio_codes(logits, metadata, num_codebooks=1)
|
||||
|
||||
assert len(seen_probs) == 1
|
||||
assert torch.count_nonzero(seen_probs[0][0]).item() == 4
|
||||
|
||||
def test_sampling_metadata_invalid_temperature_is_sanitized(self, monkeypatch):
|
||||
t = self._make_minimal_talker()
|
||||
seen = []
|
||||
monkeypatch.setattr(
|
||||
torch, "multinomial", lambda probs, num_samples: seen.append(probs) or probs.argmax(-1, True)
|
||||
)
|
||||
logits = torch.tensor([[0.0, 2.0], [3.0, 0.0]])
|
||||
|
||||
result = t._sample_audio_codes(
|
||||
logits,
|
||||
self._sampling_metadata(torch.tensor([float("nan"), float("inf")])),
|
||||
num_codebooks=1,
|
||||
)
|
||||
|
||||
assert len(seen) == 1
|
||||
assert torch.isfinite(seen[0]).all()
|
||||
assert result.tolist() == [1, 0]
|
||||
|
||||
def test_sampling_metadata_is_expanded_per_request_and_vq(self, monkeypatch):
|
||||
t = self._make_minimal_talker()
|
||||
seen = []
|
||||
monkeypatch.setattr(
|
||||
torch, "multinomial", lambda probs, num_samples: seen.append(probs) or probs.argmax(-1, True)
|
||||
)
|
||||
cb_logits = torch.tensor([[[4.0, 3.0, 2.0, 1.0]] * 2] * 2) # [2 requests, 2 VQs, 4 vocab]
|
||||
logits = cb_logits.reshape(-1, cb_logits.shape[-1])
|
||||
metadata = self._sampling_metadata(
|
||||
temperature=torch.tensor([0.5, 1.0]),
|
||||
top_k=torch.tensor([1, 4]),
|
||||
top_p=torch.tensor([1.0, 0.8]),
|
||||
)
|
||||
|
||||
temperatures, top_ks, top_ps = t._expand_audio_sampling_params(metadata, logits, num_codebooks=2)
|
||||
t._sample_audio_codes(logits, metadata, num_codebooks=2)
|
||||
|
||||
torch.testing.assert_close(temperatures, torch.tensor([0.5, 0.5, 1.0, 1.0]))
|
||||
assert top_ks.tolist() == [1, 1, 4, 4]
|
||||
torch.testing.assert_close(top_ps, torch.tensor([1.0, 1.0, 0.8, 0.8]))
|
||||
assert len(seen) == 1
|
||||
torch.testing.assert_close(seen[0][0], seen[0][1])
|
||||
torch.testing.assert_close(seen[0][2], seen[0][3])
|
||||
assert torch.count_nonzero(seen[0][0]).item() == 1
|
||||
assert torch.count_nonzero(seen[0][2]).item() > 1
|
||||
|
||||
def test_delay_masking_forces_boc_during_delay(self):
|
||||
"""During delay phase, codebooks beyond delay_count must have only BOC allowed."""
|
||||
from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
|
||||
@@ -468,7 +549,9 @@ class TestSamplerMethods:
|
||||
t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None
|
||||
t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026)
|
||||
t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None
|
||||
t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long)
|
||||
t._sample_audio_codes = lambda logits_2d, *args, **kwargs: torch.zeros(
|
||||
int(logits_2d.shape[0]), dtype=torch.long
|
||||
)
|
||||
t._update_delay_state_batched = lambda *args, **kwargs: None
|
||||
t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t)
|
||||
|
||||
@@ -499,7 +582,9 @@ class TestSamplerMethods:
|
||||
t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None
|
||||
t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026)
|
||||
t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None
|
||||
t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long)
|
||||
t._sample_audio_codes = lambda logits_2d, *args, **kwargs: torch.zeros(
|
||||
int(logits_2d.shape[0]), dtype=torch.long
|
||||
)
|
||||
t._update_delay_state_batched = lambda *args, **kwargs: None
|
||||
t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t)
|
||||
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import asyncio
|
||||
from collections.abc import Sequence
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.worker import gpu_input_batch
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import PreparedRequest
|
||||
from vllm_omni.outputs import OmniModelRunnerOutput
|
||||
from vllm_omni.worker.gpu_ar_model_runner import (
|
||||
ExecuteModelState,
|
||||
@@ -25,6 +32,79 @@ def _make_runner(engine_output_type: str | None, downstream_req_ids: set[str]) -
|
||||
return runner
|
||||
|
||||
|
||||
def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatch):
|
||||
requested = {"temperature": 0.7, "top_p": 0.8, "top_k": 17}
|
||||
request = OpenAICreateSpeechRequest.model_validate(
|
||||
{"input": "Hello", "extra_params": requested},
|
||||
)
|
||||
|
||||
class Adapter:
|
||||
def validate(self, request):
|
||||
return None
|
||||
|
||||
async def build(self, request, sampling_params_list, has_inline_ref_audio):
|
||||
return PreparedRequest(prompt={"prompt": request.input}, tts_params={}, model_type="higgs_audio_v3")
|
||||
|
||||
engine = SimpleNamespace(
|
||||
errored=False,
|
||||
default_sampling_params_list=[SamplingParams(temperature=1.0, top_p=0.95, top_k=50)],
|
||||
generate=lambda **kwargs: kwargs["sampling_params_list"],
|
||||
)
|
||||
serving = object.__new__(OmniOpenAIServingSpeech)
|
||||
serving.engine_client = engine
|
||||
serving.model_config = SimpleNamespace(async_chunk=True)
|
||||
serving._tts_model_type = "higgs_audio_v3"
|
||||
serving._get_tts_adapter = lambda: Adapter()
|
||||
serving._track_ref_audio_artifact_warmup = lambda *args, **kwargs: None
|
||||
|
||||
_, stage_sampling_params, _ = asyncio.run(serving._prepare_speech_generation(request, request_id="speech-test"))
|
||||
stage0_params = stage_sampling_params[0]
|
||||
|
||||
monkeypatch.setattr(gpu_input_batch, "PIN_MEMORY", False)
|
||||
input_batch = InputBatch(
|
||||
max_num_reqs=1,
|
||||
max_model_len=8,
|
||||
max_num_batched_tokens=8,
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
)
|
||||
input_batch.add_request(
|
||||
CachedRequestState(
|
||||
req_id="speech-test",
|
||||
prompt_token_ids=[1],
|
||||
mm_features=[],
|
||||
sampling_params=stage0_params,
|
||||
generator=None,
|
||||
block_ids=([],),
|
||||
num_computed_tokens=0,
|
||||
output_token_ids=[],
|
||||
)
|
||||
)
|
||||
input_batch.sampling_metadata = input_batch._make_sampling_metadata()
|
||||
|
||||
received = []
|
||||
runner = object.__new__(GPUARModelRunner)
|
||||
runner.input_batch = input_batch
|
||||
runner.model = SimpleNamespace(
|
||||
prefer_model_sampler=True,
|
||||
skips_model_sampler_output_token_history=True,
|
||||
sample=lambda logits, metadata: received.append(metadata) or "model-sampler",
|
||||
)
|
||||
runner.sampler = SimpleNamespace()
|
||||
logits = torch.zeros((1, 4))
|
||||
|
||||
output = runner._sample(logits, spec_decode_metadata=None)
|
||||
|
||||
assert output == "model-sampler"
|
||||
assert len(received) == 1
|
||||
sampling_metadata = received[0]
|
||||
torch.testing.assert_close(sampling_metadata.temperature, torch.tensor([requested["temperature"]]))
|
||||
torch.testing.assert_close(sampling_metadata.top_p, torch.tensor([requested["top_p"]]))
|
||||
assert sampling_metadata.top_k.tolist() == [requested["top_k"]]
|
||||
|
||||
|
||||
def test_resolve_pooler_payload_req_ids_audio_terminal_stage_keeps_payload():
|
||||
runner = _make_runner(engine_output_type="audio", downstream_req_ids=set())
|
||||
|
||||
|
||||
Reference in new issue
Block a user