[New Model] Step-Audio2 (#464)

Signed-off-by: linyueqian <linyueqian@outlook.com>
Signed-off-by: wuli666 <djjpro975@gmail.com>
Co-authored-by: linyueqian <linyueqian@outlook.com>
Co-authored-by: wuli666 <djjpro975@gmail.com>
This commit is contained in:
authored and GitHub committed 2026-06-13 01:00:33 +00:00
1 parent 1733831f78
commit 9bc8705c98
26 files changed
+5347 -2

No files matched your search

@@ -0,0 +1,142 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
# Token2Wav imports the optional `step-audio2` extras (flashcosyvoice / s3tokenizer
# / hyperpyyaml) at module load. Skip instead of erroring at collection time on CI
# images that do not install them (e.g. the CPU core_model runner); a genuine
# import error (e.g. vLLM API drift) still surfaces. See
# examples/offline_inference/step_audio2/README.md.
try:
from vllm_omni.model_executor.models.step_audio2.step_audio2_token2wav import (
StepAudio2Token2WavForConditionalGeneration,
)
except ModuleNotFoundError as exc:
pytest.skip(
f"Step-Audio2 Token2Wav extras not installed ({exc.name})",
allow_module_level=True,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _DummyModelConfig:
def __init__(self) -> None:
self.hf_config = SimpleNamespace(
token2wav_path="/tmp/fake-token2wav",
token2wav_float16=False,
)
self.model = "/tmp/fake-model"
def get_hidden_size(self) -> int:
return 8
def _build_model() -> StepAudio2Token2WavForConditionalGeneration:
vllm_config = SimpleNamespace(
model_config=_DummyModelConfig(),
device_config=SimpleNamespace(device="cpu"),
)
return StepAudio2Token2WavForConditionalGeneration(vllm_config=vllm_config)
def test_step_audio2_token2wav_sync_path_not_misdetected_by_empty_runtime_info(monkeypatch):
model = _build_model()
monkeypatch.setattr("os.path.exists", lambda _: True)
monkeypatch.setattr(
model.token2wav,
"forward",
lambda *args, **kwargs: torch.zeros(1, 16, dtype=torch.float32),
)
out = model.forward(
input_ids=torch.tensor([1, 2, 3]),
positions=torch.tensor([0, 1, 2]),
runtime_additional_information=[{}],
)
audio = out.multimodal_outputs["model_outputs"]
# sync path returns the full waveform wrapped in a single-element list,
# same contract as the async-chunk path (see tests below)
assert isinstance(audio, list)
assert len(audio) == 1
assert isinstance(audio[0], torch.Tensor)
assert model._stream_states == []
def test_step_audio2_token2wav_async_chunk_batch_guard():
model = _build_model()
with pytest.raises(RuntimeError, match="only supports batch=1"):
model._forward_async_chunk(
input_ids=torch.tensor([1, 2, 3]),
runtime_additional_information=[
{"left_context_size": 0},
{"left_context_size": 1},
],
)
def test_step_audio2_token2wav_async_chunk_last_chunk_resets_state(monkeypatch):
model = _build_model()
monkeypatch.setattr("os.path.exists", lambda _: True)
calls = {"setup": 0, "stream": 0, "reset": 0}
def _setup_stream_for(prompt_wav, state):
calls["setup"] += 1
state.setup_done = True
state.stream_cache = {}
state.hift_cache_dict = {
"mel": torch.zeros(1, 1, 0),
"source": torch.zeros(1, 1, 0),
"speech": torch.zeros(1, 0),
}
def _stream_chunk_for(audio_tokens, prompt_wav, last_chunk, state):
calls["stream"] += 1
return torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32)
def _reset_stream_for(state):
calls["reset"] += 1
state.setup_done = False
state.finished = True
state.stream_cache = None
state.hift_cache_dict = {}
monkeypatch.setattr(model.token2wav, "setup_stream_for", _setup_stream_for)
monkeypatch.setattr(model.token2wav, "stream_chunk_for", _stream_chunk_for)
monkeypatch.setattr(model.token2wav, "reset_stream_for", _reset_stream_for)
out = model.forward(
input_ids=torch.tensor([10, 11, 12]),
positions=torch.tensor([0, 1, 2]),
runtime_additional_information=[{"left_context_size": 1}],
)
audio_list = out.multimodal_outputs["model_outputs"]
assert isinstance(audio_list, list)
assert len(audio_list) == 1
assert torch.allclose(audio_list[0], torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32))
assert calls == {"setup": 1, "stream": 1, "reset": 1}
def test_step_audio2_token2wav_async_chunk_empty_eof_returns_zero_chunk():
model = _build_model()
out = model._forward_async_chunk(
input_ids=torch.tensor([], dtype=torch.int64),
runtime_additional_information=[{"left_context_size": 1}],
)
audio_list = out.multimodal_outputs["model_outputs"]
assert isinstance(audio_list, list)
assert len(audio_list) == 1
assert torch.equal(audio_list[0], torch.zeros(1, dtype=torch.float32))
assert model._stream_states
assert model._stream_states[0].finished is True
@@ -0,0 +1,132 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.models.step_audio2.step_audio2_constants import (
DEFAULT_STREAM_CONFIG,
DEFAULT_TOKEN_CONFIG,
)
from vllm_omni.model_executor.stage_input_processors.step_audio2 import (
thinker2token2wav_async_chunk,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _req(external_req_id: str, *, prompt_token_ids: list[int], all_token_ids: list[int], finished: bool):
return SimpleNamespace(
external_req_id=external_req_id,
prompt_token_ids=prompt_token_ids,
all_token_ids=all_token_ids,
is_finished=lambda: finished,
)
def test_step_audio2_async_chunk_uses_decode_only_tokens_not_prompt_history():
audio_start = DEFAULT_TOKEN_CONFIG.audio_start
audio_eos = DEFAULT_TOKEN_CONFIG.audio_eos
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
prompt = [11, 12, audio_start + 5] # historical prompt audio token
generated = [99, audio_start + 1, audio_start + 2, audio_start + audio_eos]
request = _req(
"rid-decode-only",
prompt_token_ids=prompt,
all_token_ids=prompt + generated,
finished=True,
)
payload = thinker2token2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output=None,
request=request,
)
assert payload is not None
assert payload["code_predictor_codes"] == [1, 2]
assert payload["left_context_size"] == 1
assert payload["finished"].item() is True
def test_step_audio2_async_chunk_returns_none_when_not_enough_tokens():
audio_start = DEFAULT_TOKEN_CONFIG.audio_start
required = DEFAULT_STREAM_CONFIG.chunk_size + DEFAULT_STREAM_CONFIG.pre_lookahead_len
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
generated = [audio_start + i for i in range(required - 1)]
request = _req(
"rid-not-ready",
prompt_token_ids=[1, 2, 3],
all_token_ids=[1, 2, 3] + generated,
finished=False,
)
payload = thinker2token2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output=None,
request=request,
)
assert payload is None
def test_step_audio2_async_chunk_emits_non_last_chunk_and_advances_consumed_by_chunk_size():
audio_start = DEFAULT_TOKEN_CONFIG.audio_start
chunk_size = DEFAULT_STREAM_CONFIG.chunk_size
required = chunk_size + DEFAULT_STREAM_CONFIG.pre_lookahead_len
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
generated = [audio_start + i for i in range(required + 10)]
request = _req(
"rid-ready",
prompt_token_ids=[7, 8],
all_token_ids=[7, 8] + generated,
finished=False,
)
payload = thinker2token2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output=None,
request=request,
)
assert payload is not None
assert payload["left_context_size"] == 0
assert payload["finished"].item() is False
assert payload["code_predictor_codes"] == list(range(required))
assert transfer_manager.code_prompt_token_ids["rid-ready"] == list(range(chunk_size))
def test_step_audio2_async_chunk_emits_eof_when_finished_with_no_remaining_audio():
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
transfer_manager.code_prompt_token_ids["rid-eof"] = [1, 2, 3]
request = _req(
"rid-eof",
prompt_token_ids=[10, 11],
all_token_ids=[
10,
11,
DEFAULT_TOKEN_CONFIG.audio_start + 1,
DEFAULT_TOKEN_CONFIG.audio_start + 2,
DEFAULT_TOKEN_CONFIG.audio_start + 3,
],
finished=True,
)
payload = thinker2token2wav_async_chunk(
transfer_manager=transfer_manager,
pooling_output=None,
request=request,
)
assert payload == {
"code_predictor_codes": [],
"left_context_size": 1,
"finished": torch.tensor(True, dtype=torch.bool),
}