[New Model] Step-Audio2 (#464)
Signed-off-by: linyueqian <linyueqian@outlook.com> Signed-off-by: wuli666 <djjpro975@gmail.com> Co-authored-by: linyueqian <linyueqian@outlook.com> Co-authored-by: wuli666 <djjpro975@gmail.com>
This commit is contained in:
26 files changed
+5347
-2
No files matched your search
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# Token2Wav imports the optional `step-audio2` extras (flashcosyvoice / s3tokenizer
|
||||
# / hyperpyyaml) at module load. Skip instead of erroring at collection time on CI
|
||||
# images that do not install them (e.g. the CPU core_model runner); a genuine
|
||||
# import error (e.g. vLLM API drift) still surfaces. See
|
||||
# examples/offline_inference/step_audio2/README.md.
|
||||
try:
|
||||
from vllm_omni.model_executor.models.step_audio2.step_audio2_token2wav import (
|
||||
StepAudio2Token2WavForConditionalGeneration,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
pytest.skip(
|
||||
f"Step-Audio2 Token2Wav extras not installed ({exc.name})",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _DummyModelConfig:
|
||||
def __init__(self) -> None:
|
||||
self.hf_config = SimpleNamespace(
|
||||
token2wav_path="/tmp/fake-token2wav",
|
||||
token2wav_float16=False,
|
||||
)
|
||||
self.model = "/tmp/fake-model"
|
||||
|
||||
def get_hidden_size(self) -> int:
|
||||
return 8
|
||||
|
||||
|
||||
def _build_model() -> StepAudio2Token2WavForConditionalGeneration:
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=_DummyModelConfig(),
|
||||
device_config=SimpleNamespace(device="cpu"),
|
||||
)
|
||||
return StepAudio2Token2WavForConditionalGeneration(vllm_config=vllm_config)
|
||||
|
||||
|
||||
def test_step_audio2_token2wav_sync_path_not_misdetected_by_empty_runtime_info(monkeypatch):
|
||||
model = _build_model()
|
||||
|
||||
monkeypatch.setattr("os.path.exists", lambda _: True)
|
||||
monkeypatch.setattr(
|
||||
model.token2wav,
|
||||
"forward",
|
||||
lambda *args, **kwargs: torch.zeros(1, 16, dtype=torch.float32),
|
||||
)
|
||||
|
||||
out = model.forward(
|
||||
input_ids=torch.tensor([1, 2, 3]),
|
||||
positions=torch.tensor([0, 1, 2]),
|
||||
runtime_additional_information=[{}],
|
||||
)
|
||||
|
||||
audio = out.multimodal_outputs["model_outputs"]
|
||||
# sync path returns the full waveform wrapped in a single-element list,
|
||||
# same contract as the async-chunk path (see tests below)
|
||||
assert isinstance(audio, list)
|
||||
assert len(audio) == 1
|
||||
assert isinstance(audio[0], torch.Tensor)
|
||||
assert model._stream_states == []
|
||||
|
||||
|
||||
def test_step_audio2_token2wav_async_chunk_batch_guard():
|
||||
model = _build_model()
|
||||
|
||||
with pytest.raises(RuntimeError, match="only supports batch=1"):
|
||||
model._forward_async_chunk(
|
||||
input_ids=torch.tensor([1, 2, 3]),
|
||||
runtime_additional_information=[
|
||||
{"left_context_size": 0},
|
||||
{"left_context_size": 1},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_step_audio2_token2wav_async_chunk_last_chunk_resets_state(monkeypatch):
|
||||
model = _build_model()
|
||||
|
||||
monkeypatch.setattr("os.path.exists", lambda _: True)
|
||||
calls = {"setup": 0, "stream": 0, "reset": 0}
|
||||
|
||||
def _setup_stream_for(prompt_wav, state):
|
||||
calls["setup"] += 1
|
||||
state.setup_done = True
|
||||
state.stream_cache = {}
|
||||
state.hift_cache_dict = {
|
||||
"mel": torch.zeros(1, 1, 0),
|
||||
"source": torch.zeros(1, 1, 0),
|
||||
"speech": torch.zeros(1, 0),
|
||||
}
|
||||
|
||||
def _stream_chunk_for(audio_tokens, prompt_wav, last_chunk, state):
|
||||
calls["stream"] += 1
|
||||
return torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32)
|
||||
|
||||
def _reset_stream_for(state):
|
||||
calls["reset"] += 1
|
||||
state.setup_done = False
|
||||
state.finished = True
|
||||
state.stream_cache = None
|
||||
state.hift_cache_dict = {}
|
||||
|
||||
monkeypatch.setattr(model.token2wav, "setup_stream_for", _setup_stream_for)
|
||||
monkeypatch.setattr(model.token2wav, "stream_chunk_for", _stream_chunk_for)
|
||||
monkeypatch.setattr(model.token2wav, "reset_stream_for", _reset_stream_for)
|
||||
|
||||
out = model.forward(
|
||||
input_ids=torch.tensor([10, 11, 12]),
|
||||
positions=torch.tensor([0, 1, 2]),
|
||||
runtime_additional_information=[{"left_context_size": 1}],
|
||||
)
|
||||
|
||||
audio_list = out.multimodal_outputs["model_outputs"]
|
||||
assert isinstance(audio_list, list)
|
||||
assert len(audio_list) == 1
|
||||
assert torch.allclose(audio_list[0], torch.tensor([0.1, 0.2, 0.3], dtype=torch.float32))
|
||||
assert calls == {"setup": 1, "stream": 1, "reset": 1}
|
||||
|
||||
|
||||
def test_step_audio2_token2wav_async_chunk_empty_eof_returns_zero_chunk():
|
||||
model = _build_model()
|
||||
|
||||
out = model._forward_async_chunk(
|
||||
input_ids=torch.tensor([], dtype=torch.int64),
|
||||
runtime_additional_information=[{"left_context_size": 1}],
|
||||
)
|
||||
|
||||
audio_list = out.multimodal_outputs["model_outputs"]
|
||||
assert isinstance(audio_list, list)
|
||||
assert len(audio_list) == 1
|
||||
assert torch.equal(audio_list[0], torch.zeros(1, dtype=torch.float32))
|
||||
assert model._stream_states
|
||||
assert model._stream_states[0].finished is True
|
||||
@@ -0,0 +1,132 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.step_audio2.step_audio2_constants import (
|
||||
DEFAULT_STREAM_CONFIG,
|
||||
DEFAULT_TOKEN_CONFIG,
|
||||
)
|
||||
from vllm_omni.model_executor.stage_input_processors.step_audio2 import (
|
||||
thinker2token2wav_async_chunk,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _req(external_req_id: str, *, prompt_token_ids: list[int], all_token_ids: list[int], finished: bool):
|
||||
return SimpleNamespace(
|
||||
external_req_id=external_req_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
all_token_ids=all_token_ids,
|
||||
is_finished=lambda: finished,
|
||||
)
|
||||
|
||||
|
||||
def test_step_audio2_async_chunk_uses_decode_only_tokens_not_prompt_history():
|
||||
audio_start = DEFAULT_TOKEN_CONFIG.audio_start
|
||||
audio_eos = DEFAULT_TOKEN_CONFIG.audio_eos
|
||||
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
|
||||
|
||||
prompt = [11, 12, audio_start + 5] # historical prompt audio token
|
||||
generated = [99, audio_start + 1, audio_start + 2, audio_start + audio_eos]
|
||||
request = _req(
|
||||
"rid-decode-only",
|
||||
prompt_token_ids=prompt,
|
||||
all_token_ids=prompt + generated,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
payload = thinker2token2wav_async_chunk(
|
||||
transfer_manager=transfer_manager,
|
||||
pooling_output=None,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["code_predictor_codes"] == [1, 2]
|
||||
assert payload["left_context_size"] == 1
|
||||
assert payload["finished"].item() is True
|
||||
|
||||
|
||||
def test_step_audio2_async_chunk_returns_none_when_not_enough_tokens():
|
||||
audio_start = DEFAULT_TOKEN_CONFIG.audio_start
|
||||
required = DEFAULT_STREAM_CONFIG.chunk_size + DEFAULT_STREAM_CONFIG.pre_lookahead_len
|
||||
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
|
||||
|
||||
generated = [audio_start + i for i in range(required - 1)]
|
||||
request = _req(
|
||||
"rid-not-ready",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
all_token_ids=[1, 2, 3] + generated,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
payload = thinker2token2wav_async_chunk(
|
||||
transfer_manager=transfer_manager,
|
||||
pooling_output=None,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert payload is None
|
||||
|
||||
|
||||
def test_step_audio2_async_chunk_emits_non_last_chunk_and_advances_consumed_by_chunk_size():
|
||||
audio_start = DEFAULT_TOKEN_CONFIG.audio_start
|
||||
chunk_size = DEFAULT_STREAM_CONFIG.chunk_size
|
||||
required = chunk_size + DEFAULT_STREAM_CONFIG.pre_lookahead_len
|
||||
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
|
||||
|
||||
generated = [audio_start + i for i in range(required + 10)]
|
||||
request = _req(
|
||||
"rid-ready",
|
||||
prompt_token_ids=[7, 8],
|
||||
all_token_ids=[7, 8] + generated,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
payload = thinker2token2wav_async_chunk(
|
||||
transfer_manager=transfer_manager,
|
||||
pooling_output=None,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["left_context_size"] == 0
|
||||
assert payload["finished"].item() is False
|
||||
assert payload["code_predictor_codes"] == list(range(required))
|
||||
assert transfer_manager.code_prompt_token_ids["rid-ready"] == list(range(chunk_size))
|
||||
|
||||
|
||||
def test_step_audio2_async_chunk_emits_eof_when_finished_with_no_remaining_audio():
|
||||
transfer_manager = SimpleNamespace(code_prompt_token_ids=defaultdict(list))
|
||||
transfer_manager.code_prompt_token_ids["rid-eof"] = [1, 2, 3]
|
||||
|
||||
request = _req(
|
||||
"rid-eof",
|
||||
prompt_token_ids=[10, 11],
|
||||
all_token_ids=[
|
||||
10,
|
||||
11,
|
||||
DEFAULT_TOKEN_CONFIG.audio_start + 1,
|
||||
DEFAULT_TOKEN_CONFIG.audio_start + 2,
|
||||
DEFAULT_TOKEN_CONFIG.audio_start + 3,
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
payload = thinker2token2wav_async_chunk(
|
||||
transfer_manager=transfer_manager,
|
||||
pooling_output=None,
|
||||
request=request,
|
||||
)
|
||||
|
||||
assert payload == {
|
||||
"code_predictor_codes": [],
|
||||
"left_context_size": 1,
|
||||
"finished": torch.tensor(True, dtype=torch.bool),
|
||||
}
|
||||
Reference in new issue
Block a user