[Model] Add single-stage Qwen3-TTS with in-Talker streaming codec decode (#8259)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
43 files changed
+3565
-90
No files matched your search
@@ -114,10 +114,10 @@ def _raw(*, first=False, required=False, terminal=False, samples=(1, 1, 1, 1)):
|
||||
)
|
||||
|
||||
|
||||
async def _codec_output(obj, raw):
|
||||
replica = obj.stage_pools[1].replica_id
|
||||
processed = await obj._process_llm_stage_outputs(1, replica, raw, set())
|
||||
await obj._handle_processed_outputs(1, replica, processed)
|
||||
async def _codec_output(obj, raw, stage=1):
|
||||
replica = obj.stage_pools[stage].replica_id
|
||||
processed = await obj._process_llm_stage_outputs(stage, replica, raw, set())
|
||||
await obj._handle_processed_outputs(stage, replica, processed)
|
||||
|
||||
|
||||
def _messages(obj):
|
||||
@@ -132,6 +132,57 @@ def _audio(output):
|
||||
return torch.cat(audio) if isinstance(audio, list) else audio
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", list(RequestOutputKind))
|
||||
@pytest.mark.parametrize("suffix_overtook", [False, True])
|
||||
async def test_single_stage_first_audio_obeys_output_kind_and_terminal_order(kind, suffix_overtook):
|
||||
obj = _orchestrator(kind, final_stage_id=0)
|
||||
obj.stage_pools = [obj.stage_pools[1]]
|
||||
suffix = _raw(required=True, samples=(20, 21))
|
||||
terminal = _raw(required=True, terminal=True, samples=(30, 31))
|
||||
if suffix_overtook:
|
||||
await _codec_output(obj, suffix, stage=0)
|
||||
await _codec_output(obj, terminal, stage=0)
|
||||
assert obj.output_async_queue.empty()
|
||||
await obj._route_upstream_first_audio(0, 2, _raw(first=True, samples=(10, 11)))
|
||||
if not suffix_overtook:
|
||||
await _codec_output(obj, suffix, stage=0)
|
||||
await _codec_output(obj, terminal, stage=0)
|
||||
messages = _messages(obj)
|
||||
expected = {
|
||||
RequestOutputKind.DELTA: [[10, 11], [20, 21], [30, 31]],
|
||||
RequestOutputKind.CUMULATIVE: [[10, 11], [10, 11, 20, 21], [10, 11, 20, 21, 30, 31]],
|
||||
RequestOutputKind.FINAL_ONLY: [[10, 11, 20, 21, 30, 31]],
|
||||
}
|
||||
assert [_audio(message.engine_outputs).tolist() for message in messages] == expected[kind]
|
||||
assert all((message.stage_id, message.replica_id) == (0, 2) for message in messages)
|
||||
assert messages[-1].finished
|
||||
assert not obj.request_states["r"].pending_first_audio_outputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_stage_cancellation_discards_overtaking_terminal_and_late_first_audio():
|
||||
obj = _orchestrator(final_stage_id=0)
|
||||
obj.stage_pools = [obj.stage_pools[1]]
|
||||
await _codec_output(obj, _raw(required=True, terminal=True), stage=0)
|
||||
assert obj.request_states["r"].pending_first_audio_outputs
|
||||
await obj._cleanup_request_ids(["r"], abort=True)
|
||||
await obj._route_upstream_first_audio(0, 2, _raw(first=True))
|
||||
assert not obj.request_states and obj.output_async_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_stage_first_audio_processing_failure_releases_waiting_terminal():
|
||||
obj = _orchestrator(final_stage_id=0)
|
||||
obj.stage_pools = [obj.stage_pools[1]]
|
||||
await _codec_output(obj, _raw(required=True, terminal=True), stage=0)
|
||||
obj.stage_pools[0].process_llm_raw_outputs = AsyncMock(side_effect=RuntimeError("processing failed"))
|
||||
await obj._route_upstream_first_audio(0, 2, _raw(first=True))
|
||||
error = obj.output_async_queue.get_nowait()
|
||||
assert isinstance(error, ErrorMessage) and error.stage_id == 0
|
||||
assert not obj.request_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("kind", list(RequestOutputKind))
|
||||
@pytest.mark.parametrize("codec_overtook", [False, True])
|
||||
|
||||
@@ -1,9 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from vllm.v1.engine.core import EngineCoreProc
|
||||
from vllm.v1.executor.uniproc_executor import UniProcExecutor
|
||||
|
||||
from vllm_omni.engine.stage_engine_core_proc import StageEngineCoreProc
|
||||
from vllm_omni.engine.stage_engine_core_proc import StageEngineCoreProc, _bind_first_audio_sink
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"first_decoder,stream_decoder,stream_first_audio,expected",
|
||||
[(True, False, False, True), (False, True, False, False), (False, True, True, True), (False, False, True, False)],
|
||||
)
|
||||
def test_first_audio_sink_preserves_first_decoder_and_opts_in_stream_decoder(
|
||||
mocker,
|
||||
first_decoder,
|
||||
stream_decoder,
|
||||
stream_first_audio,
|
||||
expected,
|
||||
):
|
||||
executor = UniProcExecutor.__new__(UniProcExecutor)
|
||||
executor.driver_worker = mocker.Mock()
|
||||
runner = executor.driver_worker.worker.model_runner
|
||||
runner.model.first_frame_decoder = object() if first_decoder else None
|
||||
runner.model.stream_decoder = object() if stream_decoder else None
|
||||
runner.model.stream_first_audio = stream_first_audio
|
||||
assert _bind_first_audio_sink(executor, mocker.Mock(), mocker.Mock()) is expected
|
||||
assert runner.model_state.set_first_audio_sink.call_count == int(expected)
|
||||
|
||||
|
||||
def test_preprocess_add_request_preserves_omni_fields():
|
||||
|
||||
Reference in new issue
Block a user