[Core][Model] Enable Qwen3-TTS MRV2 and optimize the TTS pipeline (#7781)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
103 files changed
+10232
-661
No files matched your search
@@ -311,7 +311,7 @@ def _engine_core_outputs(tag: str, timestamp: float) -> SimpleNamespace:
|
||||
return SimpleNamespace(outputs=[tag], timestamp=timestamp, scheduler_stats=None, finished_requests=None)
|
||||
|
||||
|
||||
def _terminal_engine_core_outputs(request_id: str) -> EngineCoreOutputs:
|
||||
def _terminal_engine_core_outputs(request_id: str, timestamp: float = 1.0) -> EngineCoreOutputs:
|
||||
return EngineCoreOutputs(
|
||||
outputs=[
|
||||
EngineCoreOutput(
|
||||
@@ -320,7 +320,7 @@ def _terminal_engine_core_outputs(request_id: str) -> EngineCoreOutputs:
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
timestamp=1.0,
|
||||
timestamp=timestamp,
|
||||
finished_requests={request_id},
|
||||
)
|
||||
|
||||
@@ -520,6 +520,7 @@ async def _enqueue_add_request(
|
||||
original_prompt,
|
||||
sampling_params_list,
|
||||
final_stage_id: int,
|
||||
final_output_stage_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
orchestrator_fixture.request_sync_q.put_nowait(
|
||||
StageSubmissionMessage(
|
||||
@@ -530,6 +531,7 @@ async def _enqueue_add_request(
|
||||
output_prompt_text=None,
|
||||
sampling_params_list=sampling_params_list,
|
||||
final_stage_id=final_stage_id,
|
||||
final_output_stage_ids=final_output_stage_ids,
|
||||
preprocess_ms=0.0,
|
||||
request_timestamp=time.time(),
|
||||
enqueue_ts=time.perf_counter(),
|
||||
@@ -789,6 +791,10 @@ async def test_run_async_chunk(orchestrator_factory) -> None:
|
||||
|
||||
stage1.push_engine_core_outputs(_engine_core_outputs("stage1-final", 3.0))
|
||||
|
||||
await _wait_for(
|
||||
lambda: orchestrator_fixture.orchestrator.request_states["req-async"].pending_final_output is not None
|
||||
)
|
||||
stage0.push_engine_core_outputs(_engine_core_outputs("stage0-final", 3.1))
|
||||
output_msg = await _get_output_message(orchestrator_fixture)
|
||||
|
||||
assert output_msg.request_id == "req-async"
|
||||
|
||||
@@ -259,6 +259,17 @@ def test_native_text_tpot_only_accepts_positive_finished_metric(
|
||||
assert processor.pop_native_text_metrics("r")["vllm_tpot_ms"] == expected_tpot_ms
|
||||
|
||||
|
||||
def test_suppressed_native_tokens_use_authoritative_count_even_without_stats(monkeypatch):
|
||||
monkeypatch.setattr(VLLMOutputProcessor, "_update_stats_from_output", lambda *args, **kwargs: None)
|
||||
processor = object.__new__(MultimodalOutputProcessor)
|
||||
processor._native_text_metrics_by_request = {}
|
||||
state = _make_state(RequestOutputKind.FINAL_ONLY)
|
||||
processor._update_stats_from_output(
|
||||
state, SimpleNamespace(new_token_ids=[2150], num_generation_tokens=2048), None, None
|
||||
)
|
||||
assert processor.pop_native_text_metrics("r")["num_generation_tokens"] == 2048
|
||||
|
||||
|
||||
def test_delta_drains_output_modality_per_step():
|
||||
"""DELTA drains the mm_type key (output modality) but preserves hidden-state keys."""
|
||||
s = _make_state(RequestOutputKind.DELTA)
|
||||
@@ -616,6 +627,7 @@ def test_no_detokenizer_process_outputs_returns_nonterminal_audio_chunk(monkeypa
|
||||
finish_reason=None,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
ec_transfer_params=None,
|
||||
routed_experts=None,
|
||||
num_cached_tokens=0,
|
||||
)
|
||||
@@ -631,6 +643,7 @@ def _make_mm_only_output_processor(monkeypatch):
|
||||
processor = object.__new__(MultimodalOutputProcessor)
|
||||
processor.output_modality = OutputModality.AUDIO
|
||||
processor.request_states = {"r": _make_no_detok_state(RequestOutputKind.DELTA)}
|
||||
processor.tracing_enabled = False
|
||||
monkeypatch.setattr(
|
||||
VLLMOutputProcessor,
|
||||
"process_outputs",
|
||||
@@ -660,6 +673,7 @@ def _audio_engine_output(*, is_segment_finished: bool, is_last_chunk: bool):
|
||||
finish_reason=FinishReason.STOP,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
ec_transfer_params=None,
|
||||
routed_experts=None,
|
||||
num_cached_tokens=0,
|
||||
is_segment_finished=is_segment_finished,
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for MRv2 Omni AR text detach and generation payload outputs."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.engine import OmniEngineCoreOutput
|
||||
from vllm_omni.engine.output_modality import OutputModality
|
||||
from vllm_omni.engine.output_processor import (
|
||||
MultimodalOutputProcessor,
|
||||
OmniRequestState,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _Detokenizer:
|
||||
def __init__(self):
|
||||
self.output_token_ids = []
|
||||
self.output_text = ""
|
||||
self.update_calls = []
|
||||
|
||||
def update(self, token_ids, stop):
|
||||
self.update_calls.append((list(token_ids), stop))
|
||||
self.output_token_ids.extend(token_ids)
|
||||
self.output_text += "X" * len(token_ids)
|
||||
return None
|
||||
|
||||
def get_next_output_text(self, finished, delta):
|
||||
return self.output_text
|
||||
|
||||
def num_output_tokens(self):
|
||||
return len(self.output_token_ids)
|
||||
|
||||
|
||||
def _make_processor(output_modality, detokenizer=None):
|
||||
state = OmniRequestState(
|
||||
request_id="r",
|
||||
external_req_id="r",
|
||||
parent_req=None,
|
||||
request_index=0,
|
||||
lora_request=None,
|
||||
output_kind=RequestOutputKind.FINAL_ONLY,
|
||||
prompt="prompt" if detokenizer else None,
|
||||
prompt_token_ids=[1] if detokenizer else [],
|
||||
prompt_embeds=None,
|
||||
logprobs_processor=MagicMock(logprobs=None, cumulative_logprob=None, prompt_logprobs=None)
|
||||
if detokenizer
|
||||
else None,
|
||||
detokenizer=detokenizer,
|
||||
max_tokens_param=None,
|
||||
arrival_time=0.0,
|
||||
queue=None,
|
||||
log_stats=False,
|
||||
stream_interval=1,
|
||||
)
|
||||
processor = MultimodalOutputProcessor(tokenizer=None, log_stats=False, output_modality=output_modality)
|
||||
processor.request_states["r"] = state
|
||||
processor.external_req_ids["r"].append("r")
|
||||
return processor, state
|
||||
|
||||
|
||||
def _rehydrated_output(*, pooling_output=None, finish_reason=FinishReason.STOP):
|
||||
"""Build the post-StagePool shape without violating vLLM's wire schema."""
|
||||
return SimpleNamespace(
|
||||
request_id="r",
|
||||
new_token_ids=[42],
|
||||
pooling_output=pooling_output,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
ec_transfer_params=None,
|
||||
routed_experts=None,
|
||||
trace_headers=None,
|
||||
prefill_stats=None,
|
||||
spec_decode_metrics=None,
|
||||
new_sampling_mask=None,
|
||||
num_nans_in_logits=0,
|
||||
multimodal_output=None,
|
||||
is_segment_finished=False,
|
||||
is_non_final_audio_chunk=False,
|
||||
output_type=None,
|
||||
num_generation_tokens=None,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
def test_text_tokens_are_detokenized_when_mrv2_ar_output_has_pooling_payload():
|
||||
detokenizer = _Detokenizer()
|
||||
processor, state = _make_processor(OutputModality.LATENT, detokenizer)
|
||||
|
||||
# StagePool has already rehydrated the MRv2 carrier. Use a structural
|
||||
# output here because upstream EngineCoreOutput strictly validates
|
||||
# pooling_output as torch.Tensor | None on construction.
|
||||
output = _rehydrated_output(pooling_output={"hidden": torch.ones(1, 4)})
|
||||
|
||||
processed = processor.process_outputs([output])
|
||||
|
||||
assert detokenizer.update_calls == [([42], True)]
|
||||
completion = processed.request_outputs[0].outputs[0]
|
||||
assert list(completion.token_ids) == [42]
|
||||
assert completion.text == "X"
|
||||
assert not state.mm_accumulated.is_empty
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_generation_stage_mm_delivered_only_with_terminal(streaming):
|
||||
# Non-terminal chunks accumulate; the terminal output carries the audio.
|
||||
# When the first output is already terminal it is delivered directly.
|
||||
processor, _ = _make_processor(OutputModality.AUDIO)
|
||||
audio = torch.ones(1, 320)
|
||||
sr = torch.tensor(24000, dtype=torch.int32)
|
||||
mm_output = OmniEngineCoreOutput(
|
||||
request_id="r",
|
||||
new_token_ids=[],
|
||||
multimodal_output={"model_outputs": audio, "sr": sr},
|
||||
finish_reason=None if streaming else FinishReason.STOP,
|
||||
)
|
||||
|
||||
processed = processor.process_outputs([mm_output])
|
||||
if streaming:
|
||||
assert processed.request_outputs == []
|
||||
processed = processor.process_outputs(
|
||||
[OmniEngineCoreOutput(request_id="r", new_token_ids=[], finish_reason=FinishReason.STOP)]
|
||||
)
|
||||
|
||||
assert len(processed.request_outputs) == 1
|
||||
completion = processed.request_outputs[0].outputs[0]
|
||||
assert completion.text == ""
|
||||
assert torch.equal(completion.multimodal_output["audio"], audio)
|
||||
assert completion.multimodal_output["sr"].item() == 24000
|
||||
@@ -0,0 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.request_snapshot import copy_request_snapshot
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_full_snapshot_preserves_fields_aliases_and_isolation():
|
||||
samples = [0.25, 0.5]
|
||||
source = {"prompt_token_ids": [1, 2], "additional_information": {"samples": samples}, "alias": samples}
|
||||
result = copy_request_snapshot(source)
|
||||
assert result == copy.deepcopy(source)
|
||||
assert result["alias"] is result["additional_information"]["samples"]
|
||||
result["alias"].append(0.75)
|
||||
result["prompt_token_ids"][0] = 9
|
||||
assert samples == [0.25, 0.5]
|
||||
assert source["prompt_token_ids"] == [1, 2]
|
||||
@@ -1393,7 +1393,7 @@ class TestConnectRemoteEngineCoresCoordinator:
|
||||
yield mocker.Mock()
|
||||
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.zmq_socket_ctx", return_value=fake_socket_ctx())
|
||||
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
|
||||
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
|
||||
with connect_remote_engine_cores(
|
||||
vllm_config=vllm_config,
|
||||
omni_master_server=omni_master_server,
|
||||
@@ -1428,7 +1428,7 @@ class TestConnectRemoteEngineCoresCoordinator:
|
||||
yield mocker.Mock()
|
||||
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.zmq_socket_ctx", return_value=fake_socket_ctx())
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
|
||||
with connect_remote_engine_cores(
|
||||
vllm_config=vllm_config,
|
||||
omni_master_server=omni_master_server,
|
||||
@@ -1454,7 +1454,7 @@ class TestConnectRemoteEngineCoresCoordinator:
|
||||
yield mocker.Mock()
|
||||
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.zmq_socket_ctx", return_value=fake_socket_ctx())
|
||||
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
|
||||
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
|
||||
|
||||
with connect_remote_diffusion_proc(
|
||||
omni_master_server=omni_master_server,
|
||||
@@ -1466,7 +1466,8 @@ class TestConnectRemoteEngineCoresCoordinator:
|
||||
omni_master_server.get_zmq_addresses.assert_called_once_with(7, replica_id=2)
|
||||
omni_master_server.get_allocation.assert_called_once_with(7, replica_id=2)
|
||||
mock_wait.assert_called_once()
|
||||
_, core_engines, parallel_config, *_ = mock_wait.call_args.args
|
||||
_, core_engines, parallel_config, _, _, launch = mock_wait.call_args.args
|
||||
assert launch.addresses is omni_master_server.get_zmq_addresses.return_value
|
||||
assert core_engines[0].local is False
|
||||
assert parallel_config.data_parallel_size_local == 0
|
||||
|
||||
@@ -1507,7 +1508,7 @@ class TestLaunchOmniCoreEngines:
|
||||
"vllm_omni.engine.stage_engine_startup.CoreEngineProcManager",
|
||||
return_value=local_engine_manager,
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
|
||||
with _launch_omni_core_engines(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=mocker.Mock(),
|
||||
@@ -1584,7 +1585,7 @@ class TestLaunchOmniCoreEngines:
|
||||
"vllm_omni.engine.stage_engine_startup.CoreEngineProcManager",
|
||||
return_value=mocker.Mock(),
|
||||
)
|
||||
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
|
||||
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
|
||||
with _launch_omni_core_engines(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=mocker.Mock(),
|
||||
|
||||
@@ -8,6 +8,7 @@ import sys
|
||||
import types
|
||||
from dataclasses import fields, replace
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from pydantic.fields import FieldInfo
|
||||
@@ -61,9 +62,14 @@ _DEPLOY_DIR = Path(__file__).parents[2] / "vllm_omni" / "deploy"
|
||||
def _effective_backend_values(config_cls: type, engine_args: dict) -> dict[str, object]:
|
||||
backend_fields = fields(config_cls)
|
||||
backend_field_names = {backend_field.name for backend_field in backend_fields}
|
||||
backend_config = config_cls(
|
||||
**{name: copy.deepcopy(value) for name, value in engine_args.items() if name in backend_field_names}
|
||||
)
|
||||
# This is a schema/normalization comparison, not a Hub download test.
|
||||
with (
|
||||
patch("vllm.engine.arg_utils.get_model_path", side_effect=lambda model, *_: model),
|
||||
patch("vllm_omni.diffusion.data.get_model_path", side_effect=lambda model, *_: model),
|
||||
):
|
||||
backend_config = config_cls(
|
||||
**{name: copy.deepcopy(value) for name, value in engine_args.items() if name in backend_field_names}
|
||||
)
|
||||
effective_values: dict[str, object] = {}
|
||||
for backend_field in backend_fields:
|
||||
value = getattr(backend_config, backend_field.name)
|
||||
@@ -78,6 +84,9 @@ _DIFFUSION_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniDiffusi
|
||||
_TOPOLOGY_ONLY_ENGINE_ARGS = frozenset({"inline_diffusion"})
|
||||
_OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
|
||||
{
|
||||
"final_output",
|
||||
"use_v2_model_runner",
|
||||
"supports_native_mrv2_data_plane",
|
||||
"active_stream_window",
|
||||
"codec_frame_rate_hz",
|
||||
"custom_voice_dir",
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Terminal audio must not retire a request before upstream terminal metrics."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def build():
|
||||
o = Orchestrator.__new__(Orchestrator)
|
||||
o.stage_pools = [
|
||||
SimpleNamespace(
|
||||
final_output=False, build_stage_metrics=lambda *a, **k: SimpleNamespace(stage_id=0, num_tokens_out=42)
|
||||
),
|
||||
SimpleNamespace(
|
||||
final_output=True, build_stage_metrics=lambda *a, **k: SimpleNamespace(stage_id=1, num_tokens_out=0)
|
||||
),
|
||||
]
|
||||
state = OrchestratorRequestState(
|
||||
request_id="r",
|
||||
final_stage_id=1,
|
||||
final_output_stage_ids={1},
|
||||
stage_submit_ts={0: 0, 1: 0},
|
||||
sampling_params_list=[None, None],
|
||||
)
|
||||
o.request_states = {"r": state}
|
||||
o.output_async_queue = asyncio.Queue()
|
||||
o.async_chunk = True
|
||||
o._pd_pair = None
|
||||
o._cfg_tracker = MagicMock()
|
||||
o._cfg_tracker.is_companion.return_value = False
|
||||
o._cfg_tracker.cleanup_parent.return_value = []
|
||||
o._is_duplex_session_request = lambda s: False
|
||||
o._duplex_output_decision = lambda *a: None
|
||||
o._stage_receives_async_chunks = lambda s: True
|
||||
cleaned = []
|
||||
|
||||
async def cleanup(ids):
|
||||
cleaned.extend(ids)
|
||||
for r in ids:
|
||||
o.request_states.pop(r, None)
|
||||
|
||||
o._cleanup_request_ids = cleanup
|
||||
return o, state, cleaned
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("order", [(0, 1), (1, 0)])
|
||||
async def test_terminal_metrics_precede_final_audio_for_either_arrival_order(order):
|
||||
o, state, cleaned = build()
|
||||
for stage in order:
|
||||
await o._handle_processed_outputs(stage, 0, [SimpleNamespace(request_id="r", finished=True, error=None)])
|
||||
if stage == 1 and not state.finished_stage_ids.issuperset({0, 1}):
|
||||
assert o.output_async_queue.empty()
|
||||
assert "r" in o.request_states
|
||||
assert cleaned == []
|
||||
metrics = o.output_async_queue.get_nowait()
|
||||
final = o.output_async_queue.get_nowait()
|
||||
assert type(metrics).__name__ == "StageMetricsMessage"
|
||||
assert metrics.stage_id == 0 and metrics.metrics.num_tokens_out == 42
|
||||
assert type(final).__name__ == "OutputMessage" and final.finished
|
||||
assert o.output_async_queue.empty()
|
||||
assert cleaned == ["r"]
|
||||
assert state.pending_final_output is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_pending_output_is_not_published_by_late_upstream():
|
||||
o, state, cleaned = build()
|
||||
await o._handle_processed_outputs(1, 0, [SimpleNamespace(request_id="r", finished=True, error=None)])
|
||||
assert state.pending_final_output is not None
|
||||
await o._cleanup_request_ids(["r"])
|
||||
await o._handle_processed_outputs(0, 0, [SimpleNamespace(request_id="r", finished=True, error=None)])
|
||||
assert o.output_async_queue.empty()
|
||||
assert cleaned == ["r"]
|
||||
Reference in new issue
Block a user