[Core][Model] Enable Qwen3-TTS MRV2 and optimize the TTS pipeline (#7781)

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 authored and GitHub committed 2026-09-20 23:03:08 -07:00
1 parent d139c600c7
commit a9e2117f5e
103 files changed
+10232 -661

No files matched your search

+8 -2
View File
@@ -311,7 +311,7 @@ def _engine_core_outputs(tag: str, timestamp: float) -> SimpleNamespace:
return SimpleNamespace(outputs=[tag], timestamp=timestamp, scheduler_stats=None, finished_requests=None)
def _terminal_engine_core_outputs(request_id: str) -> EngineCoreOutputs:
def _terminal_engine_core_outputs(request_id: str, timestamp: float = 1.0) -> EngineCoreOutputs:
return EngineCoreOutputs(
outputs=[
EngineCoreOutput(
@@ -320,7 +320,7 @@ def _terminal_engine_core_outputs(request_id: str) -> EngineCoreOutputs:
finish_reason=FinishReason.STOP,
)
],
timestamp=1.0,
timestamp=timestamp,
finished_requests={request_id},
)
@@ -520,6 +520,7 @@ async def _enqueue_add_request(
original_prompt,
sampling_params_list,
final_stage_id: int,
final_output_stage_ids: list[int] | None = None,
) -> None:
orchestrator_fixture.request_sync_q.put_nowait(
StageSubmissionMessage(
@@ -530,6 +531,7 @@ async def _enqueue_add_request(
output_prompt_text=None,
sampling_params_list=sampling_params_list,
final_stage_id=final_stage_id,
final_output_stage_ids=final_output_stage_ids,
preprocess_ms=0.0,
request_timestamp=time.time(),
enqueue_ts=time.perf_counter(),
@@ -789,6 +791,10 @@ async def test_run_async_chunk(orchestrator_factory) -> None:
stage1.push_engine_core_outputs(_engine_core_outputs("stage1-final", 3.0))
await _wait_for(
lambda: orchestrator_fixture.orchestrator.request_states["req-async"].pending_final_output is not None
)
stage0.push_engine_core_outputs(_engine_core_outputs("stage0-final", 3.1))
output_msg = await _get_output_message(orchestrator_fixture)
assert output_msg.request_id == "req-async"
+14
View File
@@ -259,6 +259,17 @@ def test_native_text_tpot_only_accepts_positive_finished_metric(
assert processor.pop_native_text_metrics("r")["vllm_tpot_ms"] == expected_tpot_ms
def test_suppressed_native_tokens_use_authoritative_count_even_without_stats(monkeypatch):
monkeypatch.setattr(VLLMOutputProcessor, "_update_stats_from_output", lambda *args, **kwargs: None)
processor = object.__new__(MultimodalOutputProcessor)
processor._native_text_metrics_by_request = {}
state = _make_state(RequestOutputKind.FINAL_ONLY)
processor._update_stats_from_output(
state, SimpleNamespace(new_token_ids=[2150], num_generation_tokens=2048), None, None
)
assert processor.pop_native_text_metrics("r")["num_generation_tokens"] == 2048
def test_delta_drains_output_modality_per_step():
"""DELTA drains the mm_type key (output modality) but preserves hidden-state keys."""
s = _make_state(RequestOutputKind.DELTA)
@@ -616,6 +627,7 @@ def test_no_detokenizer_process_outputs_returns_nonterminal_audio_chunk(monkeypa
finish_reason=None,
stop_reason=None,
kv_transfer_params=None,
ec_transfer_params=None,
routed_experts=None,
num_cached_tokens=0,
)
@@ -631,6 +643,7 @@ def _make_mm_only_output_processor(monkeypatch):
processor = object.__new__(MultimodalOutputProcessor)
processor.output_modality = OutputModality.AUDIO
processor.request_states = {"r": _make_no_detok_state(RequestOutputKind.DELTA)}
processor.tracing_enabled = False
monkeypatch.setattr(
VLLMOutputProcessor,
"process_outputs",
@@ -660,6 +673,7 @@ def _audio_engine_output(*, is_segment_finished: bool, is_last_chunk: bool):
finish_reason=FinishReason.STOP,
stop_reason=None,
kv_transfer_params=None,
ec_transfer_params=None,
routed_experts=None,
num_cached_tokens=0,
is_segment_finished=is_segment_finished,
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for MRv2 Omni AR text detach and generation payload outputs."""
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import FinishReason
from vllm_omni.engine import OmniEngineCoreOutput
from vllm_omni.engine.output_modality import OutputModality
from vllm_omni.engine.output_processor import (
MultimodalOutputProcessor,
OmniRequestState,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _Detokenizer:
def __init__(self):
self.output_token_ids = []
self.output_text = ""
self.update_calls = []
def update(self, token_ids, stop):
self.update_calls.append((list(token_ids), stop))
self.output_token_ids.extend(token_ids)
self.output_text += "X" * len(token_ids)
return None
def get_next_output_text(self, finished, delta):
return self.output_text
def num_output_tokens(self):
return len(self.output_token_ids)
def _make_processor(output_modality, detokenizer=None):
state = OmniRequestState(
request_id="r",
external_req_id="r",
parent_req=None,
request_index=0,
lora_request=None,
output_kind=RequestOutputKind.FINAL_ONLY,
prompt="prompt" if detokenizer else None,
prompt_token_ids=[1] if detokenizer else [],
prompt_embeds=None,
logprobs_processor=MagicMock(logprobs=None, cumulative_logprob=None, prompt_logprobs=None)
if detokenizer
else None,
detokenizer=detokenizer,
max_tokens_param=None,
arrival_time=0.0,
queue=None,
log_stats=False,
stream_interval=1,
)
processor = MultimodalOutputProcessor(tokenizer=None, log_stats=False, output_modality=output_modality)
processor.request_states["r"] = state
processor.external_req_ids["r"].append("r")
return processor, state
def _rehydrated_output(*, pooling_output=None, finish_reason=FinishReason.STOP):
"""Build the post-StagePool shape without violating vLLM's wire schema."""
return SimpleNamespace(
request_id="r",
new_token_ids=[42],
pooling_output=pooling_output,
finish_reason=finish_reason,
stop_reason=None,
kv_transfer_params=None,
ec_transfer_params=None,
routed_experts=None,
trace_headers=None,
prefill_stats=None,
spec_decode_metrics=None,
new_sampling_mask=None,
num_nans_in_logits=0,
multimodal_output=None,
is_segment_finished=False,
is_non_final_audio_chunk=False,
output_type=None,
num_generation_tokens=None,
finished=True,
)
def test_text_tokens_are_detokenized_when_mrv2_ar_output_has_pooling_payload():
detokenizer = _Detokenizer()
processor, state = _make_processor(OutputModality.LATENT, detokenizer)
# StagePool has already rehydrated the MRv2 carrier. Use a structural
# output here because upstream EngineCoreOutput strictly validates
# pooling_output as torch.Tensor | None on construction.
output = _rehydrated_output(pooling_output={"hidden": torch.ones(1, 4)})
processed = processor.process_outputs([output])
assert detokenizer.update_calls == [([42], True)]
completion = processed.request_outputs[0].outputs[0]
assert list(completion.token_ids) == [42]
assert completion.text == "X"
assert not state.mm_accumulated.is_empty
@pytest.mark.parametrize("streaming", [False, True])
def test_generation_stage_mm_delivered_only_with_terminal(streaming):
# Non-terminal chunks accumulate; the terminal output carries the audio.
# When the first output is already terminal it is delivered directly.
processor, _ = _make_processor(OutputModality.AUDIO)
audio = torch.ones(1, 320)
sr = torch.tensor(24000, dtype=torch.int32)
mm_output = OmniEngineCoreOutput(
request_id="r",
new_token_ids=[],
multimodal_output={"model_outputs": audio, "sr": sr},
finish_reason=None if streaming else FinishReason.STOP,
)
processed = processor.process_outputs([mm_output])
if streaming:
assert processed.request_outputs == []
processed = processor.process_outputs(
[OmniEngineCoreOutput(request_id="r", new_token_ids=[], finish_reason=FinishReason.STOP)]
)
assert len(processed.request_outputs) == 1
completion = processed.request_outputs[0].outputs[0]
assert completion.text == ""
assert torch.equal(completion.multimodal_output["audio"], audio)
assert completion.multimodal_output["sr"].item() == 24000
+21
View File
@@ -0,0 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import copy
import pytest
from vllm_omni.engine.request_snapshot import copy_request_snapshot
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_full_snapshot_preserves_fields_aliases_and_isolation():
samples = [0.25, 0.5]
source = {"prompt_token_ids": [1, 2], "additional_information": {"samples": samples}, "alias": samples}
result = copy_request_snapshot(source)
assert result == copy.deepcopy(source)
assert result["alias"] is result["additional_information"]["samples"]
result["alias"].append(0.75)
result["prompt_token_ids"][0] = 9
assert samples == [0.25, 0.5]
assert source["prompt_token_ids"] == [1, 2]
+7 -6
View File
@@ -1393,7 +1393,7 @@ class TestConnectRemoteEngineCoresCoordinator:
yield mocker.Mock()
mocker.patch("vllm_omni.engine.stage_engine_startup.zmq_socket_ctx", return_value=fake_socket_ctx())
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
with connect_remote_engine_cores(
vllm_config=vllm_config,
omni_master_server=omni_master_server,
@@ -1428,7 +1428,7 @@ class TestConnectRemoteEngineCoresCoordinator:
yield mocker.Mock()
mocker.patch("vllm_omni.engine.stage_engine_startup.zmq_socket_ctx", return_value=fake_socket_ctx())
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
with connect_remote_engine_cores(
vllm_config=vllm_config,
omni_master_server=omni_master_server,
@@ -1454,7 +1454,7 @@ class TestConnectRemoteEngineCoresCoordinator:
yield mocker.Mock()
mocker.patch("vllm_omni.engine.stage_engine_startup.zmq_socket_ctx", return_value=fake_socket_ctx())
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
with connect_remote_diffusion_proc(
omni_master_server=omni_master_server,
@@ -1466,7 +1466,8 @@ class TestConnectRemoteEngineCoresCoordinator:
omni_master_server.get_zmq_addresses.assert_called_once_with(7, replica_id=2)
omni_master_server.get_allocation.assert_called_once_with(7, replica_id=2)
mock_wait.assert_called_once()
_, core_engines, parallel_config, *_ = mock_wait.call_args.args
_, core_engines, parallel_config, _, _, launch = mock_wait.call_args.args
assert launch.addresses is omni_master_server.get_zmq_addresses.return_value
assert core_engines[0].local is False
assert parallel_config.data_parallel_size_local == 0
@@ -1507,7 +1508,7 @@ class TestLaunchOmniCoreEngines:
"vllm_omni.engine.stage_engine_startup.CoreEngineProcManager",
return_value=local_engine_manager,
)
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
with _launch_omni_core_engines(
vllm_config=vllm_config,
executor_class=mocker.Mock(),
@@ -1584,7 +1585,7 @@ class TestLaunchOmniCoreEngines:
"vllm_omni.engine.stage_engine_startup.CoreEngineProcManager",
return_value=mocker.Mock(),
)
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup")
mock_wait = mocker.patch("vllm_omni.engine.stage_engine_startup.wait_for_engine_startup", autospec=True)
with _launch_omni_core_engines(
vllm_config=vllm_config,
executor_class=mocker.Mock(),
+12 -3
View File
@@ -8,6 +8,7 @@ import sys
import types
from dataclasses import fields, replace
from pathlib import Path
from unittest.mock import patch
import pytest
from pydantic.fields import FieldInfo
@@ -61,9 +62,14 @@ _DEPLOY_DIR = Path(__file__).parents[2] / "vllm_omni" / "deploy"
def _effective_backend_values(config_cls: type, engine_args: dict) -> dict[str, object]:
backend_fields = fields(config_cls)
backend_field_names = {backend_field.name for backend_field in backend_fields}
backend_config = config_cls(
**{name: copy.deepcopy(value) for name, value in engine_args.items() if name in backend_field_names}
)
# This is a schema/normalization comparison, not a Hub download test.
with (
patch("vllm.engine.arg_utils.get_model_path", side_effect=lambda model, *_: model),
patch("vllm_omni.diffusion.data.get_model_path", side_effect=lambda model, *_: model),
):
backend_config = config_cls(
**{name: copy.deepcopy(value) for name, value in engine_args.items() if name in backend_field_names}
)
effective_values: dict[str, object] = {}
for backend_field in backend_fields:
value = getattr(backend_config, backend_field.name)
@@ -78,6 +84,9 @@ _DIFFUSION_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniDiffusi
_TOPOLOGY_ONLY_ENGINE_ARGS = frozenset({"inline_diffusion"})
_OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
{
"final_output",
"use_v2_model_runner",
"supports_native_mrv2_data_plane",
"active_stream_window",
"codec_frame_rate_hz",
"custom_voice_dir",
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Terminal audio must not retire a request before upstream terminal metrics."""
import asyncio
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def build():
o = Orchestrator.__new__(Orchestrator)
o.stage_pools = [
SimpleNamespace(
final_output=False, build_stage_metrics=lambda *a, **k: SimpleNamespace(stage_id=0, num_tokens_out=42)
),
SimpleNamespace(
final_output=True, build_stage_metrics=lambda *a, **k: SimpleNamespace(stage_id=1, num_tokens_out=0)
),
]
state = OrchestratorRequestState(
request_id="r",
final_stage_id=1,
final_output_stage_ids={1},
stage_submit_ts={0: 0, 1: 0},
sampling_params_list=[None, None],
)
o.request_states = {"r": state}
o.output_async_queue = asyncio.Queue()
o.async_chunk = True
o._pd_pair = None
o._cfg_tracker = MagicMock()
o._cfg_tracker.is_companion.return_value = False
o._cfg_tracker.cleanup_parent.return_value = []
o._is_duplex_session_request = lambda s: False
o._duplex_output_decision = lambda *a: None
o._stage_receives_async_chunks = lambda s: True
cleaned = []
async def cleanup(ids):
cleaned.extend(ids)
for r in ids:
o.request_states.pop(r, None)
o._cleanup_request_ids = cleanup
return o, state, cleaned
@pytest.mark.asyncio
@pytest.mark.parametrize("order", [(0, 1), (1, 0)])
async def test_terminal_metrics_precede_final_audio_for_either_arrival_order(order):
o, state, cleaned = build()
for stage in order:
await o._handle_processed_outputs(stage, 0, [SimpleNamespace(request_id="r", finished=True, error=None)])
if stage == 1 and not state.finished_stage_ids.issuperset({0, 1}):
assert o.output_async_queue.empty()
assert "r" in o.request_states
assert cleaned == []
metrics = o.output_async_queue.get_nowait()
final = o.output_async_queue.get_nowait()
assert type(metrics).__name__ == "StageMetricsMessage"
assert metrics.stage_id == 0 and metrics.metrics.num_tokens_out == 42
assert type(final).__name__ == "OutputMessage" and final.finished
assert o.output_async_queue.empty()
assert cleaned == ["r"]
assert state.pending_final_output is None
@pytest.mark.asyncio
async def test_cancelled_pending_output_is_not_published_by_late_upstream():
o, state, cleaned = build()
await o._handle_processed_outputs(1, 0, [SimpleNamespace(request_id="r", finished=True, error=None)])
assert state.pending_final_output is not None
await o._cleanup_request_ids(["r"])
await o._handle_processed_outputs(0, 0, [SimpleNamespace(request_id="r", finished=True, error=None)])
assert o.output_async_queue.empty()
assert cleaned == ["r"]