[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163)

Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
Sy03 authored and GitHub committed 2026-09-26 22:51:04 +08:00
1 parent 8a49c65bf9
commit e1303e5067
55 files changed
+4222 -230

No files matched your search

+237
View File
@@ -0,0 +1,237 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import queue
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.engine import OmniEngineCoreOutputs
from vllm_omni.worker_v2.first_audio_sender import engine_output_queue_sink
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _scheduler(**client_by_request):
return SimpleNamespace(
requests={request_id: SimpleNamespace(client_index=client) for request_id, client in client_by_request.items()}
)
def test_sink_groups_outputs_by_client_and_carries_audio():
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(a=0, b=1, c=0))
sample_rate = torch.tensor(24000, dtype=torch.int32)
rows = [torch.full((4,), float(i)) for i in range(3)]
sink.prepare(["a", "b", "c"])(["a", "b", "c"], rows, sample_rate)
delivered = {}
while not output_queue.empty():
client_index, outputs = output_queue.get_nowait()
assert isinstance(outputs, OmniEngineCoreOutputs)
delivered[client_index] = outputs.outputs
assert sorted(delivered) == [0, 1]
assert [o.request_id for o in delivered[0]] == ["a", "c"]
assert [o.request_id for o in delivered[1]] == ["b"]
for output in delivered[0] + delivered[1]:
# First audio carries no tokens and does not finish the request.
assert output.new_token_ids == []
assert output.finish_reason is None
assert output.multimodal_output["sr"] is sample_rate
assert torch.equal(delivered[0][1].multimodal_output["model_outputs"], rows[2])
def test_sink_skips_requests_that_already_left_the_scheduler():
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(kept=0))
sink.prepare(["gone", "kept"])(["gone", "kept"], [torch.zeros(2), torch.ones(2)], torch.tensor(24000))
sink.prepare(["gone"])(["gone"], [torch.zeros(2)], torch.tensor(24000))
client_index, outputs = output_queue.get_nowait()
assert client_index == 0
assert [o.request_id for o in outputs.outputs] == ["kept"]
assert output_queue.empty()
class _DoneEvent:
def record(self):
pass
def synchronize(self):
pass
def _run_sender(items):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(**{rid: 0 for item in items for rid in item[2]}))
sender = FirstAudioSender(sink)
for item in items:
sender._queue.put((*item, sink.prepare(item[2])))
sender.close()
delivered = []
while not output_queue.empty():
_, outputs = output_queue.get_nowait()
delivered.append(
([o.request_id for o in outputs.outputs], [o.multimodal_output["model_outputs"] for o in outputs.outputs])
)
return delivered
def test_sender_drops_rows_whose_frame_is_not_audio():
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
sample_rate = torch.tensor(24000, dtype=torch.int32)
valid = torch.tensor([True, False, True])
delivered = _run_sender(
[
(_DoneEvent(), pcm, ["a", "b", "c"], sample_rate, valid),
# Every row invalid: nothing reaches the sink.
(_DoneEvent(), pcm[:1], ["d"], sample_rate, torch.tensor([False])),
# No validity: every row is delivered (previous behaviour).
(_DoneEvent(), pcm[:1], ["e"], sample_rate, None),
]
)
assert [ids for ids, _rows in delivered] == [["a", "c"], ["e"]]
assert torch.equal(delivered[0][1][1], pcm[2])
def test_prepared_delivery_survives_scheduler_completion():
output_queue: queue.Queue = queue.Queue()
scheduler = _scheduler(r=3)
sink = engine_output_queue_sink(output_queue, scheduler)
deliver = sink.prepare(["r"])
scheduler.requests.clear()
deliver(["r"], [torch.ones(2)], torch.tensor(24000))
client, outputs = output_queue.get_nowait()
assert client == 3
assert outputs.outputs[0].multimodal_output["_omni_first_audio"]
def test_delivery_failure_emits_request_error():
from vllm.v1.engine import FinishReason
output_queue: queue.Queue = queue.Queue()
delivery = engine_output_queue_sink(output_queue, _scheduler(r=0)).prepare(["r"])
delivery.fail(["r"])
_, outputs = output_queue.get_nowait()
assert outputs.outputs[0].finish_reason == FinishReason.ERROR
@pytest.mark.parametrize("client_index", [0, 3])
def test_submit_reports_only_requests_with_a_prepared_route(monkeypatch, client_index):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
# Exercise the routing contract without allocating CUDA or pinned memory.
original_empty = torch.empty
monkeypatch.setattr(torch, "empty", lambda *args, pin_memory=False, **kwargs: original_empty(*args, **kwargs))
monkeypatch.setattr(torch.cuda, "Event", _DoneEvent)
output_queue: queue.Queue = queue.Queue()
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler(kept=client_index)))
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
try:
accepted = sender.submit(["gone", "kept", "also-gone"], pcm, torch.tensor(24000))
finally:
sender.close()
assert accepted == ["kept"]
client, outputs = output_queue.get_nowait()
assert client == client_index
assert [output.request_id for output in outputs.outputs] == ["kept"]
assert torch.equal(outputs.outputs[0].multimodal_output["model_outputs"], pcm[1])
assert output_queue.empty()
def test_submit_without_any_route_leaves_audio_to_the_codec(monkeypatch):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
def unexpected_copy(*args, **kwargs):
pytest.fail("No delivery is possible; do not start a device copy")
monkeypatch.setattr(torch, "empty", unexpected_copy)
monkeypatch.setattr(torch.cuda, "Event", unexpected_copy)
output_queue: queue.Queue = queue.Queue()
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler()))
try:
accepted = sender.submit(["gone"], torch.ones(1, 2), torch.tensor(24000))
finally:
sender.close()
assert accepted == []
assert output_queue.empty()
@pytest.mark.parametrize("sink", [lambda ids, rows, sr: None, SimpleNamespace(prepare=None)])
def test_sender_rejects_sink_without_route_preparation(sink):
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
with pytest.raises(TypeError, match="prepare"):
FirstAudioSender(sink)
def test_sender_failure_only_aborts_valid_undelivered_rows():
from vllm.v1.engine import FinishReason
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
class FailSecondClientOnce(queue.Queue):
def put_nowait(self, item):
client, outputs = item
if client == 1 and outputs.outputs[0].finish_reason is None:
raise RuntimeError("client enqueue failed")
super().put_nowait(item)
output_queue = FailSecondClientOnce()
sink = engine_output_queue_sink(output_queue, _scheduler(eos=1, sent=0, failed=1, pending=2, next=3))
sender = FirstAudioSender(sink)
ids = ["eos", "sent", "failed", "pending", "gone"]
sender._queue.put(
(
_DoneEvent(),
torch.ones(5, 2),
ids,
torch.tensor(24000),
torch.tensor([False, True, True, True, True]),
sink.prepare(ids),
)
)
sender._queue.put((_DoneEvent(), torch.ones(1, 2), ["next"], torch.tensor(24000), None, sink.prepare(["next"])))
sender.close()
actual: list[tuple[int, str, FinishReason | None]] = []
while not output_queue.empty():
client, outputs = output_queue.get_nowait()
actual.extend((client, o.request_id, o.finish_reason) for o in outputs.outputs)
assert actual == [
(0, "sent", None),
(1, "failed", FinishReason.ERROR),
(2, "pending", FinishReason.ERROR),
(3, "next", None),
]
def test_sender_copy_failure_reports_all_prepared_routes():
from vllm.v1.engine import FinishReason
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
class FailedEvent:
def synchronize(self):
raise RuntimeError("copy failed before validity became readable")
output_queue: queue.Queue = queue.Queue()
sink = engine_output_queue_sink(output_queue, _scheduler(r=3))
sender = FirstAudioSender(sink)
sender._queue.put(
(FailedEvent(), torch.ones(2, 2), ["r", "gone"], torch.tensor(24000), None, sink.prepare(["r", "gone"]))
)
sender.close()
client, outputs = output_queue.get_nowait()
assert client == 3
assert [(o.request_id, o.finish_reason) for o in outputs.outputs] == [("r", FinishReason.ERROR)]
assert output_queue.empty()
+201
View File
@@ -391,3 +391,204 @@ def test_mtp_requires_model_declared_output_key(output_key):
model = SimpleNamespace(mtp=lambda: None, mtp_output_key=output_key)
with pytest.raises(TypeError, match="must declare mtp_output_key"):
state._init_mtp_runner(model)
_EAGER_DIM = 3
_CODEBOOK = 2048
_EOS = 2150
class _EagerBatch(_DummyInputBatch):
def __init__(self, spans, indices=None):
indices = list(range(len(spans))) if indices is None else indices
super().__init__(indices)
starts = [0]
for n_tok in spans:
starts.append(starts[-1] + n_tok)
self.num_scheduled_tokens = list(spans)
self.query_start_loc_np = starts[:-1]
self.query_start_loc = torch.tensor(starts, dtype=torch.int32)
def _make_eager_state(max_num_reqs=4):
state = _make_state(max_num_reqs=max_num_reqs, has_preprocess=True)
_init_static(state, max_num_reqs, dim=_EAGER_DIM)
from vllm_omni.worker_v2.model_states.eager_mtp import EagerMTPState
state._eager_state = EagerMTPState(state)
state._first_audio_requests = set()
state._eager_mtp = True
state._eager_fastpath = False
state._eager_rows = None
state._eager_ready = {}
state._eager_settled = {}
state._eager_embeds = torch.zeros((max_num_reqs, _EAGER_DIM))
state._decode_preprocess = None
state._mtp_sample_uniforms = None
state.vllm_config.cache_config = SimpleNamespace(enable_prefix_caching=False)
model = state.model
model.first_frame_decoder = None
model.mtp_frame_valid = lambda layer0: (layer0 >= 0) & (layer0 < _CODEBOOK)
model.embed_input_ids = lambda ids: ids.float().reshape(-1, 1, 1).expand(-1, 1, _EAGER_DIM)
model.mtp_calls = []
def mtp(input_ids, input_embeds, last_hidden, text_step, **kwargs):
model.mtp_calls.append((input_ids.clone(), last_hidden.clone(), text_step.clone()))
codes = torch.stack([input_ids, input_ids + 1, input_ids + 2], dim=1)
# Frame embedding sum plus the text step, like the Qwen3 talkers.
return input_embeds + 100 + text_step, codes
model.mtp = mtp
return state
def _eager_outputs(num_tokens):
return {
"codes": {"audio": torch.zeros((num_tokens, 3), dtype=torch.long)},
"meta": {"codec_frame_valid": torch.zeros((num_tokens,), dtype=torch.int8)},
}
def test_eager_mtp_publishes_frames_in_the_step_that_sampled_cb0():
state = _make_eager_state()
_fill_buffers(state, "prefill", "decode", "ended")
# prefill: final 3-token chunk; decode: input CB0 42; ended: input CB0 was
# EOS (async scheduling ran the finished request once more).
batch = _EagerBatch([3, 1, 1])
input_ids = torch.tensor([0, 0, 0, 42, _EOS])
state._eager_rows = (batch, [(0, 0, "prefill", True), (1, 1, "decode", False), (2, 2, "ended", False)], input_ids)
text_hidden = torch.arange(5 * _EAGER_DIM, dtype=torch.float32).reshape(5, _EAGER_DIM)
sampled = torch.tensor([[7], [_EOS], [9]])
outputs = _eager_outputs(5)
state.run_eager_mtp(batch, text_hidden, sampled, outputs)
ids, hidden, text_step = state.model.mtp_calls[0]
assert ids.tolist() == [7, _EOS, 9]
# The frame's hidden is this step's last-token hidden of each span.
assert torch.equal(hidden, text_hidden[[2, 3, 4]])
# The next step adds its own text step.
assert not text_step.any()
codes = outputs["codes"]["audio"]
assert codes[2].tolist() == [7, 8, 9]
assert codes[3].tolist() == [_EOS, _EOS + 1, _EOS + 2]
assert codes[4].tolist() == [9, 10, 11]
assert not codes[:2].any()
# Valid only when both the sampled and (for decode) the input CB0 are codec ids.
assert outputs["meta"]["codec_frame_valid"].tolist() == [0, 0, 1, 0, 0]
assert state._eager_ready == {0: "prefill", 1: "decode", 2: "ended"}
assert torch.equal(state._eager_embeds[0], torch.full((_EAGER_DIM,), 107.0))
assert state._eager_rows is None
def test_eager_mtp_ignores_rows_recorded_for_another_batch():
state = _make_eager_state()
_fill_buffers(state, "r0")
state._eager_rows = (_EagerBatch([1]), [(0, 0, "r0", False)], torch.tensor([5]))
outputs = _eager_outputs(1)
state.run_eager_mtp(_EagerBatch([1]), torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), outputs)
assert state.model.mtp_calls == []
assert not outputs["codes"]["audio"].any()
def test_eager_mtp_requires_retained_frame_outputs():
state = _make_eager_state()
_fill_buffers(state, "r0")
batch = _EagerBatch([1])
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
with pytest.raises(RuntimeError, match="requires retained codes.audio"):
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), {"codes": {"audio": None}})
def test_eager_decode_input_is_previous_frame_plus_text_step():
state = _make_eager_state()
_fill_buffers(state, "r0", "r1")
state._eager_ready = {0: "r0", 1: "r1"}
state._eager_embeds[0] = 5.0
state._eager_embeds[1] = 7.0
embeds = torch.zeros((2, _EAGER_DIM))
batches = [
(0, 0, (torch.zeros(_EAGER_DIM), torch.full((_EAGER_DIM,), 2.0))),
(1, 1, (None, torch.ones(_EAGER_DIM))),
]
state._eager_state._apply_eager_frames(batches, embeds, _EagerBatch([1, 1]), None)
assert embeds.tolist() == [[7.0] * _EAGER_DIM, [8.0] * _EAGER_DIM]
# The deferred MTP must not run for an eager decode row.
assert state.model.mtp_calls == []
def test_eager_decode_without_a_frame_fails_loudly():
state = _make_eager_state()
_fill_buffers(state, "r0")
batches = [(0, 0, (torch.zeros(_EAGER_DIM), torch.zeros(_EAGER_DIM)))]
with pytest.raises(RuntimeError, match="frame missing"):
state._eager_state._apply_eager_frames(batches, torch.zeros((1, _EAGER_DIM)), _EagerBatch([1]), None)
def test_run_preprocess_records_rows_that_keep_a_sample():
state = _make_eager_state()
_fill_buffers(state, "chunk", "final", "decode")
state._eager_ready = {2: "decode"}
state._eager_embeds[2] = 4.0
text_step = torch.ones(_EAGER_DIM)
def preprocess(input_ids, input_embeds, **info):
updates = {"mtp_inputs": (torch.zeros(_EAGER_DIM), text_step)} if input_ids.shape[0] == 1 else {}
return input_ids, input_embeds, updates
state.model.preprocess = preprocess
batch = _EagerBatch([3, 3, 1])
model_inputs = {"input_ids": torch.zeros(7, dtype=torch.long), "inputs_embeds": torch.zeros((7, _EAGER_DIM))}
req_states = SimpleNamespace(
prompt_len=np.array([10, 10, 10], dtype=np.int32),
num_computed_tokens=np.array([0, 7, 10], dtype=np.int32),
)
state.run_preprocess(batch, model_inputs, req_states)
recorded_batch, entries, _ids = state._eager_rows
assert recorded_batch is batch
# A non-final prefill chunk samples nothing that is kept.
assert entries == [(1, 1, "final", True), (2, 2, "decode", False)]
assert model_inputs["inputs_embeds"][6].tolist() == [5.0] * _EAGER_DIM
assert state.model.mtp_calls == []
@pytest.mark.parametrize("first_was_valid", [False, True])
def test_first_audio_requirement_survives_eos_only_if_audio_was_queued(first_was_valid):
state = _make_eager_state()
_fill_buffers(state, "r0")
state._first_audio_requests.add("r0")
state._eager_state._first_audio_valid = torch.tensor([first_was_valid, False, False, False])
batch = _EagerBatch([1])
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
outputs = _eager_outputs(1)
outputs["meta"]["first_audio"] = torch.zeros(1, dtype=torch.bool)
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[_EOS]]), outputs)
assert outputs["meta"]["first_audio"].tolist() == [first_was_valid]
@pytest.mark.parametrize("accepted", [[], ["r1"], ["r0", "r1"]])
def test_first_audio_marker_requires_accepted_delivery(monkeypatch, accepted):
from contextlib import nullcontext
state = _make_eager_state()
_fill_buffers(state, "r0", "r1")
state.model.first_frame_decoder = SimpleNamespace(
sample_rate=24000, decode=lambda codes: torch.ones(codes.shape[0], 2)
)
state._first_audio_stream = SimpleNamespace(wait_stream=lambda stream: None)
state._first_audio_sender = SimpleNamespace(submit=lambda ids, pcm, sr, valid: accepted)
monkeypatch.setattr(torch.cuda, "current_stream", lambda device: None)
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
monkeypatch.setattr(torch.Tensor, "record_stream", lambda tensor, stream: None)
batch = _EagerBatch([1, 1])
state._eager_rows = (batch, [(0, 0, "r0", True), (1, 1, "r1", True)], torch.zeros(2, dtype=torch.long))
outputs = _eager_outputs(2)
outputs["meta"]["first_audio"] = torch.zeros(2, dtype=torch.bool)
state.run_eager_mtp(batch, torch.zeros(2, _EAGER_DIM), torch.tensor([[7], [8]]), outputs)
assert state._first_audio_requests == set(accepted)
# A missing route must leave the normal codec path responsible for frame 0;
# otherwise it skips the frame and the orchestrator waits forever for it.
assert outputs["meta"]["first_audio"].tolist() == ["r0" in accepted, "r1" in accepted]