[Model] Optimize Qwen3-TTS streaming throughput and first audio on CUDA (#8163)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
55 files changed
+4222
-230
No files matched your search
@@ -0,0 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import queue
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.engine import OmniEngineCoreOutputs
|
||||
from vllm_omni.worker_v2.first_audio_sender import engine_output_queue_sink
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _scheduler(**client_by_request):
|
||||
return SimpleNamespace(
|
||||
requests={request_id: SimpleNamespace(client_index=client) for request_id, client in client_by_request.items()}
|
||||
)
|
||||
|
||||
|
||||
def test_sink_groups_outputs_by_client_and_carries_audio():
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(a=0, b=1, c=0))
|
||||
sample_rate = torch.tensor(24000, dtype=torch.int32)
|
||||
rows = [torch.full((4,), float(i)) for i in range(3)]
|
||||
|
||||
sink.prepare(["a", "b", "c"])(["a", "b", "c"], rows, sample_rate)
|
||||
|
||||
delivered = {}
|
||||
while not output_queue.empty():
|
||||
client_index, outputs = output_queue.get_nowait()
|
||||
assert isinstance(outputs, OmniEngineCoreOutputs)
|
||||
delivered[client_index] = outputs.outputs
|
||||
assert sorted(delivered) == [0, 1]
|
||||
assert [o.request_id for o in delivered[0]] == ["a", "c"]
|
||||
assert [o.request_id for o in delivered[1]] == ["b"]
|
||||
for output in delivered[0] + delivered[1]:
|
||||
# First audio carries no tokens and does not finish the request.
|
||||
assert output.new_token_ids == []
|
||||
assert output.finish_reason is None
|
||||
assert output.multimodal_output["sr"] is sample_rate
|
||||
assert torch.equal(delivered[0][1].multimodal_output["model_outputs"], rows[2])
|
||||
|
||||
|
||||
def test_sink_skips_requests_that_already_left_the_scheduler():
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(kept=0))
|
||||
|
||||
sink.prepare(["gone", "kept"])(["gone", "kept"], [torch.zeros(2), torch.ones(2)], torch.tensor(24000))
|
||||
sink.prepare(["gone"])(["gone"], [torch.zeros(2)], torch.tensor(24000))
|
||||
|
||||
client_index, outputs = output_queue.get_nowait()
|
||||
assert client_index == 0
|
||||
assert [o.request_id for o in outputs.outputs] == ["kept"]
|
||||
assert output_queue.empty()
|
||||
|
||||
|
||||
class _DoneEvent:
|
||||
def record(self):
|
||||
pass
|
||||
|
||||
def synchronize(self):
|
||||
pass
|
||||
|
||||
|
||||
def _run_sender(items):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(**{rid: 0 for item in items for rid in item[2]}))
|
||||
sender = FirstAudioSender(sink)
|
||||
for item in items:
|
||||
sender._queue.put((*item, sink.prepare(item[2])))
|
||||
sender.close()
|
||||
delivered = []
|
||||
while not output_queue.empty():
|
||||
_, outputs = output_queue.get_nowait()
|
||||
delivered.append(
|
||||
([o.request_id for o in outputs.outputs], [o.multimodal_output["model_outputs"] for o in outputs.outputs])
|
||||
)
|
||||
return delivered
|
||||
|
||||
|
||||
def test_sender_drops_rows_whose_frame_is_not_audio():
|
||||
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
sample_rate = torch.tensor(24000, dtype=torch.int32)
|
||||
valid = torch.tensor([True, False, True])
|
||||
|
||||
delivered = _run_sender(
|
||||
[
|
||||
(_DoneEvent(), pcm, ["a", "b", "c"], sample_rate, valid),
|
||||
# Every row invalid: nothing reaches the sink.
|
||||
(_DoneEvent(), pcm[:1], ["d"], sample_rate, torch.tensor([False])),
|
||||
# No validity: every row is delivered (previous behaviour).
|
||||
(_DoneEvent(), pcm[:1], ["e"], sample_rate, None),
|
||||
]
|
||||
)
|
||||
|
||||
assert [ids for ids, _rows in delivered] == [["a", "c"], ["e"]]
|
||||
assert torch.equal(delivered[0][1][1], pcm[2])
|
||||
|
||||
|
||||
def test_prepared_delivery_survives_scheduler_completion():
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
scheduler = _scheduler(r=3)
|
||||
sink = engine_output_queue_sink(output_queue, scheduler)
|
||||
deliver = sink.prepare(["r"])
|
||||
scheduler.requests.clear()
|
||||
deliver(["r"], [torch.ones(2)], torch.tensor(24000))
|
||||
client, outputs = output_queue.get_nowait()
|
||||
assert client == 3
|
||||
assert outputs.outputs[0].multimodal_output["_omni_first_audio"]
|
||||
|
||||
|
||||
def test_delivery_failure_emits_request_error():
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
delivery = engine_output_queue_sink(output_queue, _scheduler(r=0)).prepare(["r"])
|
||||
delivery.fail(["r"])
|
||||
_, outputs = output_queue.get_nowait()
|
||||
assert outputs.outputs[0].finish_reason == FinishReason.ERROR
|
||||
|
||||
|
||||
@pytest.mark.parametrize("client_index", [0, 3])
|
||||
def test_submit_reports_only_requests_with_a_prepared_route(monkeypatch, client_index):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
# Exercise the routing contract without allocating CUDA or pinned memory.
|
||||
original_empty = torch.empty
|
||||
monkeypatch.setattr(torch, "empty", lambda *args, pin_memory=False, **kwargs: original_empty(*args, **kwargs))
|
||||
monkeypatch.setattr(torch.cuda, "Event", _DoneEvent)
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler(kept=client_index)))
|
||||
pcm = torch.arange(6, dtype=torch.float32).reshape(3, 2)
|
||||
try:
|
||||
accepted = sender.submit(["gone", "kept", "also-gone"], pcm, torch.tensor(24000))
|
||||
finally:
|
||||
sender.close()
|
||||
|
||||
assert accepted == ["kept"]
|
||||
client, outputs = output_queue.get_nowait()
|
||||
assert client == client_index
|
||||
assert [output.request_id for output in outputs.outputs] == ["kept"]
|
||||
assert torch.equal(outputs.outputs[0].multimodal_output["model_outputs"], pcm[1])
|
||||
assert output_queue.empty()
|
||||
|
||||
|
||||
def test_submit_without_any_route_leaves_audio_to_the_codec(monkeypatch):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
def unexpected_copy(*args, **kwargs):
|
||||
pytest.fail("No delivery is possible; do not start a device copy")
|
||||
|
||||
monkeypatch.setattr(torch, "empty", unexpected_copy)
|
||||
monkeypatch.setattr(torch.cuda, "Event", unexpected_copy)
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sender = FirstAudioSender(engine_output_queue_sink(output_queue, _scheduler()))
|
||||
try:
|
||||
accepted = sender.submit(["gone"], torch.ones(1, 2), torch.tensor(24000))
|
||||
finally:
|
||||
sender.close()
|
||||
|
||||
assert accepted == []
|
||||
assert output_queue.empty()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sink", [lambda ids, rows, sr: None, SimpleNamespace(prepare=None)])
|
||||
def test_sender_rejects_sink_without_route_preparation(sink):
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
with pytest.raises(TypeError, match="prepare"):
|
||||
FirstAudioSender(sink)
|
||||
|
||||
|
||||
def test_sender_failure_only_aborts_valid_undelivered_rows():
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
class FailSecondClientOnce(queue.Queue):
|
||||
def put_nowait(self, item):
|
||||
client, outputs = item
|
||||
if client == 1 and outputs.outputs[0].finish_reason is None:
|
||||
raise RuntimeError("client enqueue failed")
|
||||
super().put_nowait(item)
|
||||
|
||||
output_queue = FailSecondClientOnce()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(eos=1, sent=0, failed=1, pending=2, next=3))
|
||||
sender = FirstAudioSender(sink)
|
||||
ids = ["eos", "sent", "failed", "pending", "gone"]
|
||||
sender._queue.put(
|
||||
(
|
||||
_DoneEvent(),
|
||||
torch.ones(5, 2),
|
||||
ids,
|
||||
torch.tensor(24000),
|
||||
torch.tensor([False, True, True, True, True]),
|
||||
sink.prepare(ids),
|
||||
)
|
||||
)
|
||||
sender._queue.put((_DoneEvent(), torch.ones(1, 2), ["next"], torch.tensor(24000), None, sink.prepare(["next"])))
|
||||
sender.close()
|
||||
|
||||
actual: list[tuple[int, str, FinishReason | None]] = []
|
||||
while not output_queue.empty():
|
||||
client, outputs = output_queue.get_nowait()
|
||||
actual.extend((client, o.request_id, o.finish_reason) for o in outputs.outputs)
|
||||
assert actual == [
|
||||
(0, "sent", None),
|
||||
(1, "failed", FinishReason.ERROR),
|
||||
(2, "pending", FinishReason.ERROR),
|
||||
(3, "next", None),
|
||||
]
|
||||
|
||||
|
||||
def test_sender_copy_failure_reports_all_prepared_routes():
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.worker_v2.first_audio_sender import FirstAudioSender
|
||||
|
||||
class FailedEvent:
|
||||
def synchronize(self):
|
||||
raise RuntimeError("copy failed before validity became readable")
|
||||
|
||||
output_queue: queue.Queue = queue.Queue()
|
||||
sink = engine_output_queue_sink(output_queue, _scheduler(r=3))
|
||||
sender = FirstAudioSender(sink)
|
||||
sender._queue.put(
|
||||
(FailedEvent(), torch.ones(2, 2), ["r", "gone"], torch.tensor(24000), None, sink.prepare(["r", "gone"]))
|
||||
)
|
||||
sender.close()
|
||||
client, outputs = output_queue.get_nowait()
|
||||
assert client == 3
|
||||
assert [(o.request_id, o.finish_reason) for o in outputs.outputs] == [("r", FinishReason.ERROR)]
|
||||
assert output_queue.empty()
|
||||
@@ -391,3 +391,204 @@ def test_mtp_requires_model_declared_output_key(output_key):
|
||||
model = SimpleNamespace(mtp=lambda: None, mtp_output_key=output_key)
|
||||
with pytest.raises(TypeError, match="must declare mtp_output_key"):
|
||||
state._init_mtp_runner(model)
|
||||
|
||||
|
||||
_EAGER_DIM = 3
|
||||
_CODEBOOK = 2048
|
||||
_EOS = 2150
|
||||
|
||||
|
||||
class _EagerBatch(_DummyInputBatch):
|
||||
def __init__(self, spans, indices=None):
|
||||
indices = list(range(len(spans))) if indices is None else indices
|
||||
super().__init__(indices)
|
||||
starts = [0]
|
||||
for n_tok in spans:
|
||||
starts.append(starts[-1] + n_tok)
|
||||
self.num_scheduled_tokens = list(spans)
|
||||
self.query_start_loc_np = starts[:-1]
|
||||
self.query_start_loc = torch.tensor(starts, dtype=torch.int32)
|
||||
|
||||
|
||||
def _make_eager_state(max_num_reqs=4):
|
||||
state = _make_state(max_num_reqs=max_num_reqs, has_preprocess=True)
|
||||
_init_static(state, max_num_reqs, dim=_EAGER_DIM)
|
||||
from vllm_omni.worker_v2.model_states.eager_mtp import EagerMTPState
|
||||
|
||||
state._eager_state = EagerMTPState(state)
|
||||
state._first_audio_requests = set()
|
||||
state._eager_mtp = True
|
||||
state._eager_fastpath = False
|
||||
state._eager_rows = None
|
||||
state._eager_ready = {}
|
||||
state._eager_settled = {}
|
||||
state._eager_embeds = torch.zeros((max_num_reqs, _EAGER_DIM))
|
||||
state._decode_preprocess = None
|
||||
state._mtp_sample_uniforms = None
|
||||
state.vllm_config.cache_config = SimpleNamespace(enable_prefix_caching=False)
|
||||
model = state.model
|
||||
model.first_frame_decoder = None
|
||||
model.mtp_frame_valid = lambda layer0: (layer0 >= 0) & (layer0 < _CODEBOOK)
|
||||
model.embed_input_ids = lambda ids: ids.float().reshape(-1, 1, 1).expand(-1, 1, _EAGER_DIM)
|
||||
model.mtp_calls = []
|
||||
|
||||
def mtp(input_ids, input_embeds, last_hidden, text_step, **kwargs):
|
||||
model.mtp_calls.append((input_ids.clone(), last_hidden.clone(), text_step.clone()))
|
||||
codes = torch.stack([input_ids, input_ids + 1, input_ids + 2], dim=1)
|
||||
# Frame embedding sum plus the text step, like the Qwen3 talkers.
|
||||
return input_embeds + 100 + text_step, codes
|
||||
|
||||
model.mtp = mtp
|
||||
return state
|
||||
|
||||
|
||||
def _eager_outputs(num_tokens):
|
||||
return {
|
||||
"codes": {"audio": torch.zeros((num_tokens, 3), dtype=torch.long)},
|
||||
"meta": {"codec_frame_valid": torch.zeros((num_tokens,), dtype=torch.int8)},
|
||||
}
|
||||
|
||||
|
||||
def test_eager_mtp_publishes_frames_in_the_step_that_sampled_cb0():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "prefill", "decode", "ended")
|
||||
# prefill: final 3-token chunk; decode: input CB0 42; ended: input CB0 was
|
||||
# EOS (async scheduling ran the finished request once more).
|
||||
batch = _EagerBatch([3, 1, 1])
|
||||
input_ids = torch.tensor([0, 0, 0, 42, _EOS])
|
||||
state._eager_rows = (batch, [(0, 0, "prefill", True), (1, 1, "decode", False), (2, 2, "ended", False)], input_ids)
|
||||
text_hidden = torch.arange(5 * _EAGER_DIM, dtype=torch.float32).reshape(5, _EAGER_DIM)
|
||||
sampled = torch.tensor([[7], [_EOS], [9]])
|
||||
outputs = _eager_outputs(5)
|
||||
|
||||
state.run_eager_mtp(batch, text_hidden, sampled, outputs)
|
||||
|
||||
ids, hidden, text_step = state.model.mtp_calls[0]
|
||||
assert ids.tolist() == [7, _EOS, 9]
|
||||
# The frame's hidden is this step's last-token hidden of each span.
|
||||
assert torch.equal(hidden, text_hidden[[2, 3, 4]])
|
||||
# The next step adds its own text step.
|
||||
assert not text_step.any()
|
||||
codes = outputs["codes"]["audio"]
|
||||
assert codes[2].tolist() == [7, 8, 9]
|
||||
assert codes[3].tolist() == [_EOS, _EOS + 1, _EOS + 2]
|
||||
assert codes[4].tolist() == [9, 10, 11]
|
||||
assert not codes[:2].any()
|
||||
# Valid only when both the sampled and (for decode) the input CB0 are codec ids.
|
||||
assert outputs["meta"]["codec_frame_valid"].tolist() == [0, 0, 1, 0, 0]
|
||||
assert state._eager_ready == {0: "prefill", 1: "decode", 2: "ended"}
|
||||
assert torch.equal(state._eager_embeds[0], torch.full((_EAGER_DIM,), 107.0))
|
||||
assert state._eager_rows is None
|
||||
|
||||
|
||||
def test_eager_mtp_ignores_rows_recorded_for_another_batch():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
state._eager_rows = (_EagerBatch([1]), [(0, 0, "r0", False)], torch.tensor([5]))
|
||||
outputs = _eager_outputs(1)
|
||||
state.run_eager_mtp(_EagerBatch([1]), torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), outputs)
|
||||
assert state.model.mtp_calls == []
|
||||
assert not outputs["codes"]["audio"].any()
|
||||
|
||||
|
||||
def test_eager_mtp_requires_retained_frame_outputs():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
batch = _EagerBatch([1])
|
||||
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
|
||||
with pytest.raises(RuntimeError, match="requires retained codes.audio"):
|
||||
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[3]]), {"codes": {"audio": None}})
|
||||
|
||||
|
||||
def test_eager_decode_input_is_previous_frame_plus_text_step():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0", "r1")
|
||||
state._eager_ready = {0: "r0", 1: "r1"}
|
||||
state._eager_embeds[0] = 5.0
|
||||
state._eager_embeds[1] = 7.0
|
||||
embeds = torch.zeros((2, _EAGER_DIM))
|
||||
batches = [
|
||||
(0, 0, (torch.zeros(_EAGER_DIM), torch.full((_EAGER_DIM,), 2.0))),
|
||||
(1, 1, (None, torch.ones(_EAGER_DIM))),
|
||||
]
|
||||
state._eager_state._apply_eager_frames(batches, embeds, _EagerBatch([1, 1]), None)
|
||||
assert embeds.tolist() == [[7.0] * _EAGER_DIM, [8.0] * _EAGER_DIM]
|
||||
# The deferred MTP must not run for an eager decode row.
|
||||
assert state.model.mtp_calls == []
|
||||
|
||||
|
||||
def test_eager_decode_without_a_frame_fails_loudly():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
batches = [(0, 0, (torch.zeros(_EAGER_DIM), torch.zeros(_EAGER_DIM)))]
|
||||
with pytest.raises(RuntimeError, match="frame missing"):
|
||||
state._eager_state._apply_eager_frames(batches, torch.zeros((1, _EAGER_DIM)), _EagerBatch([1]), None)
|
||||
|
||||
|
||||
def test_run_preprocess_records_rows_that_keep_a_sample():
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "chunk", "final", "decode")
|
||||
state._eager_ready = {2: "decode"}
|
||||
state._eager_embeds[2] = 4.0
|
||||
text_step = torch.ones(_EAGER_DIM)
|
||||
|
||||
def preprocess(input_ids, input_embeds, **info):
|
||||
updates = {"mtp_inputs": (torch.zeros(_EAGER_DIM), text_step)} if input_ids.shape[0] == 1 else {}
|
||||
return input_ids, input_embeds, updates
|
||||
|
||||
state.model.preprocess = preprocess
|
||||
batch = _EagerBatch([3, 3, 1])
|
||||
model_inputs = {"input_ids": torch.zeros(7, dtype=torch.long), "inputs_embeds": torch.zeros((7, _EAGER_DIM))}
|
||||
req_states = SimpleNamespace(
|
||||
prompt_len=np.array([10, 10, 10], dtype=np.int32),
|
||||
num_computed_tokens=np.array([0, 7, 10], dtype=np.int32),
|
||||
)
|
||||
state.run_preprocess(batch, model_inputs, req_states)
|
||||
|
||||
recorded_batch, entries, _ids = state._eager_rows
|
||||
assert recorded_batch is batch
|
||||
# A non-final prefill chunk samples nothing that is kept.
|
||||
assert entries == [(1, 1, "final", True), (2, 2, "decode", False)]
|
||||
assert model_inputs["inputs_embeds"][6].tolist() == [5.0] * _EAGER_DIM
|
||||
assert state.model.mtp_calls == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_was_valid", [False, True])
|
||||
def test_first_audio_requirement_survives_eos_only_if_audio_was_queued(first_was_valid):
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0")
|
||||
state._first_audio_requests.add("r0")
|
||||
state._eager_state._first_audio_valid = torch.tensor([first_was_valid, False, False, False])
|
||||
batch = _EagerBatch([1])
|
||||
state._eager_rows = (batch, [(0, 0, "r0", False)], torch.tensor([5]))
|
||||
outputs = _eager_outputs(1)
|
||||
outputs["meta"]["first_audio"] = torch.zeros(1, dtype=torch.bool)
|
||||
state.run_eager_mtp(batch, torch.zeros(1, _EAGER_DIM), torch.tensor([[_EOS]]), outputs)
|
||||
assert outputs["meta"]["first_audio"].tolist() == [first_was_valid]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accepted", [[], ["r1"], ["r0", "r1"]])
|
||||
def test_first_audio_marker_requires_accepted_delivery(monkeypatch, accepted):
|
||||
from contextlib import nullcontext
|
||||
|
||||
state = _make_eager_state()
|
||||
_fill_buffers(state, "r0", "r1")
|
||||
state.model.first_frame_decoder = SimpleNamespace(
|
||||
sample_rate=24000, decode=lambda codes: torch.ones(codes.shape[0], 2)
|
||||
)
|
||||
state._first_audio_stream = SimpleNamespace(wait_stream=lambda stream: None)
|
||||
state._first_audio_sender = SimpleNamespace(submit=lambda ids, pcm, sr, valid: accepted)
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", lambda device: None)
|
||||
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
|
||||
monkeypatch.setattr(torch.Tensor, "record_stream", lambda tensor, stream: None)
|
||||
batch = _EagerBatch([1, 1])
|
||||
state._eager_rows = (batch, [(0, 0, "r0", True), (1, 1, "r1", True)], torch.zeros(2, dtype=torch.long))
|
||||
outputs = _eager_outputs(2)
|
||||
outputs["meta"]["first_audio"] = torch.zeros(2, dtype=torch.bool)
|
||||
|
||||
state.run_eager_mtp(batch, torch.zeros(2, _EAGER_DIM), torch.tensor([[7], [8]]), outputs)
|
||||
|
||||
assert state._first_audio_requests == set(accepted)
|
||||
# A missing route must leave the normal codec path responsible for frame 0;
|
||||
# otherwise it skips the frame and the orchestrator waits forever for it.
|
||||
assert outputs["meta"]["first_audio"].tolist() == ["r0" in accepted, "r1" in accepted]
|
||||
Reference in new issue
Block a user