[Refactor/Bugfix] Use Delta Messages for Streaming (#2911)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
Signed-off-by: Sy03 <1370724210@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Sy03 <1370724210@qq.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-04-26 15:11:53 +12:00
1 parent 6e35bfbbae
commit e03ab82dd2
27 files changed
+623 -177

No files matched your search

@@ -82,6 +82,30 @@ class TestVoxtralTTSFixedVoice:
f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
)
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_speech_english_streaming(self, omni_server) -> None:
"""Test basic streaming English TTS generation."""
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
payload = {
"input": "Hello, how are you?",
"voice": "casual_female",
"language": "English",
"stream": True,
"response_format": "pcm",
}
with httpx.Client(timeout=120.0) as client:
with client.stream("POST", url, json=payload) as response:
assert response.status_code == 200
assert response.headers.get("content-type") == "audio/pcm"
total = sum(len(c) for c in response.iter_bytes())
assert total > MIN_AUDIO_BYTES, (
f"Streamed audio too small ({total} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
+211
View File
@@ -0,0 +1,211 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for OmniRequestState multimodal DELTA drain and consolidation guard."""
from unittest.mock import MagicMock
import pytest
import torch
from vllm.outputs import PoolingRequestOutput
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import FinishReason
from vllm_omni.engine.output_modality import OutputModalityNames
from vllm_omni.engine.output_processor import OmniRequestState
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# Audio is explicitly listed as a drainable modality
AUDIO = OutputModalityNames.AUDIO
# Latent is explicitly not drainable, but the choice doesn't matter here as
# long as isn't listed as drainable. I.e., could also be arbitrary keys for
# the purposes of these tests
LATENT = OutputModalityNames.LATENT
# NOTE: detokenizer and logprobs aren't really used here, but we mock them since
# some of the utils called in vLLM superclass assert require them to be None.
_DETOK = MagicMock(
output_token_ids=[0],
get_next_output_text=MagicMock(return_value=""),
num_output_tokens=MagicMock(return_value=1),
)
_LOGPROBS = MagicMock(logprobs=None, cumulative_logprob=None, prompt_logprobs=None)
_DEFAULT_STATE_KWARGS = dict(
request_id="r",
external_req_id="r",
parent_req=None,
request_index=0,
lora_request=None,
prompt=None,
prompt_token_ids=[0],
prompt_embeds=None,
logprobs_processor=_LOGPROBS,
detokenizer=_DETOK,
max_tokens_param=None,
arrival_time=0.0,
queue=None,
log_stats=False,
stream_interval=1,
)
def _make_state(output_kind: RequestOutputKind):
return OmniRequestState(**_DEFAULT_STATE_KWARGS, output_kind=output_kind)
def test_init_empty_dict():
"""Ensure mm_accumulated is initially empty."""
assert _make_state(RequestOutputKind.CUMULATIVE).mm_accumulated == {}
assert _make_state(RequestOutputKind.DELTA).mm_accumulated == {}
def test_delta_drains_output_modality_per_step():
"""DELTA drains the mm_type key (output modality) but preserves hidden-state keys."""
s = _make_state(RequestOutputKind.DELTA)
audio1, audio2, hs1, hs2 = [torch.ones(num_elem) for num_elem in range(1, 5)]
# Add audio and hidden state tensors
s.add_multimodal_tensor(audio1, mm_type=AUDIO) # should be drained
s.add_multimodal_tensor(hs1, mm_type=LATENT) # shouldn't be drained
out1 = s._new_completion_output([1], None, None)
out1_audio = out1.multimodal_output[AUDIO]
out1_hidden = out1.multimodal_output[LATENT]
assert isinstance(out1_audio, torch.Tensor)
assert torch.equal(out1.multimodal_output[AUDIO], audio1)
assert isinstance(out1_hidden, torch.Tensor)
assert torch.equal(out1.multimodal_output[LATENT], hs1)
# After emission, hidden states should remain, but audio is drained
assert set(s.mm_accumulated.keys()) == {LATENT}
s.add_multimodal_tensor(audio2, AUDIO)
s.add_multimodal_tensor(hs2, mm_type=LATENT)
out2 = s._new_completion_output([2], None, None)
out2_audio = out2.multimodal_output[AUDIO]
out2_hidden = out2.multimodal_output[LATENT]
assert isinstance(out2_audio, torch.Tensor)
assert torch.equal(out2_audio, audio2)
# Since hidden isn't drained, it's grown to a list
assert isinstance(out2_hidden, list) and len(out2_hidden) == 2
assert torch.equal(hs1, out2_hidden[0])
assert torch.equal(hs2, out2_hidden[1])
def test_cumulative_emits_consolidated_audio_each_step():
"""Ensure cumulative accumulates and consolidates modality keys every step."""
s = _make_state(RequestOutputKind.CUMULATIVE)
# NOTE: audio is usually emitted as (1, size) chunks; we need to be sure
# to not change the tensor dimension when we consolidate
audio1 = torch.ones(1, 500)
s.add_multimodal_tensor(audio1, mm_type=AUDIO)
req_out = s.make_request_output([1], None, None, None)
assert req_out is not None
cons_audio = req_out.outputs[0].multimodal_output[AUDIO]
# Single chunk keeps original shape [1, 500]
assert isinstance(cons_audio, torch.Tensor) and cons_audio.shape == audio1.shape
audio2 = torch.ones(1, 300)
s.add_multimodal_tensor(audio2, mm_type=AUDIO)
req_out = s.make_request_output([2], None, None, None)
assert req_out is not None
cons_audio = req_out.outputs[0].multimodal_output[AUDIO]
# After consolidation, audio chunks are concatenated on last axis,
# preserving the [1, N] channel dimension
total_audio_len = audio1.shape[-1] + audio2.shape[-1]
assert isinstance(audio2, torch.Tensor) and cons_audio.shape == (1, total_audio_len)
assert "audio" in s.mm_accumulated
def test_finish_consolidates_hidden_states():
"""Ensure consolidation merges hidden-state tensor lists on finish."""
s = _make_state(RequestOutputKind.CUMULATIVE)
s.add_multimodal_tensor(torch.ones(5, 4), mm_type=LATENT)
s.add_multimodal_tensor(torch.ones(3, 4), mm_type=LATENT)
result = s.make_request_output([1], None, FinishReason.STOP, None)
assert result is not None and not isinstance(result, PoolingRequestOutput)
hs = result.outputs[0].multimodal_output[LATENT]
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 8
def test_finish_consolidation_for_hs_delta():
"""Ensure finish doesn't drop the accumulated hidden states."""
s = _make_state(RequestOutputKind.DELTA)
# hidden state accumulation (nothing drained)
s.add_multimodal_tensor({"foo": torch.ones(5, 4)}, mm_type=LATENT)
result = s.make_request_output([0], None, FinishReason.STOP, None)
assert result is not None and not isinstance(result, PoolingRequestOutput)
hs = result.outputs[0].multimodal_output["foo"]
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 5
# Since we don't drain the hidden states, if we add 3 elements, we should get 8
s.add_multimodal_tensor({"foo": torch.ones(3, 4)}, mm_type=LATENT)
result = s.make_request_output([0], None, FinishReason.STOP, None)
assert result is not None and not isinstance(result, PoolingRequestOutput)
hs = result.outputs[0].multimodal_output["foo"]
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 8
assert "foo" in s.mm_accumulated
def test_finish_consolidation_drains_mm_delta():
"""Ensure making the request output drains modality deltas (e.g., audio)."""
s = _make_state(RequestOutputKind.DELTA)
# multimodal data accumulation (drained)
s.add_multimodal_tensor({AUDIO: torch.ones(5, 4)}, mm_type=AUDIO)
result = s.make_request_output([0], None, FinishReason.STOP, None)
assert result is not None and not isinstance(result, PoolingRequestOutput)
hs = result.outputs[0].multimodal_output[AUDIO]
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 5
# Since we did drain the hidden states, we no longer get the 5 back
s.add_multimodal_tensor({AUDIO: torch.ones(3, 4)}, mm_type=AUDIO)
result = s.make_request_output([0], None, FinishReason.STOP, None)
assert result is not None and not isinstance(result, PoolingRequestOutput)
hs = result.outputs[0].multimodal_output[AUDIO]
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 3
assert AUDIO not in s.mm_accumulated # drained
@pytest.mark.parametrize("mm_type", [AUDIO, "hidden"])
def test_final_only_consolidates_drainable_keys(mm_type):
"""FINAL_ONLY never drains per-step, so modality keys and hidden state
keys both accumulate and are consolidated on finish."""
s = _make_state(RequestOutputKind.FINAL_ONLY)
# NOTE: Currently there is brittlness in the tensor stacking, so we just
# test a 1D tensor here. The intention is just to ensure audio /hidden
# behave the same.
s.add_multimodal_tensor(torch.ones(500), mm_type=mm_type)
# Non-finish step returns None without calling _new_completion_output
assert s.make_request_output([1], None, None, None) is None
assert mm_type in s.mm_accumulated
s.add_multimodal_tensor(torch.ones(300), mm_type=mm_type)
result = s.make_request_output([2], None, FinishReason.STOP, None)
assert result is not None and not isinstance(result, PoolingRequestOutput)
audio = result.outputs[0].multimodal_output[mm_type]
assert isinstance(audio, torch.Tensor)
assert audio.shape == (800,)
def test_cumulative_token_ids_always_set():
"""cumulative_token_ids is set for all output kinds."""
for kind in (RequestOutputKind.DELTA, RequestOutputKind.CUMULATIVE, RequestOutputKind.FINAL_ONLY):
s = _make_state(kind)
out = s._new_completion_output([42], None, None)
assert hasattr(out, "cumulative_token_ids")
# The mock detokenizer has output_token_ids=[0]
assert list(out.cumulative_token_ids) == [0]
def test_cumulative_token_ids_is_a_copy():
"""cumulative_token_ids must be a snapshot, not a live reference."""
s = _make_state(RequestOutputKind.DELTA)
out = s._new_completion_output([42], None, None)
assert out.cumulative_token_ids is not _DETOK.output_token_ids
+134
View File
@@ -0,0 +1,134 @@
import asyncio
from types import SimpleNamespace
import pytest
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm_omni.entrypoints.async_omni import AsyncOmni
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
async def _noop(**kw):
pass
def get_fake_add_request(submitted_request_ids):
async def fake_add_request_async(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
del prompt, sampling_params_list, final_stage_id, kwargs
submitted_request_ids.append(request_id)
return fake_add_request_async
def get_fake_abort(aborted_request_batches):
async def fake_abort_async(request_ids):
aborted_request_batches.append(list(request_ids))
return fake_abort_async
async def fake_process_results(request_id, metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts):
del metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts
if request_id.startswith("cancel-"):
await asyncio.Future()
return
yield SimpleNamespace(
stage_id=0,
request_output=SimpleNamespace(outputs=[]),
finished=True,
)
def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) -> AsyncOmni:
omni = object.__new__(AsyncOmni)
omni._pause_cond = asyncio.Condition()
omni._paused = False
omni.engine = SimpleNamespace(
num_stages=1,
add_request_async=fake_add_request,
abort_async=fake_abort_request,
)
omni.log_stats = False
omni.request_states = {}
omni._final_output_handler = lambda: None
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
omni._compute_final_stage_id = lambda output_modalities: 0
omni._process_orchestrator_results = fake_process_results
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
return omni
def test_generate_accepts_request_after_repeated_cancellations():
async def run_test():
submitted_request_ids = []
aborted_request_batches = []
async def collect_outputs(request_id):
outputs = []
async for output in AsyncOmni.generate(
omni,
prompt={"prompt": "prompt"},
request_id=request_id,
sampling_params_list=[SimpleNamespace()],
output_modalities=["image"],
):
outputs.append(output)
return outputs
omni = get_async_omni_instance(
fake_add_request=get_fake_add_request(submitted_request_ids),
fake_abort_request=get_fake_abort(aborted_request_batches),
)
assert len(await collect_outputs("baseline")) == 1
for idx in range(3):
task = asyncio.create_task(collect_outputs(f"cancel-{idx}"))
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert len(await collect_outputs("after-cancel")) == 1
assert submitted_request_ids == [
"baseline",
"cancel-0",
"cancel-1",
"cancel-2",
"after-cancel",
]
assert aborted_request_batches == [
["cancel-0"],
["cancel-1"],
["cancel-2"],
]
asyncio.run(run_test())
@pytest.mark.parametrize(
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY, RequestOutputKind.CUMULATIVE]
)
def test_output_kind_is_preserved_with_explicit_sampling_params(output_kind):
"""Ensure we don't change the output kind in async generate if params are provided directly."""
captured_params = []
async def capturing_add_request(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
del prompt, final_stage_id, kwargs
captured_params.extend(sampling_params_list)
async def run():
omni = get_async_omni_instance(fake_add_request=capturing_add_request)
sp = SamplingParams(output_kind=output_kind)
async for _ in omni.generate(
prompt={"prompt": "test"},
request_id="test-req",
sampling_params_list=[sp],
output_modalities=["text"],
):
pass
asyncio.run(run())
assert captured_params[0].output_kind == output_kind
@@ -1,85 +0,0 @@
import asyncio
from types import SimpleNamespace
import pytest
from vllm_omni.entrypoints.async_omni import AsyncOmni
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_generate_accepts_request_after_repeated_cancellations():
async def run_test():
submitted_request_ids = []
aborted_request_batches = []
async def fake_add_request_async(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
del prompt, sampling_params_list, final_stage_id, kwargs
submitted_request_ids.append(request_id)
async def fake_abort_async(request_ids):
aborted_request_batches.append(list(request_ids))
async def fake_process_results(request_id, metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts):
del metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts
if request_id.startswith("cancel-"):
await asyncio.Future()
return
yield SimpleNamespace(
stage_id=0,
request_output=SimpleNamespace(outputs=[]),
finished=True,
)
async def collect_outputs(request_id):
outputs = []
async for output in AsyncOmni.generate(
omni,
prompt={"prompt": "prompt"},
request_id=request_id,
sampling_params_list=[SimpleNamespace()],
output_modalities=["image"],
):
outputs.append(output)
return outputs
omni = object.__new__(AsyncOmni)
omni._pause_cond = asyncio.Condition()
omni._paused = False
omni.engine = SimpleNamespace(
num_stages=1,
add_request_async=fake_add_request_async,
abort_async=fake_abort_async,
)
omni.log_stats = False
omni.request_states = {}
omni._final_output_handler = lambda: None
omni.resolve_sampling_params_list = lambda params: params
omni._compute_final_stage_id = lambda output_modalities: 0
omni._process_orchestrator_results = fake_process_results
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
assert len(await collect_outputs("baseline")) == 1
for idx in range(3):
task = asyncio.create_task(collect_outputs(f"cancel-{idx}"))
await asyncio.sleep(0)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert len(await collect_outputs("after-cancel")) == 1
assert submitted_request_ids == [
"baseline",
"cancel-0",
"cancel-1",
"cancel-2",
"after-cancel",
]
assert aborted_request_batches == [
["cancel-0"],
["cancel-1"],
["cancel-2"],
]
asyncio.run(run_test())
+43
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass
import pytest
import torch
from pytest_mock import MockerFixture
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm_omni.config.yaml_util import create_config
from vllm_omni.diffusion.data import OmniDiffusionConfig
@@ -15,6 +16,7 @@ from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.entrypoints.utils import (
_convert_dataclasses_to_dict,
_filter_dict_like_object,
coerce_param_message_types,
filter_dataclass_kwargs,
load_and_resolve_stage_configs,
load_stage_configs_from_yaml,
@@ -424,3 +426,44 @@ class TestLoadStageConfigsFromYaml:
assert stages[0]["engine_args"]["nested"]["base"] == 1
assert stages[0]["engine_args"]["nested"]["override"] == 2
class TestCumulativeStreamingCoercion:
@pytest.mark.parametrize("skip_clone", [True, False])
def test_cumulative_default_becomes_delta_if_stream(self, skip_clone):
"""Ensure cumulative messages are coercible to delta if streaming."""
sp = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
sp.skip_clone = skip_clone
result = coerce_param_message_types([sp], is_streaming=True)[0]
assert isinstance(result, SamplingParams)
assert result.output_kind == RequestOutputKind.DELTA
assert (skip_clone and sp is result) or (not skip_clone and sp is not result)
@pytest.mark.parametrize("skip_clone", [True, False])
def test_cumulative_default_becomes_final_only_if_not_stream(self, skip_clone):
"""Ensure cumulative messages are coercible to final only if not streaming."""
sp = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
sp.skip_clone = skip_clone
result = coerce_param_message_types([sp], is_streaming=False)[0]
assert isinstance(result, SamplingParams)
assert result.output_kind == RequestOutputKind.FINAL_ONLY
assert (skip_clone and sp is result) or (not skip_clone and sp is not result)
@pytest.mark.parametrize("is_streaming", [True, False])
@pytest.mark.parametrize("output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
def test_non_cumulative_are_coerced(self, output_kind, is_streaming):
"""Ensure non-cumulative params are coerced to the target type."""
sp = SamplingParams(output_kind=output_kind)
expected = RequestOutputKind.DELTA if is_streaming else RequestOutputKind.FINAL_ONLY
result = coerce_param_message_types([sp], is_streaming=is_streaming)[0]
assert isinstance(result, SamplingParams)
assert result.output_kind == expected
def test_coercion_applies_to_all_stages(self):
"""Ensure all stages are coerced to DELTA for streaming."""
sp0 = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
sp1 = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
result = coerce_param_message_types([sp0, sp1], is_streaming=True)
assert all([isinstance(r, SamplingParams) for r in result])
assert result[0].output_kind == RequestOutputKind.DELTA
assert result[1].output_kind == RequestOutputKind.DELTA
@@ -13,7 +13,7 @@ def _source_output(request_id: str, prompt_ids: list[int], out_ids: list[int], m
return SimpleNamespace(
request_id=request_id,
prompt_token_ids=prompt_ids,
outputs=[SimpleNamespace(token_ids=out_ids, multimodal_output=mm)],
outputs=[SimpleNamespace(token_ids=out_ids, cumulative_token_ids=out_ids, multimodal_output=mm)],
)
@@ -27,7 +27,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _source_output(token_ids: list[int], mm_output: dict | None = None):
"""Create a minimal AR output mock."""
return SimpleNamespace(
outputs=[SimpleNamespace(token_ids=token_ids)],
outputs=[SimpleNamespace(token_ids=token_ids, cumulative_token_ids=token_ids)],
multimodal_output=mm_output,
)
@@ -336,6 +336,7 @@ def test_non_async_processor_prepends_ref_code_and_sets_trim_context():
output = SimpleNamespace(
multimodal_output={"codes": {"audio": audio_codes, "ref": ref_code}},
token_ids=list(range(3)),
cumulative_token_ids=list(range(3)),
)
stage = SimpleNamespace(
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
@@ -381,6 +382,7 @@ def test_non_async_processor_filters_out_of_range_codec_values():
output = SimpleNamespace(
multimodal_output={"codes": {"audio": audio_codes, "ref": ref_code}},
token_ids=list(range(4)),
cumulative_token_ids=list(range(4)),
)
stage = SimpleNamespace(
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],