[Refactor/Bugfix] Use Delta Messages for Streaming (#2911)
Signed-off-by: Alex Brooks <albrooks@redhat.com> Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Sy03 <1370724210@qq.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
27 files changed
+623
-177
No files matched your search
@@ -82,6 +82,30 @@ class TestVoxtralTTSFixedVoice:
|
||||
f"Audio content too small ({len(response.content)} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_speech_english_streaming(self, omni_server) -> None:
|
||||
"""Test basic streaming English TTS generation."""
|
||||
url = f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech"
|
||||
payload = {
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "casual_female",
|
||||
"language": "English",
|
||||
"stream": True,
|
||||
"response_format": "pcm",
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
with client.stream("POST", url, json=payload) as response:
|
||||
assert response.status_code == 200
|
||||
assert response.headers.get("content-type") == "audio/pcm"
|
||||
total = sum(len(c) for c in response.iter_bytes())
|
||||
|
||||
assert total > MIN_AUDIO_BYTES, (
|
||||
f"Streamed audio too small ({total} bytes), expected at least {MIN_AUDIO_BYTES} bytes"
|
||||
)
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for OmniRequestState multimodal DELTA drain and consolidation guard."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.engine.output_modality import OutputModalityNames
|
||||
from vllm_omni.engine.output_processor import OmniRequestState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# Audio is explicitly listed as a drainable modality
|
||||
AUDIO = OutputModalityNames.AUDIO
|
||||
|
||||
# Latent is explicitly not drainable, but the choice doesn't matter here as
|
||||
# long as isn't listed as drainable. I.e., could also be arbitrary keys for
|
||||
# the purposes of these tests
|
||||
LATENT = OutputModalityNames.LATENT
|
||||
|
||||
# NOTE: detokenizer and logprobs aren't really used here, but we mock them since
|
||||
# some of the utils called in vLLM superclass assert require them to be None.
|
||||
_DETOK = MagicMock(
|
||||
output_token_ids=[0],
|
||||
get_next_output_text=MagicMock(return_value=""),
|
||||
num_output_tokens=MagicMock(return_value=1),
|
||||
)
|
||||
_LOGPROBS = MagicMock(logprobs=None, cumulative_logprob=None, prompt_logprobs=None)
|
||||
|
||||
_DEFAULT_STATE_KWARGS = dict(
|
||||
request_id="r",
|
||||
external_req_id="r",
|
||||
parent_req=None,
|
||||
request_index=0,
|
||||
lora_request=None,
|
||||
prompt=None,
|
||||
prompt_token_ids=[0],
|
||||
prompt_embeds=None,
|
||||
logprobs_processor=_LOGPROBS,
|
||||
detokenizer=_DETOK,
|
||||
max_tokens_param=None,
|
||||
arrival_time=0.0,
|
||||
queue=None,
|
||||
log_stats=False,
|
||||
stream_interval=1,
|
||||
)
|
||||
|
||||
|
||||
def _make_state(output_kind: RequestOutputKind):
|
||||
return OmniRequestState(**_DEFAULT_STATE_KWARGS, output_kind=output_kind)
|
||||
|
||||
|
||||
def test_init_empty_dict():
|
||||
"""Ensure mm_accumulated is initially empty."""
|
||||
assert _make_state(RequestOutputKind.CUMULATIVE).mm_accumulated == {}
|
||||
assert _make_state(RequestOutputKind.DELTA).mm_accumulated == {}
|
||||
|
||||
|
||||
def test_delta_drains_output_modality_per_step():
|
||||
"""DELTA drains the mm_type key (output modality) but preserves hidden-state keys."""
|
||||
s = _make_state(RequestOutputKind.DELTA)
|
||||
audio1, audio2, hs1, hs2 = [torch.ones(num_elem) for num_elem in range(1, 5)]
|
||||
|
||||
# Add audio and hidden state tensors
|
||||
s.add_multimodal_tensor(audio1, mm_type=AUDIO) # should be drained
|
||||
s.add_multimodal_tensor(hs1, mm_type=LATENT) # shouldn't be drained
|
||||
|
||||
out1 = s._new_completion_output([1], None, None)
|
||||
out1_audio = out1.multimodal_output[AUDIO]
|
||||
out1_hidden = out1.multimodal_output[LATENT]
|
||||
assert isinstance(out1_audio, torch.Tensor)
|
||||
assert torch.equal(out1.multimodal_output[AUDIO], audio1)
|
||||
assert isinstance(out1_hidden, torch.Tensor)
|
||||
assert torch.equal(out1.multimodal_output[LATENT], hs1)
|
||||
|
||||
# After emission, hidden states should remain, but audio is drained
|
||||
assert set(s.mm_accumulated.keys()) == {LATENT}
|
||||
|
||||
s.add_multimodal_tensor(audio2, AUDIO)
|
||||
s.add_multimodal_tensor(hs2, mm_type=LATENT)
|
||||
out2 = s._new_completion_output([2], None, None)
|
||||
out2_audio = out2.multimodal_output[AUDIO]
|
||||
out2_hidden = out2.multimodal_output[LATENT]
|
||||
assert isinstance(out2_audio, torch.Tensor)
|
||||
assert torch.equal(out2_audio, audio2)
|
||||
# Since hidden isn't drained, it's grown to a list
|
||||
assert isinstance(out2_hidden, list) and len(out2_hidden) == 2
|
||||
assert torch.equal(hs1, out2_hidden[0])
|
||||
assert torch.equal(hs2, out2_hidden[1])
|
||||
|
||||
|
||||
def test_cumulative_emits_consolidated_audio_each_step():
|
||||
"""Ensure cumulative accumulates and consolidates modality keys every step."""
|
||||
s = _make_state(RequestOutputKind.CUMULATIVE)
|
||||
# NOTE: audio is usually emitted as (1, size) chunks; we need to be sure
|
||||
# to not change the tensor dimension when we consolidate
|
||||
audio1 = torch.ones(1, 500)
|
||||
s.add_multimodal_tensor(audio1, mm_type=AUDIO)
|
||||
req_out = s.make_request_output([1], None, None, None)
|
||||
assert req_out is not None
|
||||
cons_audio = req_out.outputs[0].multimodal_output[AUDIO]
|
||||
# Single chunk keeps original shape [1, 500]
|
||||
assert isinstance(cons_audio, torch.Tensor) and cons_audio.shape == audio1.shape
|
||||
|
||||
audio2 = torch.ones(1, 300)
|
||||
s.add_multimodal_tensor(audio2, mm_type=AUDIO)
|
||||
req_out = s.make_request_output([2], None, None, None)
|
||||
assert req_out is not None
|
||||
cons_audio = req_out.outputs[0].multimodal_output[AUDIO]
|
||||
# After consolidation, audio chunks are concatenated on last axis,
|
||||
# preserving the [1, N] channel dimension
|
||||
total_audio_len = audio1.shape[-1] + audio2.shape[-1]
|
||||
assert isinstance(audio2, torch.Tensor) and cons_audio.shape == (1, total_audio_len)
|
||||
|
||||
assert "audio" in s.mm_accumulated
|
||||
|
||||
|
||||
def test_finish_consolidates_hidden_states():
|
||||
"""Ensure consolidation merges hidden-state tensor lists on finish."""
|
||||
s = _make_state(RequestOutputKind.CUMULATIVE)
|
||||
s.add_multimodal_tensor(torch.ones(5, 4), mm_type=LATENT)
|
||||
s.add_multimodal_tensor(torch.ones(3, 4), mm_type=LATENT)
|
||||
|
||||
result = s.make_request_output([1], None, FinishReason.STOP, None)
|
||||
assert result is not None and not isinstance(result, PoolingRequestOutput)
|
||||
|
||||
hs = result.outputs[0].multimodal_output[LATENT]
|
||||
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 8
|
||||
|
||||
|
||||
def test_finish_consolidation_for_hs_delta():
|
||||
"""Ensure finish doesn't drop the accumulated hidden states."""
|
||||
s = _make_state(RequestOutputKind.DELTA)
|
||||
# hidden state accumulation (nothing drained)
|
||||
s.add_multimodal_tensor({"foo": torch.ones(5, 4)}, mm_type=LATENT)
|
||||
result = s.make_request_output([0], None, FinishReason.STOP, None)
|
||||
assert result is not None and not isinstance(result, PoolingRequestOutput)
|
||||
hs = result.outputs[0].multimodal_output["foo"]
|
||||
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 5
|
||||
|
||||
# Since we don't drain the hidden states, if we add 3 elements, we should get 8
|
||||
s.add_multimodal_tensor({"foo": torch.ones(3, 4)}, mm_type=LATENT)
|
||||
result = s.make_request_output([0], None, FinishReason.STOP, None)
|
||||
assert result is not None and not isinstance(result, PoolingRequestOutput)
|
||||
hs = result.outputs[0].multimodal_output["foo"]
|
||||
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 8
|
||||
assert "foo" in s.mm_accumulated
|
||||
|
||||
|
||||
def test_finish_consolidation_drains_mm_delta():
|
||||
"""Ensure making the request output drains modality deltas (e.g., audio)."""
|
||||
s = _make_state(RequestOutputKind.DELTA)
|
||||
# multimodal data accumulation (drained)
|
||||
s.add_multimodal_tensor({AUDIO: torch.ones(5, 4)}, mm_type=AUDIO)
|
||||
result = s.make_request_output([0], None, FinishReason.STOP, None)
|
||||
assert result is not None and not isinstance(result, PoolingRequestOutput)
|
||||
hs = result.outputs[0].multimodal_output[AUDIO]
|
||||
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 5
|
||||
|
||||
# Since we did drain the hidden states, we no longer get the 5 back
|
||||
s.add_multimodal_tensor({AUDIO: torch.ones(3, 4)}, mm_type=AUDIO)
|
||||
result = s.make_request_output([0], None, FinishReason.STOP, None)
|
||||
assert result is not None and not isinstance(result, PoolingRequestOutput)
|
||||
hs = result.outputs[0].multimodal_output[AUDIO]
|
||||
assert isinstance(hs, torch.Tensor) and hs.shape[0] == 3
|
||||
assert AUDIO not in s.mm_accumulated # drained
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mm_type", [AUDIO, "hidden"])
|
||||
def test_final_only_consolidates_drainable_keys(mm_type):
|
||||
"""FINAL_ONLY never drains per-step, so modality keys and hidden state
|
||||
keys both accumulate and are consolidated on finish."""
|
||||
s = _make_state(RequestOutputKind.FINAL_ONLY)
|
||||
|
||||
# NOTE: Currently there is brittlness in the tensor stacking, so we just
|
||||
# test a 1D tensor here. The intention is just to ensure audio /hidden
|
||||
# behave the same.
|
||||
s.add_multimodal_tensor(torch.ones(500), mm_type=mm_type)
|
||||
# Non-finish step returns None without calling _new_completion_output
|
||||
assert s.make_request_output([1], None, None, None) is None
|
||||
assert mm_type in s.mm_accumulated
|
||||
|
||||
s.add_multimodal_tensor(torch.ones(300), mm_type=mm_type)
|
||||
result = s.make_request_output([2], None, FinishReason.STOP, None)
|
||||
assert result is not None and not isinstance(result, PoolingRequestOutput)
|
||||
|
||||
audio = result.outputs[0].multimodal_output[mm_type]
|
||||
assert isinstance(audio, torch.Tensor)
|
||||
assert audio.shape == (800,)
|
||||
|
||||
|
||||
def test_cumulative_token_ids_always_set():
|
||||
"""cumulative_token_ids is set for all output kinds."""
|
||||
for kind in (RequestOutputKind.DELTA, RequestOutputKind.CUMULATIVE, RequestOutputKind.FINAL_ONLY):
|
||||
s = _make_state(kind)
|
||||
out = s._new_completion_output([42], None, None)
|
||||
assert hasattr(out, "cumulative_token_ids")
|
||||
# The mock detokenizer has output_token_ids=[0]
|
||||
assert list(out.cumulative_token_ids) == [0]
|
||||
|
||||
|
||||
def test_cumulative_token_ids_is_a_copy():
|
||||
"""cumulative_token_ids must be a snapshot, not a live reference."""
|
||||
s = _make_state(RequestOutputKind.DELTA)
|
||||
out = s._new_completion_output([42], None, None)
|
||||
assert out.cumulative_token_ids is not _DETOK.output_token_ids
|
||||
@@ -0,0 +1,134 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
async def _noop(**kw):
|
||||
pass
|
||||
|
||||
|
||||
def get_fake_add_request(submitted_request_ids):
|
||||
async def fake_add_request_async(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
|
||||
del prompt, sampling_params_list, final_stage_id, kwargs
|
||||
submitted_request_ids.append(request_id)
|
||||
|
||||
return fake_add_request_async
|
||||
|
||||
|
||||
def get_fake_abort(aborted_request_batches):
|
||||
async def fake_abort_async(request_ids):
|
||||
aborted_request_batches.append(list(request_ids))
|
||||
|
||||
return fake_abort_async
|
||||
|
||||
|
||||
async def fake_process_results(request_id, metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts):
|
||||
del metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts
|
||||
if request_id.startswith("cancel-"):
|
||||
await asyncio.Future()
|
||||
return
|
||||
yield SimpleNamespace(
|
||||
stage_id=0,
|
||||
request_output=SimpleNamespace(outputs=[]),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
|
||||
def get_async_omni_instance(fake_add_request=_noop, fake_abort_request=_noop) -> AsyncOmni:
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni._pause_cond = asyncio.Condition()
|
||||
omni._paused = False
|
||||
omni.engine = SimpleNamespace(
|
||||
num_stages=1,
|
||||
add_request_async=fake_add_request,
|
||||
abort_async=fake_abort_request,
|
||||
)
|
||||
omni.log_stats = False
|
||||
omni.request_states = {}
|
||||
omni._final_output_handler = lambda: None
|
||||
omni.resolve_sampling_params_list = lambda params, allow_delta_coercion: params
|
||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||
omni._process_orchestrator_results = fake_process_results
|
||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||
return omni
|
||||
|
||||
|
||||
def test_generate_accepts_request_after_repeated_cancellations():
|
||||
async def run_test():
|
||||
submitted_request_ids = []
|
||||
aborted_request_batches = []
|
||||
|
||||
async def collect_outputs(request_id):
|
||||
outputs = []
|
||||
async for output in AsyncOmni.generate(
|
||||
omni,
|
||||
prompt={"prompt": "prompt"},
|
||||
request_id=request_id,
|
||||
sampling_params_list=[SimpleNamespace()],
|
||||
output_modalities=["image"],
|
||||
):
|
||||
outputs.append(output)
|
||||
return outputs
|
||||
|
||||
omni = get_async_omni_instance(
|
||||
fake_add_request=get_fake_add_request(submitted_request_ids),
|
||||
fake_abort_request=get_fake_abort(aborted_request_batches),
|
||||
)
|
||||
|
||||
assert len(await collect_outputs("baseline")) == 1
|
||||
|
||||
for idx in range(3):
|
||||
task = asyncio.create_task(collect_outputs(f"cancel-{idx}"))
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert len(await collect_outputs("after-cancel")) == 1
|
||||
assert submitted_request_ids == [
|
||||
"baseline",
|
||||
"cancel-0",
|
||||
"cancel-1",
|
||||
"cancel-2",
|
||||
"after-cancel",
|
||||
]
|
||||
assert aborted_request_batches == [
|
||||
["cancel-0"],
|
||||
["cancel-1"],
|
||||
["cancel-2"],
|
||||
]
|
||||
|
||||
asyncio.run(run_test())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY, RequestOutputKind.CUMULATIVE]
|
||||
)
|
||||
def test_output_kind_is_preserved_with_explicit_sampling_params(output_kind):
|
||||
"""Ensure we don't change the output kind in async generate if params are provided directly."""
|
||||
|
||||
captured_params = []
|
||||
|
||||
async def capturing_add_request(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
|
||||
del prompt, final_stage_id, kwargs
|
||||
captured_params.extend(sampling_params_list)
|
||||
|
||||
async def run():
|
||||
omni = get_async_omni_instance(fake_add_request=capturing_add_request)
|
||||
sp = SamplingParams(output_kind=output_kind)
|
||||
async for _ in omni.generate(
|
||||
prompt={"prompt": "test"},
|
||||
request_id="test-req",
|
||||
sampling_params_list=[sp],
|
||||
output_modalities=["text"],
|
||||
):
|
||||
pass
|
||||
|
||||
asyncio.run(run())
|
||||
assert captured_params[0].output_kind == output_kind
|
||||
@@ -1,85 +0,0 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_generate_accepts_request_after_repeated_cancellations():
|
||||
async def run_test():
|
||||
submitted_request_ids = []
|
||||
aborted_request_batches = []
|
||||
|
||||
async def fake_add_request_async(*, request_id, prompt, sampling_params_list, final_stage_id, **kwargs):
|
||||
del prompt, sampling_params_list, final_stage_id, kwargs
|
||||
submitted_request_ids.append(request_id)
|
||||
|
||||
async def fake_abort_async(request_ids):
|
||||
aborted_request_batches.append(list(request_ids))
|
||||
|
||||
async def fake_process_results(request_id, metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts):
|
||||
del metrics, final_stage_id_for_e2e, req_start_ts, wall_start_ts
|
||||
if request_id.startswith("cancel-"):
|
||||
await asyncio.Future()
|
||||
return
|
||||
yield SimpleNamespace(
|
||||
stage_id=0,
|
||||
request_output=SimpleNamespace(outputs=[]),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
async def collect_outputs(request_id):
|
||||
outputs = []
|
||||
async for output in AsyncOmni.generate(
|
||||
omni,
|
||||
prompt={"prompt": "prompt"},
|
||||
request_id=request_id,
|
||||
sampling_params_list=[SimpleNamespace()],
|
||||
output_modalities=["image"],
|
||||
):
|
||||
outputs.append(output)
|
||||
return outputs
|
||||
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni._pause_cond = asyncio.Condition()
|
||||
omni._paused = False
|
||||
omni.engine = SimpleNamespace(
|
||||
num_stages=1,
|
||||
add_request_async=fake_add_request_async,
|
||||
abort_async=fake_abort_async,
|
||||
)
|
||||
omni.log_stats = False
|
||||
omni.request_states = {}
|
||||
omni._final_output_handler = lambda: None
|
||||
omni.resolve_sampling_params_list = lambda params: params
|
||||
omni._compute_final_stage_id = lambda output_modalities: 0
|
||||
omni._process_orchestrator_results = fake_process_results
|
||||
omni._log_summary_and_cleanup = lambda request_id: omni.request_states.pop(request_id, None)
|
||||
|
||||
assert len(await collect_outputs("baseline")) == 1
|
||||
|
||||
for idx in range(3):
|
||||
task = asyncio.create_task(collect_outputs(f"cancel-{idx}"))
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert len(await collect_outputs("after-cancel")) == 1
|
||||
assert submitted_request_ids == [
|
||||
"baseline",
|
||||
"cancel-0",
|
||||
"cancel-1",
|
||||
"cancel-2",
|
||||
"after-cancel",
|
||||
]
|
||||
assert aborted_request_batches == [
|
||||
["cancel-0"],
|
||||
["cancel-1"],
|
||||
["cancel-2"],
|
||||
]
|
||||
|
||||
asyncio.run(run_test())
|
||||
@@ -7,6 +7,7 @@ from dataclasses import dataclass
|
||||
import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
|
||||
from vllm_omni.config.yaml_util import create_config
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
@@ -15,6 +16,7 @@ from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.entrypoints.utils import (
|
||||
_convert_dataclasses_to_dict,
|
||||
_filter_dict_like_object,
|
||||
coerce_param_message_types,
|
||||
filter_dataclass_kwargs,
|
||||
load_and_resolve_stage_configs,
|
||||
load_stage_configs_from_yaml,
|
||||
@@ -424,3 +426,44 @@ class TestLoadStageConfigsFromYaml:
|
||||
|
||||
assert stages[0]["engine_args"]["nested"]["base"] == 1
|
||||
assert stages[0]["engine_args"]["nested"]["override"] == 2
|
||||
|
||||
|
||||
class TestCumulativeStreamingCoercion:
|
||||
@pytest.mark.parametrize("skip_clone", [True, False])
|
||||
def test_cumulative_default_becomes_delta_if_stream(self, skip_clone):
|
||||
"""Ensure cumulative messages are coercible to delta if streaming."""
|
||||
sp = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
|
||||
sp.skip_clone = skip_clone
|
||||
result = coerce_param_message_types([sp], is_streaming=True)[0]
|
||||
assert isinstance(result, SamplingParams)
|
||||
assert result.output_kind == RequestOutputKind.DELTA
|
||||
assert (skip_clone and sp is result) or (not skip_clone and sp is not result)
|
||||
|
||||
@pytest.mark.parametrize("skip_clone", [True, False])
|
||||
def test_cumulative_default_becomes_final_only_if_not_stream(self, skip_clone):
|
||||
"""Ensure cumulative messages are coercible to final only if not streaming."""
|
||||
sp = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
|
||||
sp.skip_clone = skip_clone
|
||||
result = coerce_param_message_types([sp], is_streaming=False)[0]
|
||||
assert isinstance(result, SamplingParams)
|
||||
assert result.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
assert (skip_clone and sp is result) or (not skip_clone and sp is not result)
|
||||
|
||||
@pytest.mark.parametrize("is_streaming", [True, False])
|
||||
@pytest.mark.parametrize("output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
|
||||
def test_non_cumulative_are_coerced(self, output_kind, is_streaming):
|
||||
"""Ensure non-cumulative params are coerced to the target type."""
|
||||
sp = SamplingParams(output_kind=output_kind)
|
||||
expected = RequestOutputKind.DELTA if is_streaming else RequestOutputKind.FINAL_ONLY
|
||||
result = coerce_param_message_types([sp], is_streaming=is_streaming)[0]
|
||||
assert isinstance(result, SamplingParams)
|
||||
assert result.output_kind == expected
|
||||
|
||||
def test_coercion_applies_to_all_stages(self):
|
||||
"""Ensure all stages are coerced to DELTA for streaming."""
|
||||
sp0 = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
|
||||
sp1 = SamplingParams(output_kind=RequestOutputKind.CUMULATIVE)
|
||||
result = coerce_param_message_types([sp0, sp1], is_streaming=True)
|
||||
assert all([isinstance(r, SamplingParams) for r in result])
|
||||
assert result[0].output_kind == RequestOutputKind.DELTA
|
||||
assert result[1].output_kind == RequestOutputKind.DELTA
|
||||
+1
-1
@@ -13,7 +13,7 @@ def _source_output(request_id: str, prompt_ids: list[int], out_ids: list[int], m
|
||||
return SimpleNamespace(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_ids,
|
||||
outputs=[SimpleNamespace(token_ids=out_ids, multimodal_output=mm)],
|
||||
outputs=[SimpleNamespace(token_ids=out_ids, cumulative_token_ids=out_ids, multimodal_output=mm)],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
def _source_output(token_ids: list[int], mm_output: dict | None = None):
|
||||
"""Create a minimal AR output mock."""
|
||||
return SimpleNamespace(
|
||||
outputs=[SimpleNamespace(token_ids=token_ids)],
|
||||
outputs=[SimpleNamespace(token_ids=token_ids, cumulative_token_ids=token_ids)],
|
||||
multimodal_output=mm_output,
|
||||
)
|
||||
|
||||
|
||||
@@ -336,6 +336,7 @@ def test_non_async_processor_prepends_ref_code_and_sets_trim_context():
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"codes": {"audio": audio_codes, "ref": ref_code}},
|
||||
token_ids=list(range(3)),
|
||||
cumulative_token_ids=list(range(3)),
|
||||
)
|
||||
stage = SimpleNamespace(
|
||||
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
|
||||
@@ -381,6 +382,7 @@ def test_non_async_processor_filters_out_of_range_codec_values():
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"codes": {"audio": audio_codes, "ref": ref_code}},
|
||||
token_ids=list(range(4)),
|
||||
cumulative_token_ids=list(range(4)),
|
||||
)
|
||||
stage = SimpleNamespace(
|
||||
engine_outputs=[SimpleNamespace(outputs=[output], finished=True)],
|
||||
|
||||
Reference in new issue
Block a user