[Feat]audio streaming input for async chunk (#3614)

Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
This commit is contained in:
ChenWenjing authored and GitHub committed 2026-05-31 17:39:25 +08:00
1 parent 6f4bd3e45c
commit 5dfdf584dd
20 files changed
+536 -184

No files matched your search

@@ -0,0 +1,92 @@
"""Unit tests for Omni AR streaming-session async placeholder handling."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request before
# Request / StreamingUpdate are bound in this module. Ruff isort would reorder them.
# isort: off
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
# isort: on
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_scheduler(*, stage_id: int = 0) -> OmniARScheduler:
sched = OmniARScheduler.__new__(OmniARScheduler)
sched._new_prompt_len_snapshot = {}
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=stage_id))
sched.num_waiting_for_streaming_input = 0
sched.log_stats = False
sched.chunk_transfer_adapter = None
return sched
def _make_request() -> Request:
return Request(
request_id="req-ar-streaming-test",
prompt_token_ids=[1, 2, 3],
sampling_params=SamplingParams(max_tokens=8),
pooling_params=None,
arrival_time=100.0,
block_hasher=None,
)
def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate:
return StreamingUpdate(
mm_features=None,
prompt_token_ids=[10, 20] if prompt_token_ids is None else prompt_token_ids,
max_tokens=32,
arrival_time=200.0,
sampling_params=SamplingParams(max_tokens=16),
)
def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None:
sched = _make_scheduler(stage_id=0)
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids([7, 8, 9])
session.num_computed_tokens = 6
session.num_output_placeholders = 1
session.spec_token_ids = [-1]
sched._update_request_as_session(session, _make_update([10, 20]))
assert session.discard_latest_async_tokens is True
assert session.num_output_placeholders == 0
assert session.spec_token_ids == []
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
# carried into the next streaming prompt before the new chunk tokens.
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 10, 20]
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 10, 20]
assert session._output_token_ids == []
assert session.num_prompt_tokens == 7
assert sched._new_prompt_len_snapshot[session.request_id] == 2
def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder() -> None:
sched = _make_scheduler(stage_id=0)
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids([7, 8, 9])
session.num_computed_tokens = 6
session.num_output_placeholders = 0
sched._update_request_as_session(session, _make_update([10, 20]))
assert session.discard_latest_async_tokens is False
assert session.num_output_placeholders == 0
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
assert session._output_token_ids == []
assert session.num_prompt_tokens == 8
assert sched._new_prompt_len_snapshot[session.request_id] == 2
@@ -1,6 +1,6 @@
"""Unit tests for OmniSchedulerMixin streaming session replacement.
"""Unit tests for generation streaming session replacement.
These tests pin the behavior of `_replace_session_with_streaming_update` against
These tests pin the behavior of `_update_request_as_session` against
current vLLM `Request` / `StreamingUpdate` (and Omni patches). When upgrading
vLLM, failures here should highlight incompatible changes to request state or
update payloads early.
@@ -19,19 +19,34 @@ import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
from vllm.sampling_params import SamplingParams
from vllm.v1.engine import EngineCoreEventType
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
# isort: on
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _SchedulerStub(OmniSchedulerMixin):
"""Minimal scheduler surface required by OmniSchedulerMixin."""
class _SkippedWaitingStub:
def __contains__(self, request: Request) -> bool:
return False
class _ChunkTransferAdapterStub:
def __init__(self) -> None:
self.segment_finished_requests: set[str] = set()
class _SchedulerStub(OmniGenerationScheduler):
"""Minimal scheduler surface required by OmniGenerationScheduler."""
def __init__(self, *, log_stats: bool = False) -> None:
self.num_waiting_for_streaming_input = 0
self.log_stats = log_stats
self.chunk_transfer_adapter = _ChunkTransferAdapterStub()
self.skipped_waiting = _SkippedWaitingStub()
def _enqueue_waiting_request(self, session: Request) -> None:
raise AssertionError("unexpected enqueue for skipped_waiting miss")
def _make_request(**kwargs) -> Request:
@@ -71,7 +86,7 @@ class TestReplaceSessionWithStreamingUpdate:
update = _make_update(prompt_token_ids=[40, 41, 42])
sched.num_waiting_for_streaming_input = 3
sched._replace_session_with_streaming_update(session, update)
sched._update_request_as_session(session, update)
assert session._output_token_ids == []
assert list(session._all_token_ids) == [40, 41, 42]
@@ -88,7 +103,7 @@ class TestReplaceSessionWithStreamingUpdate:
session = _make_request()
session.status = RequestStatus.RUNNING
update = _make_update(prompt_token_ids=None)
sched._replace_session_with_streaming_update(session, update)
sched._update_request_as_session(session, update)
assert session.prompt_token_ids == ()
assert list(session._all_token_ids) == []
@@ -108,7 +123,7 @@ class TestReplaceSessionWithStreamingUpdate:
pytest.skip("StreamingUpdate has no additional_information (Omni patch inactive?)")
update = replace(base, additional_information=None)
sched._replace_session_with_streaming_update(session, update)
sched._update_request_as_session(session, update)
assert session.additional_information is None
def test_does_not_decrement_waiting_when_not_streaming_status(self) -> None:
@@ -116,14 +131,14 @@ class TestReplaceSessionWithStreamingUpdate:
session = _make_request()
session.status = RequestStatus.RUNNING
sched.num_waiting_for_streaming_input = 5
sched._replace_session_with_streaming_update(session, _make_update())
sched._update_request_as_session(session, _make_update())
assert sched.num_waiting_for_streaming_input == 5
def test_records_queued_event_when_log_stats_enabled(self) -> None:
sched = _SchedulerStub(log_stats=True)
session = _make_request()
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
sched._replace_session_with_streaming_update(session, _make_update())
sched._update_request_as_session(session, _make_update())
assert session.events
assert session.events[-1].type == EngineCoreEventType.QUEUED
@@ -230,9 +230,11 @@ def test_send_single_request_struct_without_meta_does_not_crash(build_adapter, m
cleanup_calls = []
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: cleanup_calls.append((a, kw)))
adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": False})
adapter._send_single_request(
{"pooling_output": None, "request": request, "is_finished": False, "is_segment_finished": False}
)
assert cleanup_calls == [] # no terminal cleanup; meta.finished is unobservable
assert cleanup_calls == [] # no terminal cleanup; meta.finished is false
def test_send_single_request_empty_struct_goes_on_wire(build_adapter, monkeypatch):
@@ -248,12 +250,50 @@ def test_send_single_request_empty_struct_goes_on_wire(build_adapter, monkeypatc
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct()
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None)
adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": False})
adapter._send_single_request(
{"pooling_output": None, "request": request, "is_finished": False, "is_segment_finished": False}
)
assert connector.put.called
sent_payload = connector.put.call_args.kwargs["data"]
assert isinstance(sent_payload, OmniPayloadStruct)
assert sent_payload.meta is None # confirms it's the empty struct on the wire
assert sent_payload.meta.finished.item() is False
assert sent_payload.meta.is_segment_finished.item() is False
def test_send_single_request_struct_preserves_segment_finished(build_adapter, monkeypatch):
adapter, connector = build_adapter(stage_id=1)
request = _req("req-segment", RequestStatus.WAITING, external_req_id="ext-segment")
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct()
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None)
adapter._send_single_request(
{"pooling_output": None, "request": request, "is_finished": False, "is_segment_finished": True}
)
sent_payload = connector.put.call_args.kwargs["data"]
assert sent_payload.meta.finished.item() is False
assert sent_payload.meta.is_segment_finished.item() is True
def test_save_async_skips_stale_resumable_chunk_until_dedup_is_reset(build_adapter):
adapter, _ = build_adapter(stage_id=1)
request = _req("req-stream", RequestStatus.WAITING, external_req_id="ext-stream")
request.resumable = True
request.num_computed_tokens = 0
adapter.requests_num_chunks_sent["ext-stream"] = 111
adapter.save_async(pooling_output=None, request=request, is_segment_finished=False)
assert len(adapter._pending_save_reqs) == 0
assert adapter.requests_num_chunks_sent["ext-stream"] == 111
adapter.requests_num_chunks_sent.pop("ext-stream")
adapter.save_async(pooling_output=None, request=request, is_segment_finished=False)
assert len(adapter._pending_save_reqs) == 1
assert adapter.requests_num_chunks_sent["ext-stream"] == 0
def test_send_single_request_cleans_up_after_finished_payload(build_adapter, monkeypatch):
@@ -266,7 +306,9 @@ def test_send_single_request_cleans_up_after_finished_payload(build_adapter, mon
cleanup_calls = []
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: cleanup_calls.append((a, kw)))
adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": True})
adapter._send_single_request(
{"pooling_output": None, "request": request, "is_finished": True, "is_segment_finished": True}
)
assert len(cleanup_calls) == 1
args, _ = cleanup_calls[0]
+1
View File
@@ -964,6 +964,7 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non
pool = RecordingPool()
orchestrator = object.__new__(Orchestrator)
orchestrator.async_chunk = False
orchestrator.request_states = {
"req-stream": OrchestratorRequestState(
request_id="req-stream",
@@ -44,7 +44,6 @@ realtime_server_params = [
model=MODEL,
stage_config_path=default_stage_config,
use_stage_cli=True,
server_args=["--no-async-chunk"],
),
id="default",
),
@@ -20,7 +20,7 @@ def _streaming_context() -> SimpleNamespace:
def test_get_streaming_talker_tokens_first_segment(_streaming_context: SimpleNamespace) -> None:
inc_p, inc_o, merged, thinker_in = q3._get_streaming_talker_tokens(
inc_p, inc_o = q3._get_streaming_talker_tokens(
"r1",
[1, 2],
[10, 11],
@@ -28,13 +28,11 @@ def test_get_streaming_talker_tokens_first_segment(_streaming_context: SimpleNam
)
assert inc_p == [1, 2]
assert inc_o == [10, 11]
assert merged == [1, 2, 10, 11]
assert thinker_in == [1, 2]
def test_get_streaming_talker_tokens_second_segment_accumulates(_streaming_context: SimpleNamespace) -> None:
q3._get_streaming_talker_tokens("r2", [1, 2], [10, 11], streaming_context=_streaming_context)
inc_p, inc_o, merged, thinker_in = q3._get_streaming_talker_tokens(
inc_p, inc_o = q3._get_streaming_talker_tokens(
"r2",
[1, 2, 3, 4],
[10, 11, 12, 13],
@@ -42,14 +40,12 @@ def test_get_streaming_talker_tokens_second_segment_accumulates(_streaming_conte
)
assert inc_p == [3, 4]
assert inc_o == [12, 13]
assert merged == [1, 2, 10, 3, 4, 12, 13]
assert thinker_in == [1, 2, 10, 3, 4]
def test_get_streaming_talker_tokens_new_prompt_len_snapshot_truncates(
_streaming_context: SimpleNamespace,
) -> None:
inc_p, inc_o, merged, thinker_in = q3._get_streaming_talker_tokens(
inc_p, inc_o = q3._get_streaming_talker_tokens(
"r3",
[1, 2, 3, 4, 5, 6],
[10],
@@ -58,8 +54,6 @@ def test_get_streaming_talker_tokens_new_prompt_len_snapshot_truncates(
)
assert inc_p == [1, 2, 3, 4]
assert inc_o == [10]
assert merged == [1, 2, 3, 4, 10]
assert thinker_in == [1, 2, 3, 4]
def test_get_streaming_talker_tokens_clear_state(_streaming_context: SimpleNamespace) -> None:
@@ -82,6 +76,72 @@ def test_get_streaming_codec_delta_len_increments_and_finishes(_streaming_contex
assert state.talker2code2wav_last_seq_len == 0
def test_streaming_input_prefill_chunk_is_cached() -> None:
transfer_manager = SimpleNamespace(_pending_streaming_prefills={})
request = SimpleNamespace(
external_req_id="rt-1",
output_token_ids=[100],
all_token_ids=[151644, 872, 100],
prompt_token_ids=[151644, 872],
resumable=True,
additional_information=None,
)
thinker_emb = torch.ones(2, 3)
thinker_hid = torch.full((2, 3), 2.0)
payload = q3._construct_thinker2talker_streaming_input_async_chunk(
False,
request,
thinker_emb,
thinker_hid,
transfer_manager,
)
assert payload is None
cached = transfer_manager._pending_streaming_prefills["rt-1"]
assert torch.equal(cached["embed"]["prefill"], thinker_emb)
assert torch.equal(cached["hidden_states"]["output"], thinker_hid)
assert cached["ids"]["all"] == request.all_token_ids
assert cached["ids"]["prompt"] == request.prompt_token_ids
def test_streaming_input_prefill_flushes_with_next_decode_chunk() -> None:
transfer_manager = SimpleNamespace(
_pending_streaming_prefills={
"rt-2": {
"embed": {"prefill": torch.ones(2, 3)},
"hidden_states": {"output": torch.full((2, 3), 2.0)},
"ids": {"all": [151644, 872, 100], "prompt": [151644, 872]},
}
}
)
request = SimpleNamespace(
external_req_id="rt-2",
output_token_ids=[101],
all_token_ids=[151644, 872, 100, 101],
prompt_token_ids=[151644, 872],
resumable=True,
additional_information=None,
)
thinker_emb = torch.full((1, 3), 3.0)
thinker_hid = torch.full((1, 3), 4.0)
payload = q3._construct_thinker2talker_streaming_input_async_chunk(
False,
request,
thinker_emb,
thinker_hid,
transfer_manager,
)
assert payload is not None
assert payload.embed.prefill.shape == (3, 3)
assert payload.hidden_states.output.shape == (3, 3)
assert payload.ids.all == [151644, 872, 100]
assert payload.ids.prompt == [151644, 872]
assert "rt-2" not in transfer_manager._pending_streaming_prefills
def test_talker2code2wav_full_payload_filters_by_output_token_ids() -> None:
request = SimpleNamespace(
request_id="codec",