[Feat]audio streaming input for async chunk (#3614)
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
This commit is contained in:
1 parent
6f4bd3e45c
commit
5dfdf584dd
20 files changed
+536
-184
No files matched your search
@@ -0,0 +1,92 @@
|
||||
"""Unit tests for Omni AR streaming-session async placeholder handling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
# Imports must run in this order: vllm_omni applies patches to vllm.v1.request before
|
||||
# Request / StreamingUpdate are bound in this module. Ruff isort would reorder them.
|
||||
# isort: off
|
||||
import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_scheduler(*, stage_id: int = 0) -> OmniARScheduler:
|
||||
sched = OmniARScheduler.__new__(OmniARScheduler)
|
||||
sched._new_prompt_len_snapshot = {}
|
||||
sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=stage_id))
|
||||
sched.num_waiting_for_streaming_input = 0
|
||||
sched.log_stats = False
|
||||
sched.chunk_transfer_adapter = None
|
||||
return sched
|
||||
|
||||
|
||||
def _make_request() -> Request:
|
||||
return Request(
|
||||
request_id="req-ar-streaming-test",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=SamplingParams(max_tokens=8),
|
||||
pooling_params=None,
|
||||
arrival_time=100.0,
|
||||
block_hasher=None,
|
||||
)
|
||||
|
||||
|
||||
def _make_update(prompt_token_ids: list[int] | None = None) -> StreamingUpdate:
|
||||
return StreamingUpdate(
|
||||
mm_features=None,
|
||||
prompt_token_ids=[10, 20] if prompt_token_ids is None else prompt_token_ids,
|
||||
max_tokens=32,
|
||||
arrival_time=200.0,
|
||||
sampling_params=SamplingParams(max_tokens=16),
|
||||
)
|
||||
|
||||
|
||||
def test_stage0_streaming_update_discards_outstanding_async_placeholder_token() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids([7, 8, 9])
|
||||
session.num_computed_tokens = 6
|
||||
session.num_output_placeholders = 1
|
||||
session.spec_token_ids = [-1]
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.discard_latest_async_tokens is True
|
||||
assert session.num_output_placeholders == 0
|
||||
assert session.spec_token_ids == []
|
||||
# The async placeholder makes token 9 unconfirmed, so only 7 and 8 are
|
||||
# carried into the next streaming prompt before the new chunk tokens.
|
||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 10, 20]
|
||||
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 10, 20]
|
||||
assert session._output_token_ids == []
|
||||
assert session.num_prompt_tokens == 7
|
||||
assert sched._new_prompt_len_snapshot[session.request_id] == 2
|
||||
|
||||
|
||||
def test_stage0_streaming_update_keeps_all_computed_tokens_without_placeholder() -> None:
|
||||
sched = _make_scheduler(stage_id=0)
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids([7, 8, 9])
|
||||
session.num_computed_tokens = 6
|
||||
session.num_output_placeholders = 0
|
||||
|
||||
sched._update_request_as_session(session, _make_update([10, 20]))
|
||||
|
||||
assert session.discard_latest_async_tokens is False
|
||||
assert session.num_output_placeholders == 0
|
||||
assert session.prompt_token_ids == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||
assert list(session._all_token_ids) == [1, 2, 3, 7, 8, 9, 10, 20]
|
||||
assert session._output_token_ids == []
|
||||
assert session.num_prompt_tokens == 8
|
||||
assert sched._new_prompt_len_snapshot[session.request_id] == 2
|
||||
+25
-10
@@ -1,6 +1,6 @@
|
||||
"""Unit tests for OmniSchedulerMixin streaming session replacement.
|
||||
"""Unit tests for generation streaming session replacement.
|
||||
|
||||
These tests pin the behavior of `_replace_session_with_streaming_update` against
|
||||
These tests pin the behavior of `_update_request_as_session` against
|
||||
current vLLM `Request` / `StreamingUpdate` (and Omni patches). When upgrading
|
||||
vLLM, failures here should highlight incompatible changes to request state or
|
||||
update payloads early.
|
||||
@@ -19,19 +19,34 @@ import vllm_omni # noqa: F401 - import for side effects (patch vLLM)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreEventType
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm_omni.core.sched.omni_scheduler_mixin import OmniSchedulerMixin
|
||||
from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
|
||||
|
||||
# isort: on
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _SchedulerStub(OmniSchedulerMixin):
|
||||
"""Minimal scheduler surface required by OmniSchedulerMixin."""
|
||||
class _SkippedWaitingStub:
|
||||
def __contains__(self, request: Request) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _ChunkTransferAdapterStub:
|
||||
def __init__(self) -> None:
|
||||
self.segment_finished_requests: set[str] = set()
|
||||
|
||||
|
||||
class _SchedulerStub(OmniGenerationScheduler):
|
||||
"""Minimal scheduler surface required by OmniGenerationScheduler."""
|
||||
|
||||
def __init__(self, *, log_stats: bool = False) -> None:
|
||||
self.num_waiting_for_streaming_input = 0
|
||||
self.log_stats = log_stats
|
||||
self.chunk_transfer_adapter = _ChunkTransferAdapterStub()
|
||||
self.skipped_waiting = _SkippedWaitingStub()
|
||||
|
||||
def _enqueue_waiting_request(self, session: Request) -> None:
|
||||
raise AssertionError("unexpected enqueue for skipped_waiting miss")
|
||||
|
||||
|
||||
def _make_request(**kwargs) -> Request:
|
||||
@@ -71,7 +86,7 @@ class TestReplaceSessionWithStreamingUpdate:
|
||||
|
||||
update = _make_update(prompt_token_ids=[40, 41, 42])
|
||||
sched.num_waiting_for_streaming_input = 3
|
||||
sched._replace_session_with_streaming_update(session, update)
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session._output_token_ids == []
|
||||
assert list(session._all_token_ids) == [40, 41, 42]
|
||||
@@ -88,7 +103,7 @@ class TestReplaceSessionWithStreamingUpdate:
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.RUNNING
|
||||
update = _make_update(prompt_token_ids=None)
|
||||
sched._replace_session_with_streaming_update(session, update)
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == ()
|
||||
assert list(session._all_token_ids) == []
|
||||
@@ -108,7 +123,7 @@ class TestReplaceSessionWithStreamingUpdate:
|
||||
pytest.skip("StreamingUpdate has no additional_information (Omni patch inactive?)")
|
||||
update = replace(base, additional_information=None)
|
||||
|
||||
sched._replace_session_with_streaming_update(session, update)
|
||||
sched._update_request_as_session(session, update)
|
||||
assert session.additional_information is None
|
||||
|
||||
def test_does_not_decrement_waiting_when_not_streaming_status(self) -> None:
|
||||
@@ -116,14 +131,14 @@ class TestReplaceSessionWithStreamingUpdate:
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.RUNNING
|
||||
sched.num_waiting_for_streaming_input = 5
|
||||
sched._replace_session_with_streaming_update(session, _make_update())
|
||||
sched._update_request_as_session(session, _make_update())
|
||||
assert sched.num_waiting_for_streaming_input == 5
|
||||
|
||||
def test_records_queued_event_when_log_stats_enabled(self) -> None:
|
||||
sched = _SchedulerStub(log_stats=True)
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
sched._replace_session_with_streaming_update(session, _make_update())
|
||||
sched._update_request_as_session(session, _make_update())
|
||||
|
||||
assert session.events
|
||||
assert session.events[-1].type == EngineCoreEventType.QUEUED
|
||||
@@ -230,9 +230,11 @@ def test_send_single_request_struct_without_meta_does_not_crash(build_adapter, m
|
||||
cleanup_calls = []
|
||||
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: cleanup_calls.append((a, kw)))
|
||||
|
||||
adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": False})
|
||||
adapter._send_single_request(
|
||||
{"pooling_output": None, "request": request, "is_finished": False, "is_segment_finished": False}
|
||||
)
|
||||
|
||||
assert cleanup_calls == [] # no terminal cleanup; meta.finished is unobservable
|
||||
assert cleanup_calls == [] # no terminal cleanup; meta.finished is false
|
||||
|
||||
|
||||
def test_send_single_request_empty_struct_goes_on_wire(build_adapter, monkeypatch):
|
||||
@@ -248,12 +250,50 @@ def test_send_single_request_empty_struct_goes_on_wire(build_adapter, monkeypatc
|
||||
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct()
|
||||
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None)
|
||||
|
||||
adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": False})
|
||||
adapter._send_single_request(
|
||||
{"pooling_output": None, "request": request, "is_finished": False, "is_segment_finished": False}
|
||||
)
|
||||
|
||||
assert connector.put.called
|
||||
sent_payload = connector.put.call_args.kwargs["data"]
|
||||
assert isinstance(sent_payload, OmniPayloadStruct)
|
||||
assert sent_payload.meta is None # confirms it's the empty struct on the wire
|
||||
assert sent_payload.meta.finished.item() is False
|
||||
assert sent_payload.meta.is_segment_finished.item() is False
|
||||
|
||||
|
||||
def test_send_single_request_struct_preserves_segment_finished(build_adapter, monkeypatch):
|
||||
adapter, connector = build_adapter(stage_id=1)
|
||||
request = _req("req-segment", RequestStatus.WAITING, external_req_id="ext-segment")
|
||||
|
||||
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct()
|
||||
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None)
|
||||
|
||||
adapter._send_single_request(
|
||||
{"pooling_output": None, "request": request, "is_finished": False, "is_segment_finished": True}
|
||||
)
|
||||
|
||||
sent_payload = connector.put.call_args.kwargs["data"]
|
||||
assert sent_payload.meta.finished.item() is False
|
||||
assert sent_payload.meta.is_segment_finished.item() is True
|
||||
|
||||
|
||||
def test_save_async_skips_stale_resumable_chunk_until_dedup_is_reset(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1)
|
||||
request = _req("req-stream", RequestStatus.WAITING, external_req_id="ext-stream")
|
||||
request.resumable = True
|
||||
request.num_computed_tokens = 0
|
||||
adapter.requests_num_chunks_sent["ext-stream"] = 111
|
||||
|
||||
adapter.save_async(pooling_output=None, request=request, is_segment_finished=False)
|
||||
|
||||
assert len(adapter._pending_save_reqs) == 0
|
||||
assert adapter.requests_num_chunks_sent["ext-stream"] == 111
|
||||
|
||||
adapter.requests_num_chunks_sent.pop("ext-stream")
|
||||
adapter.save_async(pooling_output=None, request=request, is_segment_finished=False)
|
||||
|
||||
assert len(adapter._pending_save_reqs) == 1
|
||||
assert adapter.requests_num_chunks_sent["ext-stream"] == 0
|
||||
|
||||
|
||||
def test_send_single_request_cleans_up_after_finished_payload(build_adapter, monkeypatch):
|
||||
@@ -266,7 +306,9 @@ def test_send_single_request_cleans_up_after_finished_payload(build_adapter, mon
|
||||
cleanup_calls = []
|
||||
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: cleanup_calls.append((a, kw)))
|
||||
|
||||
adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": True})
|
||||
adapter._send_single_request(
|
||||
{"pooling_output": None, "request": request, "is_finished": True, "is_segment_finished": True}
|
||||
)
|
||||
|
||||
assert len(cleanup_calls) == 1
|
||||
args, _ = cleanup_calls[0]
|
||||
|
||||
@@ -964,6 +964,7 @@ async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> Non
|
||||
|
||||
pool = RecordingPool()
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.async_chunk = False
|
||||
orchestrator.request_states = {
|
||||
"req-stream": OrchestratorRequestState(
|
||||
request_id="req-stream",
|
||||
|
||||
@@ -44,7 +44,6 @@ realtime_server_params = [
|
||||
model=MODEL,
|
||||
stage_config_path=default_stage_config,
|
||||
use_stage_cli=True,
|
||||
server_args=["--no-async-chunk"],
|
||||
),
|
||||
id="default",
|
||||
),
|
||||
|
||||
@@ -20,7 +20,7 @@ def _streaming_context() -> SimpleNamespace:
|
||||
|
||||
|
||||
def test_get_streaming_talker_tokens_first_segment(_streaming_context: SimpleNamespace) -> None:
|
||||
inc_p, inc_o, merged, thinker_in = q3._get_streaming_talker_tokens(
|
||||
inc_p, inc_o = q3._get_streaming_talker_tokens(
|
||||
"r1",
|
||||
[1, 2],
|
||||
[10, 11],
|
||||
@@ -28,13 +28,11 @@ def test_get_streaming_talker_tokens_first_segment(_streaming_context: SimpleNam
|
||||
)
|
||||
assert inc_p == [1, 2]
|
||||
assert inc_o == [10, 11]
|
||||
assert merged == [1, 2, 10, 11]
|
||||
assert thinker_in == [1, 2]
|
||||
|
||||
|
||||
def test_get_streaming_talker_tokens_second_segment_accumulates(_streaming_context: SimpleNamespace) -> None:
|
||||
q3._get_streaming_talker_tokens("r2", [1, 2], [10, 11], streaming_context=_streaming_context)
|
||||
inc_p, inc_o, merged, thinker_in = q3._get_streaming_talker_tokens(
|
||||
inc_p, inc_o = q3._get_streaming_talker_tokens(
|
||||
"r2",
|
||||
[1, 2, 3, 4],
|
||||
[10, 11, 12, 13],
|
||||
@@ -42,14 +40,12 @@ def test_get_streaming_talker_tokens_second_segment_accumulates(_streaming_conte
|
||||
)
|
||||
assert inc_p == [3, 4]
|
||||
assert inc_o == [12, 13]
|
||||
assert merged == [1, 2, 10, 3, 4, 12, 13]
|
||||
assert thinker_in == [1, 2, 10, 3, 4]
|
||||
|
||||
|
||||
def test_get_streaming_talker_tokens_new_prompt_len_snapshot_truncates(
|
||||
_streaming_context: SimpleNamespace,
|
||||
) -> None:
|
||||
inc_p, inc_o, merged, thinker_in = q3._get_streaming_talker_tokens(
|
||||
inc_p, inc_o = q3._get_streaming_talker_tokens(
|
||||
"r3",
|
||||
[1, 2, 3, 4, 5, 6],
|
||||
[10],
|
||||
@@ -58,8 +54,6 @@ def test_get_streaming_talker_tokens_new_prompt_len_snapshot_truncates(
|
||||
)
|
||||
assert inc_p == [1, 2, 3, 4]
|
||||
assert inc_o == [10]
|
||||
assert merged == [1, 2, 3, 4, 10]
|
||||
assert thinker_in == [1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_get_streaming_talker_tokens_clear_state(_streaming_context: SimpleNamespace) -> None:
|
||||
@@ -82,6 +76,72 @@ def test_get_streaming_codec_delta_len_increments_and_finishes(_streaming_contex
|
||||
assert state.talker2code2wav_last_seq_len == 0
|
||||
|
||||
|
||||
def test_streaming_input_prefill_chunk_is_cached() -> None:
|
||||
transfer_manager = SimpleNamespace(_pending_streaming_prefills={})
|
||||
request = SimpleNamespace(
|
||||
external_req_id="rt-1",
|
||||
output_token_ids=[100],
|
||||
all_token_ids=[151644, 872, 100],
|
||||
prompt_token_ids=[151644, 872],
|
||||
resumable=True,
|
||||
additional_information=None,
|
||||
)
|
||||
thinker_emb = torch.ones(2, 3)
|
||||
thinker_hid = torch.full((2, 3), 2.0)
|
||||
|
||||
payload = q3._construct_thinker2talker_streaming_input_async_chunk(
|
||||
False,
|
||||
request,
|
||||
thinker_emb,
|
||||
thinker_hid,
|
||||
transfer_manager,
|
||||
)
|
||||
|
||||
assert payload is None
|
||||
cached = transfer_manager._pending_streaming_prefills["rt-1"]
|
||||
assert torch.equal(cached["embed"]["prefill"], thinker_emb)
|
||||
assert torch.equal(cached["hidden_states"]["output"], thinker_hid)
|
||||
assert cached["ids"]["all"] == request.all_token_ids
|
||||
assert cached["ids"]["prompt"] == request.prompt_token_ids
|
||||
|
||||
|
||||
def test_streaming_input_prefill_flushes_with_next_decode_chunk() -> None:
|
||||
transfer_manager = SimpleNamespace(
|
||||
_pending_streaming_prefills={
|
||||
"rt-2": {
|
||||
"embed": {"prefill": torch.ones(2, 3)},
|
||||
"hidden_states": {"output": torch.full((2, 3), 2.0)},
|
||||
"ids": {"all": [151644, 872, 100], "prompt": [151644, 872]},
|
||||
}
|
||||
}
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
external_req_id="rt-2",
|
||||
output_token_ids=[101],
|
||||
all_token_ids=[151644, 872, 100, 101],
|
||||
prompt_token_ids=[151644, 872],
|
||||
resumable=True,
|
||||
additional_information=None,
|
||||
)
|
||||
thinker_emb = torch.full((1, 3), 3.0)
|
||||
thinker_hid = torch.full((1, 3), 4.0)
|
||||
|
||||
payload = q3._construct_thinker2talker_streaming_input_async_chunk(
|
||||
False,
|
||||
request,
|
||||
thinker_emb,
|
||||
thinker_hid,
|
||||
transfer_manager,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload.embed.prefill.shape == (3, 3)
|
||||
assert payload.hidden_states.output.shape == (3, 3)
|
||||
assert payload.ids.all == [151644, 872, 100]
|
||||
assert payload.ids.prompt == [151644, 872]
|
||||
assert "rt-2" not in transfer_manager._pending_streaming_prefills
|
||||
|
||||
|
||||
def test_talker2code2wav_full_payload_filters_by_output_token_ids() -> None:
|
||||
request = SimpleNamespace(
|
||||
request_id="codec",
|
||||
|
||||
Reference in new issue
Block a user