Merge branch 'main' into fix/cfg-companion-output-race

This commit is contained in:
Zhou Taichang authored and GitHub committed 2026-07-30 21:56:54 +08:00
commit d9ab4aa1c0
372 files changed
+20648 -9658

No files matched your search

@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for vllm_omni.engine.arg_utils — invariants that must
hold for the orchestrator/engine/server CLI flag partition."""
from __future__ import annotations
from dataclasses import fields
import pytest
from vllm_omni.engine.arg_utils import (
SHARED_FIELDS,
OmniEngineArgs,
internal_blacklist_keys,
orchestrator_field_names,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_no_ambiguous_overlap_with_real_engine():
"""OrchestratorArgs ∩ OmniEngineArgs must be ⊆ SHARED_FIELDS ∪ orchestrator-captured.
Fields that the orchestrator intentionally captures to prevent uniform
propagation (async_chunk, tokenizer) are excluded — they exist in both
classes by design but flow through DeployConfig, not kwargs forwarding.
"""
# Fields on both classes by design: orchestrator captures them to prevent
# uniform per-stage propagation; redistributed via DeployConfig.
orchestrator_captured = {"async_chunk", "tokenizer"}
orch = orchestrator_field_names()
engine = {f.name for f in fields(OmniEngineArgs)}
overlap = orch & engine
unexpected = overlap - SHARED_FIELDS - orchestrator_captured
assert not unexpected, (
f"OmniEngineArgs has ambiguous overlap with OrchestratorArgs: "
f"{sorted(unexpected)}. Update SHARED_FIELDS or remove duplication."
)
def test_internal_blacklist_keys_derived_from_orchestrator():
"""Blacklist is exactly OrchestratorArgs fields minus SHARED_FIELDS.
This function replaces the old hardcoded INTERNAL_STAGE_OVERRIDE_KEYS
frozenset. Asserts the contract so future changes to OrchestratorArgs
automatically propagate to the blacklist.
"""
blacklist = internal_blacklist_keys()
assert blacklist == orchestrator_field_names() - SHARED_FIELDS
+396
View File
@@ -0,0 +1,396 @@
"""Tests for data_entry_keys."""
import msgspec
import pytest
import torch
from vllm_omni.data_entry_keys import (
CodesStruct,
EmbeddingsStruct,
HiddenStatesStruct,
IdsStruct,
MetaStruct,
OmniPayload,
OmniPayloadStruct,
deserialize_payload,
flatten_payload,
serialize_payload,
to_dict,
to_struct,
unflatten_payload,
)
from vllm_omni.engine import AdditionalInformationPayload
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestOmniPayloadStruct:
"""Runtime-validated mirror of OmniPayload (msgspec.Struct)."""
def test_to_struct_validates_dict(self):
d = {"meta": {"left_context_size": 25, "finished": torch.tensor(False)}}
s = to_struct(d)
assert s.meta.left_context_size == 25
def test_to_struct_rejects_legacy_flat_top_level(self):
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"code_predictor_codes": torch.zeros(3, 8)})
def test_to_struct_rejects_legacy_flat_meta_field(self):
# `left_context_size` at top level (legacy) instead of under `meta`
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"left_context_size": 25})
def test_to_struct_rejects_typo_in_subkey(self):
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"meta": {"finisheed": True}})
def test_to_struct_rejects_minicpmo_duplex_runtime_handoff_fields(self):
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"hidden_states": {"tts": torch.zeros(1)}})
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"ids": {"tts": [1]}})
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"meta": {"ref_audio_sr": 16000}})
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({"meta": {"listen_token_id": 151685}})
def test_to_struct_rejects_wrong_type(self):
with pytest.raises(msgspec.ValidationError, match="Expected"):
to_struct({"meta": {"left_context_size": "not_an_int"}})
def test_round_trip_dict_struct_dict(self):
original = {
"meta": {"left_context_size": 7, "finished": torch.tensor(True)},
"codes": {"audio": torch.zeros(2, 8)},
"hidden_states": {"output": torch.zeros(4, 16)},
}
s = to_struct(original)
d = to_dict(s)
assert sorted(d.keys()) == sorted(original.keys())
for top, sub in original.items():
assert sorted(d[top].keys()) == sorted(sub.keys())
def test_to_dict_drops_unset_fields(self):
s = OmniPayloadStruct(meta=MetaStruct(left_context_size=10))
d = to_dict(s)
assert d == {"meta": {"left_context_size": 10}}
def test_struct_with_all_categories(self):
d = {
"hidden_states": {"output": torch.zeros(1), "last": torch.ones(2, 4)},
"embed": {"prefill": torch.zeros(1), "tts_bos": torch.zeros(1)},
"ids": {"all": [1, 2], "prompt": [1], "speech_token": [3, 4]},
"codes": {"audio": torch.zeros(1), "ref": torch.ones(4)},
"meta": {"left_context_size": 3, "num_processed_tokens": 7},
}
s = to_struct(d)
assert isinstance(s.hidden_states, HiddenStatesStruct)
assert isinstance(s.embed, EmbeddingsStruct)
assert isinstance(s.ids, IdsStruct)
assert isinstance(s.codes, CodesStruct)
assert isinstance(s.meta, MetaStruct)
assert s.ids.all == [1, 2]
assert s.ids.speech_token == [3, 4]
assert torch.equal(s.hidden_states.last, torch.ones(2, 4))
assert s.meta.num_processed_tokens == 7
assert torch.equal(s.codes.ref, torch.ones(4))
class TestValidatePayload:
def test_raises_on_unknown_top_level(self):
from vllm_omni.data_entry_keys import validate_payload
with pytest.raises(msgspec.ValidationError, match="unknown field"):
validate_payload({"code_predictor_codes": torch.zeros(3, 8)}, context="test_boundary")
def test_raises_on_unknown_sub_key(self):
from vllm_omni.data_entry_keys import validate_payload
with pytest.raises(msgspec.ValidationError, match="unknown field"):
validate_payload({"meta": {"finisheed": True}})
def test_none_is_ok(self):
from vllm_omni.data_entry_keys import validate_payload
validate_payload(None) # should not raise
def test_valid_payload_passes(self):
from vllm_omni.data_entry_keys import validate_payload
validate_payload({"meta": {"left_context_size": 5}})
def test_context_in_error_message(self):
from vllm_omni.data_entry_keys import validate_payload
with pytest.raises(msgspec.ValidationError, match="my_call_site"):
validate_payload({"bad": 1}, context="my_call_site")
class TestWireEquivalenceStructVsDict:
"""Producer return-side migration invariant: encoding an ``OmniPayloadStruct``
via the connector serializer must decode to the same payload as encoding
the equivalent ``to_dict(struct)``.
Guards against regressions where the wire format diverges between the two
paths (e.g. msgspec adds a struct tag, or ``to_dict`` drops a non-default
sub-field that ``omit_defaults`` retains).
"""
@staticmethod
def _round_trip(obj):
from vllm_omni.distributed.omni_connectors.utils.serialization import (
OmniMsgpackDecoder,
OmniMsgpackEncoder,
)
return OmniMsgpackDecoder().decode(OmniMsgpackEncoder().encode(obj))
@staticmethod
def _assert_decoded_equal(a, b):
if isinstance(a, dict):
assert isinstance(b, dict)
assert sorted(a.keys()) == sorted(b.keys())
for k in a:
TestWireEquivalenceStructVsDict._assert_decoded_equal(a[k], b[k])
elif isinstance(a, torch.Tensor):
assert isinstance(b, torch.Tensor)
assert a.dtype == b.dtype
assert a.shape == b.shape
assert torch.equal(a, b)
elif isinstance(a, list):
assert isinstance(b, list) and len(a) == len(b)
for x, y in zip(a, b, strict=True):
TestWireEquivalenceStructVsDict._assert_decoded_equal(x, y)
else:
assert a == b
def test_basic_payload(self):
struct = OmniPayloadStruct(
codes=CodesStruct(audio=torch.tensor([1, 2, 3], dtype=torch.long)),
meta=MetaStruct(left_context_size=10, finished=torch.tensor(True, dtype=torch.bool)),
)
self._assert_decoded_equal(self._round_trip(struct), self._round_trip(to_dict(struct)))
def test_nested_sub_structs(self):
# Exercises depth-2 sub-struct encoding (embed.*) which was the case that
# exposed schema drift in the #1829 migration.
struct = OmniPayloadStruct(
codes=CodesStruct(audio=torch.tensor([5, 6], dtype=torch.long)),
meta=MetaStruct(finished=torch.tensor(False, dtype=torch.bool)),
embed=EmbeddingsStruct(
speech_token=torch.tensor([[1, 2, 3]]),
speech_feat=torch.tensor([[[0.1, 0.2], [0.3, 0.4]]]),
embedding=torch.tensor([[0.5, 0.6]]),
),
)
self._assert_decoded_equal(self._round_trip(struct), self._round_trip(to_dict(struct)))
class TestFlattenPayload:
def test_basic_nested_to_dotted(self):
nested = {
"codes": {"audio": torch.tensor([1.0])},
"meta": {"finished": torch.tensor(True, dtype=torch.bool), "left_context_size": 5},
}
flat = flatten_payload(nested)
assert torch.equal(flat["codes.audio"], torch.tensor([1.0]))
assert flat["meta.finished"].item() is True
assert flat["meta.left_context_size"] == 5
assert "codes" not in flat
assert "meta" not in flat
def test_top_level_keys_preserved(self):
nested = {
"latent": torch.tensor([9.0]),
"generated_len": 42,
}
flat = flatten_payload(nested)
assert torch.equal(flat["latent"], torch.tensor([9.0]))
assert flat["generated_len"] == 42
def test_hidden_states_layers_expanded(self):
nested = {
"hidden_states": {
"output": torch.tensor([1.0]),
"layers": {
0: torch.tensor([2.0]),
24: torch.tensor([3.0]),
},
},
}
flat = flatten_payload(nested)
assert torch.equal(flat["hidden_states.output"], torch.tensor([1.0]))
assert torch.equal(flat["hidden_states.layer_0"], torch.tensor([2.0]))
assert torch.equal(flat["hidden_states.layer_24"], torch.tensor([3.0]))
assert "hidden_states.layers" not in flat
def test_mixed_nested_and_top_level(self):
nested: OmniPayload = {
"codes": {"audio": torch.tensor([1.0])},
"latent": torch.tensor([2.0]),
"meta": {"finished": torch.tensor(False, dtype=torch.bool)},
}
flat = flatten_payload(nested)
assert set(flat.keys()) == {"codes.audio", "latent", "meta.finished"}
class TestUnflattenPayload:
def test_basic_dotted_to_nested(self):
flat = {
"codes.audio": torch.tensor([1.0]),
"meta.finished": torch.tensor(True, dtype=torch.bool),
"meta.left_context_size": 5,
}
nested = unflatten_payload(flat)
assert torch.equal(nested["codes"]["audio"], torch.tensor([1.0]))
assert nested["meta"]["finished"].item() is True
assert nested["meta"]["left_context_size"] == 5
def test_top_level_keys_preserved(self):
flat = {"latent": torch.tensor([9.0]), "generated_len": 42}
nested = unflatten_payload(flat)
assert torch.equal(nested["latent"], torch.tensor([9.0]))
assert nested["generated_len"] == 42
def test_hidden_states_layers_collected(self):
flat = {
"hidden_states.output": torch.tensor([1.0]),
"hidden_states.layer_0": torch.tensor([2.0]),
"hidden_states.layer_24": torch.tensor([3.0]),
}
nested = unflatten_payload(flat)
assert torch.equal(nested["hidden_states"]["output"], torch.tensor([1.0]))
assert torch.equal(nested["hidden_states"]["layers"][0], torch.tensor([2.0]))
assert torch.equal(nested["hidden_states"]["layers"][24], torch.tensor([3.0]))
class TestFlattenUnflattenRoundTrip:
def test_round_trip_simple(self):
original: OmniPayload = {
"codes": {"audio": torch.tensor([1.0, 2.0])},
"meta": {"finished": torch.tensor(True, dtype=torch.bool), "left_context_size": 10},
"ids": {"prompt": [1, 2, 3]},
"latent": torch.tensor([5.0]),
}
restored = unflatten_payload(flatten_payload(original))
assert torch.equal(restored["codes"]["audio"], original["codes"]["audio"])
assert restored["meta"]["finished"].item() is True
assert restored["meta"]["left_context_size"] == 10
assert restored["ids"]["prompt"] == [1, 2, 3]
assert torch.equal(restored["latent"], original["latent"])
def test_round_trip_with_layers(self):
original = {
"hidden_states": {
"output": torch.tensor([1.0]),
"layers": {0: torch.tensor([2.0]), 24: torch.tensor([3.0])},
},
}
restored = unflatten_payload(flatten_payload(original))
assert torch.equal(restored["hidden_states"]["output"], torch.tensor([1.0]))
assert torch.equal(restored["hidden_states"]["layers"][0], torch.tensor([2.0]))
assert torch.equal(restored["hidden_states"]["layers"][24], torch.tensor([3.0]))
def test_round_trip_all_categories(self):
original: OmniPayload = {
"hidden_states": {"output": torch.tensor([1.0]), "last": torch.tensor([2.0])},
"embed": {"prefill": torch.tensor([3.0]), "tts_bos": torch.tensor([4.0])},
"codes": {"audio": torch.tensor([5.0]), "ref": torch.tensor([6.0])},
"ids": {"all": [1, 2], "prompt": [3, 4]},
"meta": {"finished": torch.tensor(False, dtype=torch.bool), "ar_width": 8},
}
restored = unflatten_payload(flatten_payload(original))
assert torch.equal(restored["hidden_states"]["output"], torch.tensor([1.0]))
assert torch.equal(restored["hidden_states"]["last"], torch.tensor([2.0]))
assert torch.equal(restored["embed"]["prefill"], torch.tensor([3.0]))
assert torch.equal(restored["embed"]["tts_bos"], torch.tensor([4.0]))
assert torch.equal(restored["codes"]["audio"], torch.tensor([5.0]))
assert torch.equal(restored["codes"]["ref"], torch.tensor([6.0]))
assert restored["ids"]["all"] == [1, 2]
assert restored["ids"]["prompt"] == [3, 4]
assert restored["meta"]["finished"].item() is False
assert restored["meta"]["ar_width"] == 8
class TestSerializeDeserializePayload:
def test_tensor_round_trip(self):
original: OmniPayload = {
"hidden_states": {"output": torch.tensor([[1.0, 2.0], [3.0, 4.0]])},
}
wire = serialize_payload(original)
assert isinstance(wire, AdditionalInformationPayload)
restored = deserialize_payload(wire)
assert torch.equal(restored["hidden_states"]["output"], original["hidden_states"]["output"])
def test_list_round_trip(self):
original: OmniPayload = {
"ids": {"prompt": [10, 20, 30]},
}
wire = serialize_payload(original)
restored = deserialize_payload(wire)
assert restored["ids"]["prompt"] == [10, 20, 30]
def test_finished_tensor_round_trip(self):
original: OmniPayload = {
"meta": {"finished": torch.tensor(True, dtype=torch.bool), "left_context_size": 5},
}
wire = serialize_payload(original)
restored = deserialize_payload(wire)
assert isinstance(restored["meta"]["finished"], torch.Tensor)
assert restored["meta"]["finished"].dtype == torch.bool
assert restored["meta"]["finished"].item() is True
assert restored["meta"]["left_context_size"] == 5
def test_mixed_types_round_trip(self):
original: OmniPayload = {
"hidden_states": {"output": torch.tensor([1.0, 2.0])},
"ids": {"all": [1, 2, 3]},
"meta": {"finished": torch.tensor(False, dtype=torch.bool), "ar_width": 4},
"codes": {"audio": torch.tensor([3.0]), "ref": torch.tensor([0.1, 0.2])},
}
wire = serialize_payload(original)
restored = deserialize_payload(wire)
assert torch.equal(restored["hidden_states"]["output"], original["hidden_states"]["output"])
assert restored["ids"]["all"] == [1, 2, 3]
assert restored["meta"]["finished"].item() is False
assert restored["meta"]["ar_width"] == 4
assert torch.equal(restored["codes"]["audio"], original["codes"]["audio"])
assert torch.equal(restored["codes"]["ref"], original["codes"]["ref"])
def test_hidden_states_layers_round_trip(self):
original = {
"hidden_states": {
"output": torch.tensor([1.0]),
"layers": {0: torch.tensor([2.0]), 24: torch.tensor([3.0])},
},
}
wire = serialize_payload(original)
restored = deserialize_payload(wire)
assert torch.equal(restored["hidden_states"]["output"], torch.tensor([1.0]))
assert torch.equal(restored["hidden_states"]["layers"][0], torch.tensor([2.0]))
assert torch.equal(restored["hidden_states"]["layers"][24], torch.tensor([3.0]))
def test_tensor_dtype_preserved(self):
# bfloat16 excluded: numpy() doesn't support it; callers must cast before serializing.
for dtype in [torch.float16, torch.float32, torch.int64, torch.int32, torch.bool]:
original: OmniPayload = {"codes": {"audio": torch.tensor([1], dtype=dtype)}}
wire = serialize_payload(original)
restored = deserialize_payload(wire)
assert restored["codes"]["audio"].dtype == dtype, f"dtype mismatch for {dtype}"
def test_tensor_shape_preserved(self):
t = torch.randn(3, 4, 5)
original: OmniPayload = {"hidden_states": {"output": t}}
wire = serialize_payload(original)
restored = deserialize_payload(wire)
assert restored["hidden_states"]["output"].shape == (3, 4, 5)
assert torch.allclose(restored["hidden_states"]["output"], t)
def test_empty_payload_returns_none(self):
assert serialize_payload({}) is None
def test_none_values_skipped(self):
original: OmniPayload = {"meta": {"finished": None}}
wire = serialize_payload(original)
assert wire is None
@@ -18,6 +18,7 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state():
accumulated = MultimodalPayload.from_dict(
{
"audio": torch.tensor([1.0]),
"meta.segment_end": torch.tensor([0]),
"meta.tts_is_last_chunk": torch.tensor([0]),
"meta.turn_end": torch.tensor([0]),
"meta.stable_request_value": "keep",
@@ -26,6 +27,7 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state():
incoming = MultimodalPayload.from_dict(
{
"audio": torch.tensor([2.0]),
"meta.segment_end": torch.tensor([1]),
"meta.tts_is_last_chunk": torch.tensor([1]),
"meta.turn_end": torch.tensor([1]),
}
@@ -41,6 +43,7 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state():
drain_delta_payload(merged)
assert "audio" not in merged
assert "meta.segment_end" not in merged
assert "meta.tts_is_last_chunk" not in merged
assert "meta.turn_end" not in merged
assert merged.metadata["meta.stable_request_value"] == "keep"
+112
View File
@@ -0,0 +1,112 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for OmniRequest substitutability with the base Request.
vllm-omni rebinds ``vllm.v1.request.Request`` to ``OmniRequest`` at import
time (see ``vllm_omni/patch.py``). vLLM core constructs ``Request`` positionally
in some paths (notably ``vllm/v1/worker/gpu/warmup.py::warmup_kernels`` for V2
model-runner architectures like Qwen3ForCausalLM). The omni-specific params
must therefore be keyword-only so positional construction stays Liskov-
substitutable with the base class — including base-style calls that pass
``prompt_embeds`` (itself a positional-capable base param) positionally.
"""
import inspect
import numpy as np
import pytest
import torch
from vllm.sampling_params import SamplingParams
from vllm.v1.request import Request
from vllm_omni.engine import PromptEmbedsPayload
from vllm_omni.request import OmniRequest, OmniStreamingUpdate
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_omni_params_are_keyword_only():
"""The three omni params must be keyword-only after ``*args``.
Guards against re-introducing the import-time rebind bug where omni params
came first positionally and broke positional ``Request(...)`` construction.
"""
params = inspect.signature(OmniRequest.__init__).parameters
for name in (
"prompt_embeds",
"external_req_id",
"additional_information",
"model_intermediate_buffer",
):
assert params[name].kind is inspect.Parameter.KEYWORD_ONLY, name
def test_positional_construction_matches_base_request():
"""Reproduces the warmup_kernels call: Request(id, tokens, sp, pp)."""
req = OmniRequest("req-0", [1, 2, 3], SamplingParams(), None)
assert isinstance(req, Request)
assert req.request_id == "req-0"
assert req.prompt_token_ids == [1, 2, 3]
# Omni params default cleanly when constructed positionally.
assert req.external_req_id is None
assert req.additional_information is None
assert req.prompt_embeds_payload is None
def test_positional_prompt_embeds_does_not_collide():
"""Base-style positional call that includes ``prompt_embeds``.
``prompt_embeds`` is a positional-capable base param (after
``arrival_time``). A base-style call must not trip
``got multiple values for argument 'prompt_embeds'`` — the subclass must
not also inject its keyword override when the value arrived positionally.
Positional order: request_id, prompt_token_ids, sampling_params,
pooling_params, client_index, arrival_time, prompt_embeds.
"""
embeds = torch.arange(6, dtype=torch.float32).reshape(2, 3)
req = OmniRequest("req-pe", [1, 2], SamplingParams(), None, 0, None, embeds)
assert isinstance(req, Request)
assert torch.equal(req.prompt_embeds, embeds)
# Positional tensor is not a serialized payload.
assert req.prompt_embeds_payload is None
assert req.external_req_id is None
def test_keyword_omni_params_round_trip():
"""Keyword omni params are preserved; serialized embeds are decoded."""
arr = np.arange(6, dtype=np.float32).reshape(2, 3)
payload = PromptEmbedsPayload(data=arr.tobytes(), shape=[2, 3], dtype="float32")
req = OmniRequest(
request_id="req-1",
prompt_token_ids=[7, 8],
sampling_params=SamplingParams(),
pooling_params=None,
prompt_embeds=payload,
external_req_id="ext-1",
)
assert req.external_req_id == "ext-1"
# The serialized payload is retained and decoded into a tensor on the base.
assert req.prompt_embeds_payload is payload
assert torch.equal(req.prompt_embeds, torch.from_numpy(arr))
def test_model_intermediate_buffer_round_trips_to_streaming_update():
buffer = {"duplex": {"turn_id": 3}}
request = OmniRequest(
request_id="req-model-buffer",
prompt_token_ids=[1, 2],
sampling_params=SamplingParams(max_tokens=16),
pooling_params=None,
resumable=True,
model_intermediate_buffer=buffer,
)
update = OmniStreamingUpdate.from_request(request)
assert update is not None
assert request.model_intermediate_buffer is buffer
assert update.model_intermediate_buffer is buffer
+174
View File
@@ -0,0 +1,174 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for OmniRequestOutput class."""
import pytest
from PIL import Image
from pytest_mock import MockerFixture
from vllm_omni.outputs import OmniRequestOutput
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class TestOmniRequestOutput:
"""Tests for OmniRequestOutput class."""
def test_from_diffusion(self):
"""Test creating output from diffusion model."""
images = [Image.new("RGB", (64, 64), color="red")]
output = OmniRequestOutput.from_diffusion(
request_id="test-123",
images=images,
prompt="a cat",
metrics={"steps": 50},
)
assert output.request_id == "test-123"
assert output.images == images
assert output.prompt == "a cat"
assert output.metrics == {"steps": 50}
assert output.is_diffusion_output
assert output.num_images == 1
def test_from_pipeline(self, mocker: MockerFixture):
"""Test creating output from pipeline stage."""
mock_request_output = mocker.Mock()
mock_request_output.request_id = "pipeline-123"
mock_request_output.prompt_token_ids = [1, 2, 3]
mock_request_output.outputs = [mocker.Mock()]
mock_request_output.encoder_prompt_token_ids = None
mock_request_output.prompt_logprobs = None
mock_request_output.num_cached_tokens = 10
mock_request_output.kv_transfer_params = None
mock_request_output.multimodal_output = {"image": mocker.Mock()}
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="text",
request_output=mock_request_output,
)
assert output.request_id == "pipeline-123"
assert output.stage_id == 0
assert output.final_output_type == "text"
assert output.is_pipeline_output
def test_prompt_token_ids_property(self, mocker: MockerFixture):
"""Test prompt_token_ids property for streaming compatibility."""
mock_request_output = mocker.Mock()
mock_request_output.prompt_token_ids = [1, 2, 3, 4, 5]
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="text",
request_output=mock_request_output,
)
assert output.prompt_token_ids == [1, 2, 3, 4, 5]
def test_prompt_token_ids_none_when_no_request_output(self):
"""Test prompt_token_ids returns None when no request_output."""
output = OmniRequestOutput.from_diffusion(
request_id="test-123",
images=[],
prompt="a cat",
)
assert output.prompt_token_ids is None
def test_outputs_property(self, mocker: MockerFixture):
"""Test outputs property for chat completion compatibility."""
mock_output = mocker.Mock()
mock_request_output = mocker.Mock()
mock_request_output.outputs = [mock_output]
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="text",
request_output=mock_request_output,
)
assert output.outputs == [mock_output]
def test_outputs_empty_when_no_request_output(self):
"""Test outputs returns empty list when no request_output."""
output = OmniRequestOutput.from_diffusion(
request_id="test-123",
images=[],
prompt="a cat",
)
assert output.outputs == []
def test_encoder_prompt_token_ids_property(self, mocker: MockerFixture):
"""Test encoder_prompt_token_ids property."""
mock_request_output = mocker.Mock()
mock_request_output.encoder_prompt_token_ids = [10, 20, 30]
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="text",
request_output=mock_request_output,
)
assert output.encoder_prompt_token_ids == [10, 20, 30]
def test_num_cached_tokens_property(self, mocker: MockerFixture):
"""Test num_cached_tokens property."""
mock_request_output = mocker.Mock()
mock_request_output.num_cached_tokens = 42
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="text",
request_output=mock_request_output,
)
assert output.num_cached_tokens == 42
def test_multimodal_output_property(self, mocker: MockerFixture):
"""Test multimodal_output property."""
mock_request_output = mocker.Mock()
mock_audio = mocker.Mock()
expected_output = {"audio": mock_audio}
mock_request_output.outputs = []
mock_request_output.multimodal_output = expected_output
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="audio",
request_output=mock_request_output,
)
assert output.multimodal_output is expected_output
def test_to_dict_diffusion(self):
"""Test to_dict for diffusion output."""
output = OmniRequestOutput.from_diffusion(
request_id="test-123",
images=[Image.new("RGB", (64, 64), color="red")],
prompt="a cat",
metrics={"steps": 50},
)
result = output.to_dict()
assert result["request_id"] == "test-123"
assert result["finished"] is True
assert result["final_output_type"] == "image"
assert result["num_images"] == 1
assert result["prompt"] == "a cat"
def test_to_dict_pipeline(self, mocker: MockerFixture):
"""Test to_dict for pipeline output."""
mock_request_output = mocker.Mock()
mock_request_output.request_id = "pipeline-123"
output = OmniRequestOutput.from_pipeline(
stage_id=0,
final_output_type="text",
request_output=mock_request_output,
)
result = output.to_dict()
assert result["request_id"] == "pipeline-123"
assert result["finished"] is True
assert result["final_output_type"] == "text"
assert result["stage_id"] == 0
@@ -73,9 +73,17 @@ def orchestrator_factory():
class FakeDeadLLMStageClient(FakeStageClient):
"""LLM stage client that raises EngineDeadError on get_output_async."""
"""LLM stage client that raises EngineDeadError once a request is in-flight.
Raising on the first poll races with ``add_request``: under suite load the
orchestrator can die and drain before the test enqueues anything, so no
fatal ErrorMessage is ever published. Keep returning empty outputs until
``add_request_async`` has been called, then fail on the next poll.
"""
async def get_output_async(self):
if not self.add_request_calls:
return SimpleNamespace(outputs=[])
raise EngineDeadError("Stage-0 engine core is dead")
@@ -7,15 +7,24 @@ import asyncio
import queue
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
from unittest.mock import ANY, AsyncMock, MagicMock
import janus
import pytest
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.engine.orchestrator import (
Orchestrator,
OrchestratorRequestState,
_OrchestratorDuplexStagePort,
)
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.experimental.fullduplex.engine.contracts import (
DuplexStageRequestContext,
DuplexStageSubmission,
)
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -107,11 +116,50 @@ class FakePrewarmPool:
async def submit_initial(self, _request_id, _req_state, request, prompt_text=None):
self.submitted.append(request)
return 0
def get_bound_replica_id(self, _request_id):
return 0
def _duplex_stage_port_submission():
stage_pools = []
for stage_id in range(3):
pool = SimpleNamespace(
stage_client=SimpleNamespace(default_sampling_params=SamplingParams(max_tokens=1)),
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
submit_initial=AsyncMock(return_value=stage_id + 10),
submit_update=AsyncMock(return_value=stage_id + 20),
)
stage_pools.append(pool)
request_states: dict[str, OrchestratorRequestState] = {}
prewarm = AsyncMock()
port = _OrchestratorDuplexStagePort(
stage_pools=stage_pools,
request_states=request_states,
running_counter=None,
cleanup_request_ids=AsyncMock(),
async_chunk=True,
prewarm_async_chunk_stages=prewarm,
)
context = DuplexStageRequestContext(
request_id="req-duplex",
session_id="session-duplex",
fence=DuplexFence("session-duplex"),
stage_id=0,
final_stage_id=2,
config_generation=0,
sampling_params=tuple(SamplingParams(max_tokens=1) for _ in range(3)),
)
port.ensure_request(context)
submission = DuplexStageSubmission(
context=context,
prompt={"prompt_token_ids": [1, 2]},
already_submitted=False,
)
return port, stage_pools, request_states, prewarm, submission
def _request_output(request_id: str) -> RequestOutput:
completion = CompletionOutput(
index=0,
@@ -199,11 +247,13 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None:
stage2 = FakePrewarmPool("receiver")
orchestrator.stage_pools = [stage0, stage1, stage2]
orchestrator._emit_tx_edge = lambda **_kwargs: None
orchestrator._record_duplex_stage_submission = MagicMock()
req_state = OrchestratorRequestState(
request_id="req-prewarm",
prompt={"prompt_token_ids": [1, 2]},
sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(3)],
final_stage_id=2,
duplex_identity=SimpleNamespace(),
)
await orchestrator._prewarm_async_chunk_stages(
@@ -216,6 +266,23 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None:
assert len(stage2.submitted) == 1
assert 1 not in req_state.stage_submit_ts
assert 2 in req_state.stage_submit_ts
orchestrator._record_duplex_stage_submission.assert_called_once_with(
2,
"req-prewarm",
0,
req_state,
)
@pytest.mark.asyncio
async def test_duplex_prewarm_runs_after_first_stage0_submission() -> None:
port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission()
result = await port.submit(submission)
assert result.stage_id == 0
stage_pools[0].submit_initial.assert_awaited_once()
prewarm.assert_awaited_once_with("req-duplex", ANY, request_states["req-duplex"])
@pytest.mark.asyncio
+126 -1
View File
@@ -14,7 +14,7 @@ from vllm.v1.engine import FinishReason
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
from vllm_omni.outputs import output_processor
from vllm_omni.outputs.output_modality import OutputModalityNames
from vllm_omni.outputs.output_modality import OutputModality, OutputModalityNames
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -59,6 +59,10 @@ def _make_state(output_kind: RequestOutputKind):
return OmniRequestState(**_DEFAULT_STATE_KWARGS, output_kind=output_kind)
def test_output_modality_name_uses_string_value():
assert str(OutputModalityNames.AUDIO) == "audio"
def test_init_empty_dict():
"""Ensure mm_accumulated is initially empty."""
assert _make_state(RequestOutputKind.CUMULATIVE).mm_accumulated == {}
@@ -452,6 +456,127 @@ def test_no_detokenizer_make_request_output():
assert AUDIO in result.outputs[0].multimodal_output
def test_no_detokenizer_process_outputs_returns_nonterminal_audio_chunk(monkeypatch):
"""Generation-stage MM-only chunks must reach the orchestrator.
StagePool registers generation requests without a per-request collector,
so the processor return value is the only path from Stage2 to
Orchestrator._handle_processed_outputs().
"""
state = _make_no_detok_state(RequestOutputKind.DELTA)
processor = object.__new__(MultimodalOutputProcessor)
processor.output_modality = OutputModality.AUDIO
processor.request_states = {"r": state}
upstream_result = SimpleNamespace(request_outputs=[], reqs_to_abort=[])
monkeypatch.setattr(
VLLMOutputProcessor,
"process_outputs",
lambda *_args, **_kwargs: upstream_result,
)
engine_output = SimpleNamespace(
request_id="r",
multimodal_output={
"model_outputs": torch.arange(8, dtype=torch.float32),
"sr": torch.tensor(24000, dtype=torch.int32),
},
output_type="audio",
pooling_output=None,
new_token_ids=[],
finish_reason=None,
stop_reason=None,
kv_transfer_params=None,
routed_experts=None,
num_cached_tokens=0,
)
result = processor.process_outputs([engine_output])
assert len(result.request_outputs) == 1
assert result.request_outputs[0].request_id == "r"
assert AUDIO in result.request_outputs[0].outputs[0].multimodal_output
def _make_mm_only_output_processor(monkeypatch):
processor = object.__new__(MultimodalOutputProcessor)
processor.output_modality = OutputModality.AUDIO
processor.request_states = {"r": _make_no_detok_state(RequestOutputKind.DELTA)}
monkeypatch.setattr(
VLLMOutputProcessor,
"process_outputs",
lambda *_args, **_kwargs: SimpleNamespace(
request_outputs=[],
reqs_to_abort=[],
),
)
def finish_request(req_state):
processor.request_states.pop(req_state.request_id, None)
processor._finish_request = finish_request
return processor
def _audio_engine_output(*, is_segment_finished: bool, is_last_chunk: bool):
return SimpleNamespace(
request_id="r",
multimodal_output={
"model_outputs": torch.arange(8, dtype=torch.float32),
"meta.tts_is_last_chunk": torch.tensor([int(is_last_chunk)]),
},
output_type="audio",
pooling_output=None,
new_token_ids=[],
finish_reason=FinishReason.STOP,
stop_reason=None,
kv_transfer_params=None,
routed_experts=None,
num_cached_tokens=0,
is_segment_finished=is_segment_finished,
)
def test_mm_only_segment_finish_retains_request_state_for_next_audio_chunk(monkeypatch):
processor = _make_mm_only_output_processor(monkeypatch)
first = processor.process_outputs([_audio_engine_output(is_segment_finished=True, is_last_chunk=False)])
assert "r" in processor.request_states
assert len(first.request_outputs) == 1
assert first.request_outputs[0].finished is False
second = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)])
assert len(second.request_outputs) == 1
assert second.request_outputs[0].finished is True
assert "r" not in processor.request_states
def test_mm_only_nonfinal_audio_retains_state_without_segment_marker(monkeypatch):
processor = _make_mm_only_output_processor(monkeypatch)
first = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=False)])
assert "r" in processor.request_states
assert len(first.request_outputs) == 1
assert first.request_outputs[0].finished is False
second = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)])
assert len(second.request_outputs) == 1
assert second.request_outputs[0].finished is True
assert "r" not in processor.request_states
def test_mm_only_terminal_finish_removes_request_state(monkeypatch):
processor = _make_mm_only_output_processor(monkeypatch)
result = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)])
assert len(result.request_outputs) == 1
assert result.request_outputs[0].finished is True
assert "r" not in processor.request_states
def test_no_detokenizer_make_request_output_with_routed_experts():
"""make_request_output accepts the routed_experts arg that the multimodal
output channel (_process_mm_only_outputs) passes positionally, and attaches
+54
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import os
import socket
from contextlib import contextmanager
from types import SimpleNamespace
from typing import Any
@@ -11,6 +12,7 @@ import pytest
from pytest_mock import MockerFixture
from vllm.v1.engine.utils import EngineZmqAddresses
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
from vllm_omni.engine.stage_engine_startup import (
@@ -189,6 +191,34 @@ class TestOmniMasterServerAllocation:
output_port = int(alloc.output_bind_address.split(":")[-1])
assert len({handshake_port, input_port, output_port}) == 3
def test_route_ports_remain_os_reserved_until_master_releases_them(self):
server = OmniMasterServer(master_address="127.0.0.1", master_port=15005, stage_ids=[0])
alloc = server.get_allocation(0)
ports = [
int(alloc.handshake_bind_address.rsplit(":", 1)[-1]),
int(alloc.input_bind_address.rsplit(":", 1)[-1]),
int(alloc.output_bind_address.rsplit(":", 1)[-1]),
]
for port in ports:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
with pytest.raises(OSError):
probe.bind(("127.0.0.1", port))
server.release_route_port_reservations(0, handshake=True, data=False)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", ports[0]))
for port in ports[1:]:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
with pytest.raises(OSError):
probe.bind(("127.0.0.1", port))
server.stop()
for port in ports[1:]:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", port))
def test_get_zmq_addresses_returns_bind_addresses(self):
server = OmniMasterServer(master_address="127.0.0.1", master_port=15002, stage_ids=[0])
alloc = server.get_allocation(0)
@@ -397,6 +427,30 @@ class TestSingleStageModeDetection:
assert engine.single_stage_mode is False
assert engine._single_stage_id_filter is None
def test_stage_configs_path_loads_duplex_runtime_config(self, mocker: MockerFixture):
duplex_session = DuplexSessionRuntimeConfig(max_sessions=2)
get_pipeline_config = mocker.patch(
"vllm_omni.engine.async_omni_engine.StageConfigFactory.get_pipeline_config",
return_value=None,
)
load_deploy_config = mocker.patch(
"vllm_omni.engine.async_omni_engine.load_deploy_config",
return_value=SimpleNamespace(duplex_session=duplex_session),
)
engine = self._make_engine_no_thread(
mocker,
stage_configs_path="/fake/duplex.yaml",
)
get_pipeline_config.assert_called_once_with(
model="fake-model",
trust_remote_code=False,
deploy_config_path="/fake/duplex.yaml",
)
load_deploy_config.assert_called_once_with("/fake/duplex.yaml")
assert engine.duplex_session_config is duplex_session
def test_single_stage_mode_without_stage_id_has_no_filter(self, mocker: MockerFixture):
engine = self._make_engine_no_thread(
mocker,