[Core][Frontend] Unified Full-duplex Framework (#7413)
Signed-off-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9005d78903
commit
99ff4f3070
182 files changed
+26951
-34743
No files matched your search
@@ -21,20 +21,6 @@ from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs, FinishReason
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
from vllm_omni.engine.duplex.control_plane import DuplexControlPlane
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
AppendDuplexInputMessage,
|
||||
CloseDuplexSessionMessage,
|
||||
DuplexFence,
|
||||
OpenDuplexSessionMessage,
|
||||
SignalDuplexTurnMessage,
|
||||
)
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexInputMode,
|
||||
DuplexRuntimeCapabilities,
|
||||
DuplexSessionRuntimeState,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.engine.messages import (
|
||||
AbortRequestMessage,
|
||||
AbortResultMessage,
|
||||
@@ -54,7 +40,6 @@ from vllm_omni.engine.orchestrator import (
|
||||
)
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import MiniCPMO45DuplexRuntimeExtension
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -1118,778 +1103,6 @@ async def test_handle_abort_emits_error_result_on_failure() -> None:
|
||||
assert "cleanup exploded" in (result.error or "")
|
||||
|
||||
|
||||
def _duplex_open_message(
|
||||
session_id: str,
|
||||
*,
|
||||
incarnation: int = 0,
|
||||
session_config: dict[str, object] | None = None,
|
||||
runtime_config: dict[str, object] | None = None,
|
||||
) -> OpenDuplexSessionMessage:
|
||||
return OpenDuplexSessionMessage(
|
||||
control_id=f"open-{session_id}",
|
||||
fence=DuplexFence(session_id, incarnation=incarnation),
|
||||
session_id=session_id,
|
||||
capabilities={
|
||||
"input_modes": [DuplexInputMode.APPEND_AUDIO_CHUNK.value],
|
||||
"implementation_level": "model_native_duplex",
|
||||
},
|
||||
session_config=session_config or {},
|
||||
runtime_config=runtime_config or {},
|
||||
)
|
||||
|
||||
|
||||
async def _handle_duplex(orchestrator: Orchestrator, message: object) -> None:
|
||||
await orchestrator._require_duplex_control_plane().handle(message)
|
||||
|
||||
|
||||
def _duplex_request_state(
|
||||
orchestrator: Orchestrator,
|
||||
session: DuplexSessionRuntimeState,
|
||||
*,
|
||||
stage_id: int,
|
||||
) -> OrchestratorRequestState | None:
|
||||
context = orchestrator._require_duplex_control_plane().ensure_stage_request(
|
||||
session,
|
||||
stage_id=stage_id,
|
||||
)
|
||||
if context is None:
|
||||
return None
|
||||
return orchestrator.request_states.get(context.request_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_control_plane_keeps_public_and_runtime_config_separate() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
open_message = _duplex_open_message(
|
||||
"sid-config-channels",
|
||||
incarnation=3,
|
||||
session_config={
|
||||
"instructions": "public instructions",
|
||||
"extra_body": {"duplex_stage_max_tokens": {"0": 99}},
|
||||
},
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 3},
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151705]}},
|
||||
},
|
||||
)
|
||||
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
session = orchestrator.duplex_sessions.require(open_message.session_id)
|
||||
request_state = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert request_state.sampling_params_list[0].max_tokens == 3
|
||||
assert request_state.sampling_params_list[0].stop_token_ids == [151705]
|
||||
bridge = request_state.streaming.bridge_states["duplex"]
|
||||
assert bridge["incarnation"] == open_message.fence.incarnation
|
||||
assert bridge["session_config"] == open_message.session_config
|
||||
assert bridge["runtime_config"] == open_message.runtime_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_control_plane_preserves_turn_commit_without_model_extension() -> None:
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=[],
|
||||
duplex_runtime_extension=None,
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
assert isinstance(orchestrator.duplex_control_plane, DuplexControlPlane)
|
||||
|
||||
message = OpenDuplexSessionMessage(
|
||||
control_id="open-turn-commit",
|
||||
fence=DuplexFence("sid-turn-commit"),
|
||||
session_id="sid-turn-commit",
|
||||
capabilities={"input_modes": [DuplexInputMode.TURN_COMMIT_ONLY.value]},
|
||||
session_config={},
|
||||
)
|
||||
await _handle_duplex(orchestrator, message)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
assert result.stage_results[0]["result"]["scheduler_request_context"] is False
|
||||
|
||||
|
||||
def test_ordinary_orchestrator_bypasses_duplex_control_plane() -> None:
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=asyncio.Queue(),
|
||||
stage_pools=[],
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
)
|
||||
|
||||
assert orchestrator.duplex_control_plane is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_close_cleans_preregistered_request_without_append() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
running_counter = FakeRunningCounter()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
running_counter=running_counter,
|
||||
)
|
||||
open_message = _duplex_open_message("sid-preregister-close")
|
||||
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
open_result = rpc_q.get_nowait()
|
||||
request_id = open_result.stage_results[0]["result"]["request_id"]
|
||||
assert request_id in orchestrator.request_states
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
CloseDuplexSessionMessage(
|
||||
control_id="close-preregistered",
|
||||
fence=open_message.fence,
|
||||
session_id=open_message.session_id,
|
||||
),
|
||||
)
|
||||
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert request_id not in orchestrator.request_states
|
||||
assert orchestrator.duplex_sessions.get(open_message.session_id) is None
|
||||
assert running_counter.value == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_open_failure_rolls_back_session_and_reserved_request() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
open_message = _duplex_open_message(
|
||||
"sid-open-rollback",
|
||||
runtime_config={
|
||||
"duplex_stage_sampling_params": {
|
||||
"0": {"stop_token_ids": 123},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
|
||||
assert rpc_q.get_nowait().ok is False
|
||||
assert orchestrator.duplex_sessions.get(open_message.session_id) is None
|
||||
assert orchestrator.request_states == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_running_counter_tracks_only_submitted_request() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
running_counter = FakeRunningCounter()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
running_counter=running_counter,
|
||||
)
|
||||
open_message = _duplex_open_message("sid-duplex-counter")
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert running_counter.value == 0
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-duplex-counter",
|
||||
fence=open_message.fence,
|
||||
session_id=open_message.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert running_counter.value == 1
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
CloseDuplexSessionMessage(
|
||||
control_id="close-duplex-counter",
|
||||
fence=open_message.fence,
|
||||
session_id=open_message.session_id,
|
||||
),
|
||||
)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert running_counter.value == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_failed_append_does_not_advance_sequence_or_fence() -> None:
|
||||
class FailingPlanExtension(MiniCPMO45DuplexRuntimeExtension):
|
||||
def plan_append(self, **kwargs):
|
||||
raise RuntimeError("planned append failure")
|
||||
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=FailingPlanExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-append-rollback", turn_id=1)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
DuplexFence("sid-append-rollback"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-rollback",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert session.fence == DuplexFence("sid-append-rollback")
|
||||
assert session.input_seq == 0
|
||||
assert session.input_turn_seq == 0
|
||||
assert stage0.add_request_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_duplicate_append_operation_is_submitted_once() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-idempotent-append")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
for control_id in ("append-first", "append-timeout-retry"):
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id=control_id,
|
||||
operation_id="physical-input-1",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
first = rpc_q.get_nowait()
|
||||
retry = rpc_q.get_nowait()
|
||||
assert first.ok is True
|
||||
assert retry.ok is True
|
||||
assert first.stage_results == retry.stage_results
|
||||
assert len(stage0.add_request_calls) == 1
|
||||
assert session.input_seq == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_barge_in_aborts_bound_stage_requests_before_releasing_fence() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0], [stage1]],
|
||||
output_processors=[FakeOutputProcessor(), FakeOutputProcessor()],
|
||||
stage_vllm_configs=[
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
DuplexFence("sid-stage-signal"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(0, "req-stage0", fence=session.fence)
|
||||
session.bind_stage_request(1, "req-stage1", fence=session.fence)
|
||||
assert stage_pools[0].select_replica_id("req-stage0") == 0
|
||||
assert stage_pools[1].select_replica_id("req-stage1") == 0
|
||||
session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=session.fence,
|
||||
)
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="ctrl-signal",
|
||||
fence=session.fence,
|
||||
next_fence=DuplexFence("sid-stage-signal", epoch=1),
|
||||
session_id="sid-stage-signal",
|
||||
event="barge_in",
|
||||
),
|
||||
)
|
||||
|
||||
assert stage0.abort_calls == [["req-stage0"]]
|
||||
assert stage1.abort_calls == [["req-stage1"]]
|
||||
assert stage0.collective_rpc_calls == []
|
||||
assert stage1.collective_rpc_calls == []
|
||||
assert session.stage_bindings == {}
|
||||
assert session.fence == DuplexFence("sid-stage-signal", epoch=1)
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_late_barge_in_releases_only_cancelled_fence_bindings() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0], [stage1]],
|
||||
output_processors=[FakeOutputProcessor(), FakeOutputProcessor()],
|
||||
stage_vllm_configs=[
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
cancelled_fence = DuplexFence("sid-late-stage-signal")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
cancelled_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(0, "req-stage0", fence=cancelled_fence)
|
||||
session.bind_stage_request(1, "req-stage1", fence=cancelled_fence)
|
||||
assert stage_pools[0].select_replica_id("req-stage0") == 0
|
||||
assert stage_pools[1].select_replica_id("req-stage1") == 0
|
||||
current_fence = DuplexFence("sid-late-stage-signal", epoch=1)
|
||||
session.accept_fence(current_fence)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="ctrl-late-signal",
|
||||
fence=cancelled_fence,
|
||||
next_fence=current_fence,
|
||||
session_id="sid-late-stage-signal",
|
||||
event="barge_in",
|
||||
),
|
||||
)
|
||||
|
||||
assert stage0.abort_calls == [["req-stage0"]]
|
||||
assert stage1.abort_calls == [["req-stage1"]]
|
||||
assert session.stage_bindings == {}
|
||||
assert session.fence == current_fence
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_cancel_without_next_fence_is_rejected_without_releasing_bindings() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-cancel-contract")
|
||||
session = orchestrator.duplex_sessions.open_session(fence)
|
||||
session.bind_stage_request(0, "req-live", fence=fence)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="cancel-without-next",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="input.cancel",
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert result.error_count == 1
|
||||
assert "next_fence" in result.stage_results[0]["result"]["error"]
|
||||
assert session.fence == fence
|
||||
assert session.stage_request_ids() == ["req-live"]
|
||||
assert stage0.abort_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_session_update_replaces_runtime_config() -> None:
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=[],
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-update-config")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
session_config={"temperature": 0.7},
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="update-config",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="session.update",
|
||||
session_config={"temperature": 0.0, "instructions": "updated"},
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
assert session.session_config == {"temperature": 0.0, "instructions": "updated"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_session_update_refreshes_next_append_sampling_params() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-update-policy")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 2},
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151645]}},
|
||||
},
|
||||
)
|
||||
req_state = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert req_state is not None
|
||||
assert req_state.sampling_params_list[0].max_tokens == 2
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="update-policy",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="session.update",
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 7},
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151705]}},
|
||||
},
|
||||
),
|
||||
)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-updated-policy",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert req_state.sampling_params_list[0].max_tokens == 7
|
||||
assert req_state.sampling_params_list[0].stop_token_ids == [151705]
|
||||
submitted_request = stage0.add_request_calls[0][0]
|
||||
assert submitted_request.sampling_params.stop_token_ids == [151705]
|
||||
assert submitted_request.sampling_params.max_tokens == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_invalid_session_update_preserves_previous_config() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-invalid-update")
|
||||
original_runtime_config = {
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151645]}},
|
||||
}
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
runtime_config=original_runtime_config,
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="invalid-update",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="session.update",
|
||||
runtime_config={
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": 123}},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert session.runtime_config == original_runtime_config
|
||||
assert session.config_generation == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_arbitrary_non_cancel_signal_is_rejected() -> None:
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=[],
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-unsupported-signal")
|
||||
orchestrator.duplex_sessions.open_session(fence)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="unsupported-signal",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="turn.end",
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert "unsupported duplex runtime signal" in result.stage_results[0]["result"]["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_cancel_rejects_late_old_append_and_accepts_next_epoch() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
cancelled_fence = DuplexFence("sid-cancel-late-append")
|
||||
next_fence = DuplexFence("sid-cancel-late-append", epoch=1)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
cancelled_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="cancel-old-fence",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
session_id=session.session_id,
|
||||
event="input.cancel",
|
||||
),
|
||||
)
|
||||
|
||||
cancel_result = rpc_q.get_nowait()
|
||||
assert cancel_result.ok is True
|
||||
assert session.fence == next_fence
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="late-old-append",
|
||||
fence=cancelled_fence,
|
||||
session_id=session.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
stale_result = rpc_q.get_nowait()
|
||||
assert stale_result.ok is False
|
||||
assert stage0.add_request_calls == []
|
||||
assert session.stage_bindings == {}
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="next-epoch-append",
|
||||
fence=next_fence,
|
||||
session_id=session.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
next_result = rpc_q.get_nowait()
|
||||
assert next_result.ok is True
|
||||
assert len(stage0.add_request_calls) == 1
|
||||
assert session.stage_bindings[0].fence == next_fence
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_append_updates_bridge_turn_id_on_long_lived_stage0_request() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=asyncio.Queue(),
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
DuplexFence("sid-bridge-turn"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
session_config={
|
||||
"voice": "test",
|
||||
},
|
||||
runtime_config={
|
||||
"duplex_stage_sampling_params": {
|
||||
"0": {"stop_token_ids": [151645]},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-1",
|
||||
fence=DuplexFence("sid-bridge-turn"),
|
||||
session_id="sid-bridge-turn",
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
req_state1 = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert req_state1 is not None
|
||||
duplex_state1 = req_state1.streaming.bridge_states["duplex"]
|
||||
assert duplex_state1["session_id"] == "sid-bridge-turn"
|
||||
assert duplex_state1["epoch"] == 0
|
||||
assert duplex_state1["turn_id"] == 0
|
||||
assert duplex_state1["session_config"] == session.session_config
|
||||
assert req_state1.sampling_params_list[0].stop_token_ids == [151645]
|
||||
submitted_request = stage0.add_request_calls[0][0]
|
||||
assert submitted_request.sampling_params.stop_token_ids == [151645]
|
||||
assert submitted_request.sampling_params.max_tokens == 1
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-2",
|
||||
fence=DuplexFence("sid-bridge-turn", turn_id=1, response_seq=1),
|
||||
session_id="sid-bridge-turn",
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True, "new_user_turn": True},
|
||||
),
|
||||
)
|
||||
req_state2 = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert req_state2 is req_state1
|
||||
duplex_state2 = req_state2.streaming.bridge_states["duplex"]
|
||||
assert duplex_state2["turn_id"] == 1
|
||||
assert duplex_state2["epoch"] == 0
|
||||
expected_request_id = duplex_resource_request_id(DuplexFence("sid-bridge-turn"), "stage0")
|
||||
assert [call[0].request_id for call in stage0.add_request_calls] == [
|
||||
expected_request_id,
|
||||
expected_request_id,
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-replica tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -2641,61 +1854,6 @@ async def test_stage_pool_process_llm_raw_outputs_mutates_iteration_stats() -> N
|
||||
assert iteration_stats.num_generation_tokens == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_reaper_loop_waits_between_ticks():
|
||||
class _Plane:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def reap_expired(self) -> int:
|
||||
self.calls += 1
|
||||
return 0
|
||||
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.duplex_control_plane = _Plane()
|
||||
orchestrator._duplex_reaper_interval_s = 0.01
|
||||
orchestrator._shutdown_event = asyncio.Event()
|
||||
|
||||
task = asyncio.create_task(orchestrator._duplex_reaper_loop())
|
||||
await asyncio.sleep(0.035)
|
||||
orchestrator._shutdown_event.set()
|
||||
await task
|
||||
|
||||
assert 2 <= orchestrator.duplex_control_plane.calls <= 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("first_cleanup_delay", [0.0, 0.05], ids=["immediate", "delayed"])
|
||||
async def test_duplex_reaper_loop_survives_one_cleanup_failure(first_cleanup_delay: float) -> None:
|
||||
recovered = asyncio.Event()
|
||||
|
||||
class _Plane:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def reap_expired(self) -> int:
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
await asyncio.sleep(first_cleanup_delay)
|
||||
raise RuntimeError("transient cleanup failure")
|
||||
recovered.set()
|
||||
return 0
|
||||
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.duplex_control_plane = _Plane()
|
||||
orchestrator._duplex_reaper_interval_s = 0.01
|
||||
orchestrator._shutdown_event = asyncio.Event()
|
||||
|
||||
task = asyncio.create_task(orchestrator._duplex_reaper_loop())
|
||||
try:
|
||||
# Wait for recovery itself, including when cleanup exceeds the old 35 ms window.
|
||||
await asyncio.wait_for(recovered.wait(), timeout=5.0)
|
||||
assert orchestrator.duplex_control_plane.calls >= 2
|
||||
finally:
|
||||
orchestrator._shutdown_event.set()
|
||||
await asyncio.wait_for(task, timeout=5.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_retry_does_not_repeat_successful_stage_abort():
|
||||
class _Pool:
|
||||
@@ -2733,48 +1891,3 @@ async def test_abort_retry_does_not_repeat_successful_stage_abort():
|
||||
assert first.physical_abort_calls == 1
|
||||
assert second.physical_abort_calls == 2
|
||||
assert second.bound == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_cleanup_failure_is_deferred_to_control_plane():
|
||||
class _FailingPool:
|
||||
async def abort_requests(self, request_ids: list[str]) -> None:
|
||||
raise RuntimeError("stage abort failed")
|
||||
|
||||
def release_bindings(self, request_ids: list[str]) -> None:
|
||||
pass
|
||||
|
||||
class _Plane:
|
||||
def __init__(self) -> None:
|
||||
self.deferred: list[str] = []
|
||||
self.finalized: list[str] = []
|
||||
|
||||
def close_sessions_for_request_ids(self, request_ids: list[str], **kwargs):
|
||||
assert kwargs == {"abort": True, "cleanup_in_progress": True}
|
||||
return {"sid-cleanup": ["req-a"]}
|
||||
|
||||
def defer_request_cleanups(self, session_ids: list[str]) -> None:
|
||||
self.deferred.extend(session_ids)
|
||||
|
||||
def finalize_closed_sessions(self, session_ids: list[str]) -> None:
|
||||
self.finalized.extend(session_ids)
|
||||
|
||||
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
|
||||
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.stage_pools = [_FailingPool()]
|
||||
orchestrator.duplex_control_plane = _Plane()
|
||||
orchestrator._pd_kv_params = {}
|
||||
orchestrator.request_states = {}
|
||||
orchestrator._running_counter = None
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
|
||||
with pytest.raises(RuntimeError, match="stage abort failed"):
|
||||
await orchestrator._cleanup_request_ids(
|
||||
["req-a"],
|
||||
abort=True,
|
||||
close_duplex_sessions=True,
|
||||
)
|
||||
|
||||
assert orchestrator.duplex_control_plane.deferred == ["sid-cleanup"]
|
||||
assert orchestrator.duplex_control_plane.finalized == []
|
||||
Reference in new issue
Block a user