[Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196)
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
179 files changed
+6462
-3832
No files matched your search
@@ -1,110 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_control_client import (
|
||||
DuplexControlClient,
|
||||
DuplexControlRequestError,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.lease import DuplexLeaseActivity
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import (
|
||||
DuplexControlError,
|
||||
DuplexControlResultMessage,
|
||||
DuplexFence,
|
||||
ResumeDuplexSessionMessage,
|
||||
TouchDuplexSessionMessage,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _Transport:
|
||||
def __init__(self) -> None:
|
||||
self.calls = []
|
||||
|
||||
def execute(self, key, message, **kwargs):
|
||||
self.calls.append((key, message, kwargs))
|
||||
return DuplexControlResultMessage(
|
||||
control_id=message.control_id,
|
||||
fence=message.fence,
|
||||
operation="touch" if isinstance(message, TouchDuplexSessionMessage) else "resume",
|
||||
session_id=message.session_id,
|
||||
ok=True,
|
||||
stage_results=[],
|
||||
)
|
||||
|
||||
|
||||
class _UnsupportedTransport:
|
||||
def execute(self, key, message, **kwargs):
|
||||
del key, kwargs
|
||||
return DuplexControlResultMessage(
|
||||
control_id=message.control_id,
|
||||
fence=message.fence,
|
||||
operation="touch",
|
||||
session_id=message.session_id,
|
||||
ok=False,
|
||||
stage_results=[{"result": {"supported": False}}],
|
||||
unsupported_count=1,
|
||||
error=DuplexControlError(code="invalid_capability", message="unsupported", retryable=False),
|
||||
accepted_fence=DuplexFence(message.session_id, epoch=2),
|
||||
lease_generation=7,
|
||||
)
|
||||
|
||||
|
||||
def test_control_client_routes_touch_and_resume_by_control_id() -> None:
|
||||
transport = _Transport()
|
||||
control_ids = iter(("touch-id", "resume-id"))
|
||||
client = DuplexControlClient(transport, control_id_factory=lambda: next(control_ids))
|
||||
fence = DuplexFence("sid-client")
|
||||
|
||||
assert (
|
||||
client.touch(
|
||||
fence.session_id,
|
||||
fence=fence,
|
||||
activity=DuplexLeaseActivity.PLAYBACK_ACK,
|
||||
timeout=2.0,
|
||||
)["ok"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
client.resume(
|
||||
fence.session_id,
|
||||
fence=fence,
|
||||
expected_lease_generation=7,
|
||||
timeout=3.0,
|
||||
)["ok"]
|
||||
is True
|
||||
)
|
||||
|
||||
touch_key, touch_message, touch_kwargs = transport.calls[0]
|
||||
assert touch_key == ("duplex", "touch-id")
|
||||
assert isinstance(touch_message, TouchDuplexSessionMessage)
|
||||
assert touch_message.activity == DuplexLeaseActivity.PLAYBACK_ACK.value
|
||||
assert touch_kwargs["timeout"] == 2.0
|
||||
|
||||
resume_key, resume_message, resume_kwargs = transport.calls[1]
|
||||
assert resume_key == ("duplex", "resume-id")
|
||||
assert isinstance(resume_message, ResumeDuplexSessionMessage)
|
||||
assert resume_message.expected_lease_generation == 7
|
||||
assert resume_kwargs["timeout"] == 3.0
|
||||
|
||||
|
||||
def test_control_client_treats_ok_false_as_authoritative_failure() -> None:
|
||||
client = DuplexControlClient(_UnsupportedTransport(), control_id_factory=lambda: "unsupported")
|
||||
fence = DuplexFence("sid-unsupported")
|
||||
|
||||
with pytest.raises(DuplexControlRequestError, match="duplex touch failed") as exc_info:
|
||||
client.touch(
|
||||
fence.session_id,
|
||||
fence=fence,
|
||||
activity=DuplexLeaseActivity.HEARTBEAT,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "invalid_capability"
|
||||
assert exc_info.value.retryable is False
|
||||
assert exc_info.value.accepted_fence == DuplexFence(fence.session_id, epoch=2)
|
||||
assert exc_info.value.lease_generation == 7
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Focused tests for typed deployment-only runtime configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.stage_config import get_deploy_duplex_max_sessions
|
||||
from vllm_omni.config.stage_config import (
|
||||
DuplexSessionRuntimeConfig,
|
||||
load_deploy_config,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_duplex_session_runtime_defaults_are_typed_and_immutable(tmp_path) -> None:
|
||||
deploy_path = tmp_path / "duplex.yaml"
|
||||
deploy_path.write_text("duplex_session: {}\nstages: []\n", encoding="utf-8")
|
||||
|
||||
deploy = load_deploy_config(deploy_path)
|
||||
|
||||
assert isinstance(deploy.duplex_session, DuplexSessionRuntimeConfig)
|
||||
assert deploy.duplex_session.idle_ttl_s == 300.0
|
||||
assert deploy.duplex_session.disconnect_grace_s == 30.0
|
||||
assert deploy.duplex_session.reaper_interval_s == 5.0
|
||||
assert deploy.duplex_session.resume_replay_ttl_s == 60.0
|
||||
assert deploy.duplex_session.resume_replay_max_bytes_per_session == 8 * 1024 * 1024
|
||||
assert deploy.duplex_session.max_pending_input_bytes_per_session == 16 * 1024 * 1024
|
||||
assert deploy.duplex_session.max_pending_turns_per_session == 4
|
||||
assert deploy.duplex_session.max_sessions == 1
|
||||
assert deploy.duplex_session.completed_append_cache_size == 256
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
deploy.duplex_session.idle_ttl_s = 1.0 # type: ignore[misc]
|
||||
|
||||
|
||||
def test_duplex_session_runtime_accepts_disabled_idle_expiry(tmp_path) -> None:
|
||||
deploy_path = tmp_path / "duplex.yaml"
|
||||
deploy_path.write_text("duplex_session:\n idle_ttl_s: null\nstages: []\n", encoding="utf-8")
|
||||
|
||||
deploy = load_deploy_config(deploy_path)
|
||||
|
||||
assert deploy.duplex_session.idle_ttl_s is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "value"),
|
||||
[
|
||||
("idle_ttl_s", 0),
|
||||
("idle_ttl_s", -1),
|
||||
("disconnect_grace_s", 0),
|
||||
("reaper_interval_s", -1),
|
||||
("resume_replay_ttl_s", 0),
|
||||
("resume_replay_max_bytes_per_session", -1),
|
||||
("max_pending_input_bytes_per_session", 0),
|
||||
("max_pending_turns_per_session", -1),
|
||||
("max_sessions", 0),
|
||||
("completed_append_cache_size", 0),
|
||||
],
|
||||
)
|
||||
def test_duplex_session_runtime_rejects_non_positive_values(tmp_path, name: str, value: int) -> None:
|
||||
deploy_path = tmp_path / "duplex.yaml"
|
||||
deploy_path.write_text(f"duplex_session:\n {name}: {value}\nstages: []\n", encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match=rf"duplex_session\.{name} must be positive"):
|
||||
load_deploy_config(deploy_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deploy_yaml", "expected"),
|
||||
[
|
||||
("minicpmo_4_5.yaml", 4),
|
||||
("minicpmo_4_5_8x4090.yaml", 1),
|
||||
("minicpmo_4_5_3gpu_stage1_replicas.yaml", 4),
|
||||
],
|
||||
)
|
||||
def test_deploy_duplex_max_sessions_tracks_the_deploy_config(deploy_yaml: str, expected: int) -> None:
|
||||
# Guards the admission probe. A capacity edit, a config that declares none,
|
||||
# or an overlay inheriting one from its base must surface here rather than
|
||||
# as another nightly duplex admission timeout.
|
||||
assert get_deploy_duplex_max_sessions(deploy_yaml) == expected
|
||||
@@ -1,14 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.intermediate import get_stream_request_key
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_get_stream_request_key_requires_stable_identifier():
|
||||
with pytest.raises(ValueError, match="stable request id"):
|
||||
get_stream_request_key({"ids": {"tts": [1, 2, 3]}})
|
||||
|
||||
|
||||
def test_get_stream_request_key_accepts_global_request_id():
|
||||
assert get_stream_request_key({"global_request_id": ["duplex-sid-stage0"]}) == "duplex-sid-stage0"
|
||||
@@ -1,224 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import DuplexInputMode
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_session import DuplexSessionRuntimeManager
|
||||
from vllm_omni.experimental.fullduplex.engine.lease import (
|
||||
DuplexLeaseActivity,
|
||||
DuplexLeaseConfig,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeMonotonicClock:
|
||||
def __init__(self, value: float) -> None:
|
||||
self.value = value
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.value
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.value += seconds
|
||||
|
||||
|
||||
def _lease_config(*, idle_ttl_s: float | None = 300.0) -> DuplexLeaseConfig:
|
||||
return DuplexLeaseConfig(
|
||||
idle_ttl_s=idle_ttl_s,
|
||||
disconnect_grace_s=30.0,
|
||||
)
|
||||
|
||||
|
||||
def test_open_touch_detach_and_idle_expiry_use_monotonic_time() -> None:
|
||||
clock = FakeMonotonicClock(100.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-expiry")
|
||||
session = manager.open_session(fence, lease_config=_lease_config())
|
||||
|
||||
assert session.lease.last_activity == 100.0
|
||||
clock.advance(10.0)
|
||||
session.touch(fence, DuplexLeaseActivity.HEARTBEAT)
|
||||
session.detach(fence)
|
||||
clock.advance(29.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
clock.advance(272.0)
|
||||
expired = manager.collect_expired()
|
||||
|
||||
assert [item.session_id for item in expired] == ["sid-expiry"]
|
||||
assert expired[0].reason == "disconnect_grace_expired"
|
||||
assert manager.get("sid-expiry") is session
|
||||
manager.finalize_close_session(session)
|
||||
assert manager.get("sid-expiry") is None
|
||||
|
||||
|
||||
def test_detach_grace_and_resume_advance_token_independent_lease_generation() -> None:
|
||||
clock = FakeMonotonicClock(10.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-resume")
|
||||
session = manager.open_session(fence, lease_config=_lease_config())
|
||||
|
||||
session.detach(fence)
|
||||
clock.advance(29.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
clock.advance(2.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is True
|
||||
|
||||
generation = session.resume(fence, expected_lease_generation=0)
|
||||
|
||||
assert generation == 1
|
||||
assert session.lease.detached_at is None
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
with pytest.raises(ValueError, match="lease generation mismatch"):
|
||||
session.resume(fence, expected_lease_generation=0)
|
||||
|
||||
|
||||
def test_active_operation_prevents_mid_transaction_expiry() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-operation")
|
||||
session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=5.0))
|
||||
|
||||
session.begin_operation(fence, "append-1")
|
||||
clock.advance(10.0)
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
session.end_operation(fence, "append-1")
|
||||
assert manager.collect_expired() == []
|
||||
clock.advance(6.0)
|
||||
assert [item.session_id for item in manager.collect_expired()] == ["sid-operation"]
|
||||
|
||||
|
||||
def test_sessions_have_independent_activity_deadlines_and_resources() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence_a = DuplexFence("sid-a")
|
||||
fence_b = DuplexFence("sid-b")
|
||||
session_a = manager.open_session(fence_a, lease_config=_lease_config(idle_ttl_s=10.0))
|
||||
session_b = manager.open_session(fence_b, lease_config=_lease_config(idle_ttl_s=10.0))
|
||||
session_a.reserve_stage_request(0, "req-a-reserved", fence=fence_a)
|
||||
session_a.bind_stage_request(1, "req-a-submitted", fence=fence_a)
|
||||
session_b.bind_stage_request(0, "req-b", fence=fence_b)
|
||||
|
||||
clock.advance(6.0)
|
||||
session_b.touch(fence_b, DuplexLeaseActivity.MODEL_OUTPUT)
|
||||
clock.advance(5.0)
|
||||
expired = manager.collect_expired()
|
||||
|
||||
assert len(expired) == 1
|
||||
assert expired[0].session_id == "sid-a"
|
||||
assert expired[0].reserved_request_ids == ("req-a-reserved",)
|
||||
assert expired[0].submitted_request_ids == ("req-a-submitted",)
|
||||
assert manager.get("sid-b") is session_b
|
||||
assert session_b.resource_request_ids() == ["req-b"]
|
||||
|
||||
|
||||
def test_close_and_reaper_select_exactly_one_terminal_transition() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-race")
|
||||
manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=1.0))
|
||||
clock.advance(2.0)
|
||||
|
||||
assert len(manager.collect_expired()) == 1
|
||||
assert manager.close_session(fence, reason="explicit_close") is None
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
fence_2 = DuplexFence("sid-race-2")
|
||||
manager.open_session(fence_2, lease_config=_lease_config(idle_ttl_s=1.0))
|
||||
assert manager.close_session(fence_2, reason="explicit_close") is not None
|
||||
clock.advance(2.0)
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
|
||||
def test_stale_fence_cannot_touch_detach_or_resume_lease() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
current = DuplexFence("sid-fence", epoch=1)
|
||||
stale = DuplexFence("sid-fence", epoch=0)
|
||||
session = manager.open_session(current, lease_config=_lease_config())
|
||||
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.touch(stale, DuplexLeaseActivity.APPEND)
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.detach(stale)
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.resume(stale, expected_lease_generation=0)
|
||||
|
||||
|
||||
def test_disabled_idle_expiry_never_collects_session() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
manager.open_session(DuplexFence("sid-no-expiry"), lease_config=_lease_config(idle_ttl_s=None))
|
||||
|
||||
clock.advance(1_000_000.0)
|
||||
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
|
||||
def test_detached_session_expires_at_disconnect_grace_when_idle_ttl_is_disabled() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-disconnect-grace")
|
||||
session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=None))
|
||||
|
||||
session.detach(fence)
|
||||
clock.advance(29.0)
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
clock.advance(1.0)
|
||||
expired = manager.collect_expired()
|
||||
|
||||
assert [item.session_id for item in expired] == [fence.session_id]
|
||||
assert expired[0].reason == "disconnect_grace_expired"
|
||||
|
||||
|
||||
def test_runtime_manager_enforces_server_owned_session_admission() -> None:
|
||||
manager = DuplexSessionRuntimeManager(max_sessions=2)
|
||||
manager.open_session(DuplexFence("sid-admission-a"))
|
||||
manager.open_session(DuplexFence("sid-admission-b"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="duplex_session_capacity_exhausted"):
|
||||
manager.open_session(DuplexFence("sid-admission-c"))
|
||||
|
||||
|
||||
def test_completed_append_cache_is_bounded() -> None:
|
||||
manager = DuplexSessionRuntimeManager(completed_append_limit=2)
|
||||
fence = DuplexFence("sid-completed-cache")
|
||||
session = manager.open_session(fence)
|
||||
|
||||
for index in range(3):
|
||||
session.record_completed_append(
|
||||
f"operation-{index}",
|
||||
fence=fence,
|
||||
mode=DuplexInputMode.TURN_COMMIT_ONLY,
|
||||
final=True,
|
||||
stage_results=[{"index": index}],
|
||||
)
|
||||
|
||||
assert list(session.completed_appends) == ["operation-1", "operation-2"]
|
||||
|
||||
session.accept_fence(DuplexFence(fence.session_id, epoch=1))
|
||||
assert session.completed_appends == {}
|
||||
|
||||
|
||||
def test_lease_config_and_expiry_record_are_immutable() -> None:
|
||||
config = _lease_config()
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
config.idle_ttl_s = 1.0 # type: ignore[misc]
|
||||
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
manager.open_session(DuplexFence("sid-record"), lease_config=_lease_config(idle_ttl_s=1.0))
|
||||
clock.advance(2.0)
|
||||
record = manager.collect_expired()[0]
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
record.reason = "changed" # type: ignore[misc]
|
||||
@@ -1,480 +0,0 @@
|
||||
from dataclasses import FrozenInstanceError
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine import duplex_runtime
|
||||
from vllm_omni.experimental.fullduplex.engine.contracts import (
|
||||
duplex_data_plane_request_info,
|
||||
duplex_resource_request_belongs_to_session,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
|
||||
DuplexInputMode,
|
||||
DuplexOutputAction,
|
||||
DuplexOutputDecision,
|
||||
DuplexRuntimeCapabilities,
|
||||
DuplexSessionRuntimeManager,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.runtime import (
|
||||
MiniCPMO45DuplexRuntimeExtension,
|
||||
build_duplex_data_plane_prompt,
|
||||
duplex_scheduler_token_budget,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_duplex_fence_is_immutable():
|
||||
fence = DuplexFence("session")
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
fence.epoch = 1 # type: ignore[misc]
|
||||
|
||||
assert not hasattr(fence, "__dict__")
|
||||
|
||||
|
||||
def test_duplex_runtime_tracks_stage_bindings_and_barge_in_epoch():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
DuplexFence("sid-1"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_TOKENS},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(stage_id=0, request_id="req-stage0", fence=session.fence)
|
||||
session.bind_stage_request(stage_id=1, request_id="req-stage1", fence=session.fence)
|
||||
|
||||
update = session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence)
|
||||
next_fence = DuplexFence("sid-1", epoch=1)
|
||||
stale_request_ids = session.release_fence(session.fence)
|
||||
session.accept_fence(next_fence)
|
||||
|
||||
assert update.seq == 1
|
||||
assert session.fence == next_fence
|
||||
assert stale_request_ids == ["req-stage0", "req-stage1"]
|
||||
assert session.stage_bindings == {}
|
||||
|
||||
|
||||
def test_duplex_runtime_tracks_same_request_id_for_each_pipeline_stage():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
fence = DuplexFence("sid-shared-pipeline-request")
|
||||
session = manager.open_session(fence)
|
||||
|
||||
session.reserve_stage_request(0, "req-shared", fence=fence)
|
||||
session.bind_stage_request(0, "req-shared", fence=fence)
|
||||
session.bind_stage_request(1, "req-shared", fence=fence)
|
||||
|
||||
assert session.stage_bindings[0].request_id == "req-shared"
|
||||
assert session.stage_bindings[1].request_id == "req-shared"
|
||||
assert session.resource_request_ids() == ["req-shared"]
|
||||
assert session.input_seq == 0
|
||||
|
||||
|
||||
def test_duplex_runtime_extension_validation_rejects_missing_methods():
|
||||
class IncompleteExtension:
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return defaults
|
||||
|
||||
with pytest.raises(TypeError, match="plan_append"):
|
||||
duplex_runtime.validate_duplex_runtime_extension(IncompleteExtension())
|
||||
|
||||
|
||||
def test_duplex_runtime_extension_validation_rejects_stage_count_mismatch():
|
||||
class WrongStageCountExtension(MiniCPMO45DuplexRuntimeExtension):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return defaults[:1]
|
||||
|
||||
with pytest.raises(ValueError, match="one sampling parameter per stage"):
|
||||
duplex_runtime.validate_duplex_runtime_extension(
|
||||
WrongStageCountExtension(),
|
||||
sampling_defaults=(SamplingParams(), SamplingParams()),
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_runtime_extension_validation_rejects_sampling_type_mismatch():
|
||||
class WrongSamplingTypeExtension(MiniCPMO45DuplexRuntimeExtension):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return tuple(object() for _ in defaults)
|
||||
|
||||
with pytest.raises(TypeError, match="sampling parameter type"):
|
||||
duplex_runtime.validate_duplex_runtime_extension(
|
||||
WrongSamplingTypeExtension(),
|
||||
sampling_defaults=(SamplingParams(), SamplingParams()),
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_output_decision_metadata_is_immutable():
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={"model_listen": True},
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
decision.metadata["model_listen"] = False
|
||||
|
||||
|
||||
def _decide_minicpmo_output(
|
||||
output: object,
|
||||
*,
|
||||
segment_token_ids: tuple[int, ...] = (),
|
||||
segment_output_metadata: dict | None = None,
|
||||
):
|
||||
return MiniCPMO45DuplexRuntimeExtension().decide_output(
|
||||
stage_id=0,
|
||||
final_stage_id=1,
|
||||
segment_finished=True,
|
||||
segment_token_ids=segment_token_ids,
|
||||
segment_output_metadata=segment_output_metadata or {},
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
def test_minicpmo_extension_owns_stage_sampling_overrides():
|
||||
defaults = (
|
||||
SamplingParams(max_tokens=4),
|
||||
SamplingParams(max_tokens=8),
|
||||
)
|
||||
|
||||
configured = MiniCPMO45DuplexRuntimeExtension().configure_sampling_params(
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 20},
|
||||
"duplex_stage_sampling_params": {"1": {"stop_token_ids": [151645]}},
|
||||
},
|
||||
defaults=defaults,
|
||||
)
|
||||
|
||||
assert configured[0].max_tokens == 20
|
||||
assert configured[1].stop_token_ids == [151645]
|
||||
assert defaults[0].max_tokens == 4
|
||||
assert 151645 not in (defaults[1].stop_token_ids or [])
|
||||
|
||||
|
||||
def test_minicpmo_output_decision_uses_raw_streaming_token_snapshot():
|
||||
decision = _decide_minicpmo_output(
|
||||
SimpleNamespace(outputs=[SimpleNamespace()]),
|
||||
segment_token_ids=(151705,),
|
||||
segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}},
|
||||
)
|
||||
|
||||
assert decision is not None
|
||||
assert decision.action is DuplexOutputAction.DIRECT_RESPONSE
|
||||
assert decision.metadata["duplex_native_decision"] == "listen"
|
||||
assert decision.metadata["model_listen"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"])
|
||||
def test_minicpmo_output_decision_ignores_output_level_token_history(attr):
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
|
||||
outputs=[SimpleNamespace()],
|
||||
**{attr: [42, 151705]},
|
||||
)
|
||||
|
||||
assert _decide_minicpmo_output(output) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"])
|
||||
def test_minicpmo_output_decision_uses_completion_token_ids(attr):
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
|
||||
outputs=[SimpleNamespace(**{attr: [42, 151705]})],
|
||||
)
|
||||
|
||||
assert _decide_minicpmo_output(output) is not None
|
||||
|
||||
|
||||
def test_minicpmo_output_decision_uses_completion_stop_reason():
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
|
||||
outputs=[SimpleNamespace(stop_reason=151705)],
|
||||
)
|
||||
|
||||
assert _decide_minicpmo_output(output) is not None
|
||||
|
||||
|
||||
def test_duplex_runtime_cancel_fence_rejects_late_append_and_accepts_next_epoch():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
cancelled_fence = DuplexFence("sid-cancel-race")
|
||||
next_fence = DuplexFence("sid-cancel-race", epoch=1)
|
||||
session = manager.open_session(
|
||||
cancelled_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(
|
||||
stage_id=0,
|
||||
request_id="req-cancelled",
|
||||
fence=cancelled_fence,
|
||||
)
|
||||
|
||||
stale_request_ids = session.cancel_fence(cancelled_fence, next_fence)
|
||||
|
||||
assert stale_request_ids == ["req-cancelled"]
|
||||
assert session.fence == next_fence
|
||||
assert session.stage_bindings == {}
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=cancelled_fence,
|
||||
)
|
||||
update = session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=next_fence,
|
||||
)
|
||||
assert update.seq == 1
|
||||
|
||||
|
||||
def test_duplex_runtime_stale_close_preserves_live_session_and_bindings():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
current_fence = DuplexFence("sid-stale-close", epoch=1)
|
||||
session = manager.open_session(
|
||||
current_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(0, "req-live", fence=current_fence)
|
||||
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
manager.close_session(DuplexFence("sid-stale-close"))
|
||||
|
||||
assert manager.get("sid-stale-close") is session
|
||||
assert session.stage_request_ids() == ["req-live"]
|
||||
|
||||
|
||||
def test_duplex_runtime_reopen_rejects_late_append_from_old_incarnation():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
old_fence = DuplexFence("sid-reopen", incarnation=0)
|
||||
old_session = manager.open_session(
|
||||
old_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
manager.close_session(old_fence)
|
||||
|
||||
new_fence = DuplexFence("sid-reopen", incarnation=1)
|
||||
new_session = manager.open_session(
|
||||
new_fence,
|
||||
capabilities=old_session.capabilities,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
new_session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=old_fence,
|
||||
)
|
||||
assert (
|
||||
new_session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=new_fence,
|
||||
).seq
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_prompt_expands_incarnation_metadata():
|
||||
fence = DuplexFence("sid-incarnation", incarnation=3)
|
||||
|
||||
prompt = build_duplex_data_plane_prompt(
|
||||
request_id=duplex_resource_request_id(fence, "stage0"),
|
||||
fence=fence,
|
||||
session_config={},
|
||||
runtime_config={},
|
||||
seq=1,
|
||||
turn_seq=1,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload={"is_speech": True},
|
||||
final=False,
|
||||
)
|
||||
|
||||
assert prompt["model_intermediate_buffer"]["duplex"]["incarnation"] == 3
|
||||
|
||||
|
||||
def test_duplex_runtime_tracks_turn_local_append_sequence():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
DuplexFence("sid-turn-seq"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
first = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence)
|
||||
second = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence)
|
||||
next_turn = DuplexFence("sid-turn-seq", turn_id=1, response_seq=1)
|
||||
third = session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=next_turn,
|
||||
)
|
||||
fourth = session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=next_turn,
|
||||
)
|
||||
|
||||
assert [first.seq, second.seq, third.seq, fourth.seq] == [1, 2, 3, 4]
|
||||
assert [first.turn_seq, second.turn_seq, third.turn_seq, fourth.turn_seq] == [1, 2, 1, 2]
|
||||
assert [first.turn_id, second.turn_id, third.turn_id, fourth.turn_id] == [0, 0, 1, 1]
|
||||
|
||||
|
||||
def test_duplex_runtime_rejects_unsupported_append_mode():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
DuplexFence("sid-2"),
|
||||
capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.TURN_COMMIT_ONLY}),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="not supported"):
|
||||
session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence)
|
||||
|
||||
|
||||
def test_duplex_data_plane_request_info_extracts_structured_stage_result():
|
||||
request_id, response_stage_id = duplex_data_plane_request_info(
|
||||
{
|
||||
"stage_results": [
|
||||
{"result": {"supported": True}},
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": "duplex-sid-e0-stage0-s1",
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert request_id == "duplex-sid-e0-stage0-s1"
|
||||
assert response_stage_id == 1
|
||||
|
||||
|
||||
def test_duplex_data_plane_request_info_rejects_missing_request_id():
|
||||
assert duplex_data_plane_request_info(
|
||||
{
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": "",
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
) == (None, None)
|
||||
|
||||
|
||||
def test_duplex_scheduler_token_budget_estimates_pcm_slots():
|
||||
assert (
|
||||
duplex_scheduler_token_budget(
|
||||
{
|
||||
"audio": "AAAAAA==",
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
== 16
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_scheduler_token_budget_ignores_client_budget_fields():
|
||||
assert (
|
||||
duplex_scheduler_token_budget(
|
||||
{
|
||||
"audio": "AAAAAA==",
|
||||
"format": "pcm_f32le",
|
||||
"duplex_num_input_tokens": 999,
|
||||
"num_input_tokens": 999,
|
||||
}
|
||||
)
|
||||
== 16
|
||||
)
|
||||
|
||||
|
||||
def test_resource_state_rejects_fence_regression_and_requires_explicit_fence():
|
||||
current = DuplexFence("sid", epoch=2, turn_id=3, response_seq=4)
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
current,
|
||||
capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}),
|
||||
)
|
||||
|
||||
for stale in (
|
||||
DuplexFence("sid", epoch=1, turn_id=99, response_seq=99),
|
||||
DuplexFence("sid", epoch=2, turn_id=2, response_seq=4),
|
||||
DuplexFence("sid", epoch=2, turn_id=3, response_seq=3),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.accept_fence(stale)
|
||||
assert session.fence == current
|
||||
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
session.bind_stage_request(0, "request")
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK)
|
||||
with pytest.raises(TypeError, match="DuplexFence"):
|
||||
manager.open_session("legacy-session")
|
||||
with pytest.raises(TypeError, match="DuplexFence"):
|
||||
manager.close_session("legacy-session")
|
||||
|
||||
|
||||
def test_resource_request_id_is_derived_from_fence_and_role():
|
||||
fence = DuplexFence("sid-with-dashes", epoch=7, turn_id=11, response_seq=13)
|
||||
|
||||
assert duplex_resource_request_id(fence, "stage0") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage0"
|
||||
assert duplex_resource_request_id(fence, "stage1") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage1"
|
||||
|
||||
|
||||
def test_resource_request_id_codec_separates_session_id_from_incarnation():
|
||||
embedded_incarnation = duplex_resource_request_id(
|
||||
DuplexFence("foo-i1", incarnation=0),
|
||||
"stage0",
|
||||
)
|
||||
actual_incarnation = duplex_resource_request_id(
|
||||
DuplexFence("foo", incarnation=1),
|
||||
"stage0",
|
||||
)
|
||||
|
||||
assert embedded_incarnation != actual_incarnation
|
||||
|
||||
|
||||
def test_resource_request_id_session_membership_uses_encoded_identity():
|
||||
request_id = duplex_resource_request_id(
|
||||
DuplexFence("sid-with-dashes", incarnation=2, epoch=7),
|
||||
"stage0",
|
||||
)
|
||||
|
||||
assert duplex_resource_request_belongs_to_session(request_id, "sid-with-dashes") is True
|
||||
assert duplex_resource_request_belongs_to_session(request_id, "sid") is False
|
||||
assert duplex_resource_request_belongs_to_session("duplex-s.invalid.i.x.e.7.r.stage0", "sid") is False
|
||||
|
||||
|
||||
def test_placeholder_budget_is_planned_inside_omni_engine_boundary():
|
||||
fence = DuplexFence("sid", turn_id=1, response_seq=1)
|
||||
prompt = build_duplex_data_plane_prompt(
|
||||
request_id=duplex_resource_request_id(fence, "stage0"),
|
||||
fence=fence,
|
||||
session_config={},
|
||||
runtime_config={},
|
||||
seq=2,
|
||||
turn_seq=1,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload={
|
||||
"audio": "AAAAAA==",
|
||||
"format": "pcm_f32le",
|
||||
"duplex_num_input_tokens": 999,
|
||||
"num_input_tokens": 999,
|
||||
},
|
||||
final=False,
|
||||
)
|
||||
|
||||
assert len(prompt["prompt_token_ids"]) == 16
|
||||
assert prompt["model_intermediate_buffer"]["duplex"]["fence"] == fence
|
||||
assert prompt["model_intermediate_buffer"]["duplex"]["scheduler_token_budget"] == 16
|
||||
Whitespace-only changes.
@@ -1,99 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.commit_policy import (
|
||||
CommitAction,
|
||||
CommitSnapshot,
|
||||
decide_commit_action,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "snapshot", "expected"),
|
||||
[
|
||||
(
|
||||
"first idle commit",
|
||||
CommitSnapshot(
|
||||
auto_responds=True,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=False,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.START_AUTO_RESPONSE,
|
||||
),
|
||||
(
|
||||
"native auto-response remains model-owned during playback",
|
||||
CommitSnapshot(
|
||||
auto_responds=True,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=True,
|
||||
playback_active=True,
|
||||
),
|
||||
CommitAction.START_AUTO_RESPONSE,
|
||||
),
|
||||
(
|
||||
"native auto-response commit without speech",
|
||||
CommitSnapshot(
|
||||
auto_responds=True,
|
||||
speech_since_commit=False,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=True,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.COMMIT_ONLY,
|
||||
),
|
||||
(
|
||||
"non-auto response overlap",
|
||||
CommitSnapshot(
|
||||
auto_responds=False,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=True,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.DEFER_ACTIVE_RESPONSE,
|
||||
),
|
||||
(
|
||||
"non-auto playback overlap",
|
||||
CommitSnapshot(
|
||||
auto_responds=False,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=False,
|
||||
playback_active=True,
|
||||
),
|
||||
CommitAction.DEFER_ACTIVE_RESPONSE,
|
||||
),
|
||||
(
|
||||
"non-auto commit without overlap",
|
||||
CommitSnapshot(
|
||||
auto_responds=False,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=False,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.COMMIT_ONLY,
|
||||
),
|
||||
],
|
||||
ids=lambda value: value if isinstance(value, str) else None,
|
||||
)
|
||||
def test_decide_commit_action_covers_realtime_lifecycle_sequences(
|
||||
case: str,
|
||||
snapshot: CommitSnapshot,
|
||||
expected: CommitAction,
|
||||
) -> None:
|
||||
del case
|
||||
|
||||
assert decide_commit_action(snapshot) is expected
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.input import MiniCPMO45PcmAppendBuffer
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def pcm_payload(samples: int, *, speech: bool = True) -> dict[str, object]:
|
||||
audio = np.ones(samples, dtype=np.float32).tobytes()
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(audio).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16_000,
|
||||
"is_speech": speech,
|
||||
}
|
||||
|
||||
|
||||
def test_commit_does_not_add_silence_after_incremental_audio_was_drained():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
|
||||
emitted = buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
|
||||
committed = buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
assert emitted is not None
|
||||
assert not buffer.has_pending()
|
||||
assert committed is None
|
||||
|
||||
|
||||
def test_append_emits_one_model_unit_when_multiple_units_are_buffered():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
|
||||
emitted = buffer.append(pcm_payload(32_000), chunk_period_ms=1_000)
|
||||
|
||||
assert emitted is not None
|
||||
assert len(base64.b64decode(emitted["audio"])) == 16_000 * 4
|
||||
assert buffer.pending_byte_count == 16_000 * 4
|
||||
|
||||
|
||||
def test_speech_marker_does_not_leak_across_irregular_chunk_boundaries():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
|
||||
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
|
||||
speech_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
|
||||
silence_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
|
||||
|
||||
assert speech_unit is not None
|
||||
assert speech_unit["is_speech"] is True
|
||||
assert silence_unit is not None
|
||||
assert silence_unit["is_speech"] is False
|
||||
|
||||
|
||||
def test_force_listen_marker_does_not_leak_across_irregular_chunk_boundaries():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
forced = pcm_payload(15_000, speech=False)
|
||||
forced["force_listen"] = True
|
||||
|
||||
assert buffer.append(forced, chunk_period_ms=1_000) is None
|
||||
forced_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
|
||||
unforced_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
|
||||
|
||||
assert forced_unit is not None
|
||||
assert forced_unit["force_listen"] is True
|
||||
assert unforced_unit is not None
|
||||
assert unforced_unit["force_listen"] is False
|
||||
|
||||
|
||||
def test_pcm_append_rollback_restores_per_span_speech_markers():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
|
||||
reservation = buffer.prepare_append(
|
||||
pcm_payload(2_000, speech=False),
|
||||
operation_id="mixed-unit",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert reservation is not None
|
||||
reservation.rollback()
|
||||
first = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
|
||||
second = buffer.flush(chunk_period_ms=1_000)
|
||||
|
||||
assert first is not None
|
||||
assert first["is_speech"] is True
|
||||
assert second is not None
|
||||
assert second["is_speech"] is False
|
||||
|
||||
|
||||
def test_commit_without_speech_does_not_synthesize_terminal_audio():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
buffer.append(pcm_payload(8_000, speech=False), chunk_period_ms=1_000)
|
||||
|
||||
committed = buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
assert committed is None
|
||||
|
||||
|
||||
def test_commit_resets_cumulative_turn_accounting():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
|
||||
buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
empty = buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
assert empty is None
|
||||
|
||||
|
||||
def test_pcm_append_reservation_rollback_restores_emitted_audio():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
original = pcm_payload(16_000)
|
||||
|
||||
reservation = buffer.prepare_append(
|
||||
original,
|
||||
operation_id="append-1",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert reservation is not None
|
||||
assert not buffer.has_pending()
|
||||
reservation.rollback()
|
||||
assert buffer.has_pending()
|
||||
|
||||
retried = buffer.flush(chunk_period_ms=1_000)
|
||||
assert retried is not None
|
||||
assert base64.b64decode(retried["audio"]) == base64.b64decode(original["audio"])
|
||||
|
||||
|
||||
def test_pcm_commit_keeps_prior_append_reservation_active():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
append_reservation = buffer.prepare_append(
|
||||
pcm_payload(16_000),
|
||||
operation_id="append-before-commit",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
commit_reservation = buffer.prepare_commit(
|
||||
operation_id="commit-after-append",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert append_reservation is not None
|
||||
assert append_reservation.active
|
||||
assert commit_reservation.payload is None
|
||||
append_reservation.commit()
|
||||
commit_reservation.commit()
|
||||
|
||||
|
||||
def test_pcm_commit_reservation_rollback_restores_residual_audio():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
original = pcm_payload(8_000)
|
||||
assert (
|
||||
buffer.prepare_append(
|
||||
original,
|
||||
operation_id="buffer-half-chunk",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
reservation = buffer.prepare_commit(
|
||||
operation_id="final-half-chunk",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert reservation.payload is not None
|
||||
assert reservation.payload["final"] is True
|
||||
reservation.rollback()
|
||||
|
||||
retried = buffer.prepare_commit(
|
||||
operation_id="retry-final-half-chunk",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
assert retried.payload is not None
|
||||
assert base64.b64decode(retried.payload["audio"]) == (base64.b64decode(original["audio"]) + b"\x00" * (8_000 * 4))
|
||||
@@ -1,94 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import duplex_resource_request_id
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.data_plane import (
|
||||
NemotronVoiceChatDataPlaneContext,
|
||||
NemotronVoiceChatDataPlaneSession,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
_RUNTIME = {
|
||||
"nvc_text_bos_id": 0,
|
||||
"nvc_text_eos_id": 1,
|
||||
"nvc_text_pad_id": 12,
|
||||
"nvc_function_sotc_id": 20,
|
||||
"nvc_function_eotc_id": 21,
|
||||
"nvc_tokenizer_ref": "test-tokenizer",
|
||||
}
|
||||
|
||||
|
||||
def _projector(encode_audio=lambda *_: None) -> NemotronVoiceChatDataPlaneSession:
|
||||
projector = NemotronVoiceChatDataPlaneSession(encode_audio)
|
||||
projector.configure_runtime(_RUNTIME, tokenizer=SimpleNamespace(decode=lambda *_args, **_kwargs: "text"))
|
||||
return projector
|
||||
|
||||
|
||||
def _stage0_output(request_id: str = "req-0", *, text_token: int = 12, function_token: int | None = None) -> object:
|
||||
metadata: dict[str, object] = {"nvc_text_token_ids": [text_token]}
|
||||
if function_token is not None:
|
||||
metadata["nvc_function_token"] = [function_token]
|
||||
completion = SimpleNamespace(multimodal_output=metadata)
|
||||
return SimpleNamespace(
|
||||
stage_id=0,
|
||||
request_output=SimpleNamespace(request_id=request_id, outputs=[completion]),
|
||||
)
|
||||
|
||||
|
||||
def test_function_channel_projects_completed_call_without_ending_speech() -> None:
|
||||
projector = _projector()
|
||||
projector._decode = lambda token_ids: (
|
||||
'[{"name":"weather","arguments":{"city":"Shanghai"}}]' if token_ids == [99] else ""
|
||||
)
|
||||
context = NemotronVoiceChatDataPlaneContext(epoch=0)
|
||||
|
||||
events = []
|
||||
for function_token in (20, 99, 21):
|
||||
events.extend(projector.project_output(_stage0_output(function_token=function_token), context=context))
|
||||
|
||||
listen = [event for event in events if event.get("is_listen") is True]
|
||||
function = [event for event in events if event.get("function_call") is True]
|
||||
assert len(listen) == 3
|
||||
assert len(function) == 1
|
||||
assert function[0]["name"] == "weather"
|
||||
assert function[0]["arguments"] == '{"city":"Shanghai"}'
|
||||
|
||||
projector._decode = lambda _token_ids: "not-json"
|
||||
bad_events = []
|
||||
for function_token in (20, 98, 21):
|
||||
bad_events.extend(projector.project_output(_stage0_output(function_token=function_token)))
|
||||
assert any(event.get("error_code") == "nemotron_function_call_parse_error" for event in bad_events)
|
||||
|
||||
|
||||
def test_new_epoch_request_does_not_inherit_partial_function_or_frame_state() -> None:
|
||||
projector = _projector(lambda *_: "audio")
|
||||
old_request = duplex_resource_request_id(DuplexFence("sid", epoch=1), "stage0")
|
||||
new_request = duplex_resource_request_id(DuplexFence("sid", epoch=2), "stage0")
|
||||
|
||||
projector.begin_request(old_request)
|
||||
list(projector.project_output(_stage0_output(old_request, text_token=42, function_token=20)))
|
||||
projector.begin_request(new_request)
|
||||
|
||||
# EOTC in the new epoch must not close the old epoch's partial function.
|
||||
new_events = list(projector.project_output(_stage0_output(new_request, text_token=1, function_token=21)))
|
||||
assert not [event for event in new_events if event.get("function_call") is True]
|
||||
assert not [event for event in new_events if event.get("end_of_turn") is True]
|
||||
|
||||
audio = SimpleNamespace(
|
||||
stage_id=2,
|
||||
request_output=SimpleNamespace(
|
||||
request_id=new_request,
|
||||
outputs=[
|
||||
SimpleNamespace(multimodal_output={"model_outputs": [np.ones(1764, dtype=np.float32)], "sr": [22050]})
|
||||
],
|
||||
),
|
||||
)
|
||||
audio_events = list(projector.project_output(audio))
|
||||
assert sum(event.get("end_of_turn") is True for event in audio_events) == 1
|
||||
@@ -1,73 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.input import (
|
||||
NemotronVoiceChatPcmAppendBuffer,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _payload(samples: np.ndarray, *, sample_rate_hz: int = 16000, fmt: str = "pcm_f32le") -> dict[str, object]:
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(np.asarray(samples, dtype=np.float32).tobytes()).decode("ascii"),
|
||||
"format": fmt,
|
||||
"sample_rate_hz": sample_rate_hz,
|
||||
}
|
||||
|
||||
|
||||
def _append(buffer, samples, operation_id):
|
||||
return buffer.prepare_append(
|
||||
_payload(np.asarray(samples, dtype=np.float32)),
|
||||
operation_id=operation_id,
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
|
||||
def test_irregular_browser_packets_emit_exact_ordered_80ms_frame() -> None:
|
||||
buffer = NemotronVoiceChatPcmAppendBuffer()
|
||||
assert _append(buffer, np.arange(1000), "packet-1") is None
|
||||
|
||||
reservation = _append(buffer, np.arange(1000, 1280), "packet-2")
|
||||
|
||||
assert reservation is not None
|
||||
assert reservation.operation_id == "packet-2"
|
||||
decoded = np.frombuffer(base64.b64decode(reservation.payload["audio"]), dtype=np.float32)
|
||||
np.testing.assert_array_equal(decoded, np.arange(1280, dtype=np.float32))
|
||||
assert buffer.pending_byte_count == 0
|
||||
|
||||
|
||||
def test_append_rejects_invalid_frame_contract() -> None:
|
||||
cases = (
|
||||
(1281, 16000, "pcm_f32le", 0.0, "at most 1280"),
|
||||
(1, 8000, "pcm_f32le", 0.0, "sample_rate_hz"),
|
||||
(1, 16000, "audio/pcm", 0.0, "format"),
|
||||
(1, 16000, "pcm_f32le", np.nan, "finite"),
|
||||
)
|
||||
for samples, sample_rate_hz, fmt, value, message in cases:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
NemotronVoiceChatPcmAppendBuffer().prepare_append(
|
||||
_payload(np.full(samples, value, dtype=np.float32), sample_rate_hz=sample_rate_hz, fmt=fmt),
|
||||
operation_id="invalid",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
|
||||
def test_terminal_commit_rejects_more_than_one_frame_tail() -> None:
|
||||
buffer = NemotronVoiceChatPcmAppendBuffer()
|
||||
for index, samples in enumerate((1000, 281)):
|
||||
buffer.prepare_append(
|
||||
_payload(np.zeros(samples, dtype=np.float32)),
|
||||
operation_id=f"tail-{index}",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=False,
|
||||
)
|
||||
with pytest.raises(ValueError, match="more than one 80 ms frame"):
|
||||
buffer.prepare_commit(operation_id="commit", chunk_period_ms=80)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.contracts import DuplexInputMode
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.runtime import (
|
||||
NemotronVoiceChatDuplexRuntimeExtension,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.nemotron_voicechat.serving_adapter import (
|
||||
NemotronVoiceChatServingRuntimeAdapter,
|
||||
_render_tool_response,
|
||||
)
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
|
||||
NemotronVoiceChatThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _frame() -> dict[str, object]:
|
||||
raw = np.zeros(1280, dtype=np.float32).tobytes()
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(raw).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
|
||||
|
||||
def _plan(extension, *, input_seq: int):
|
||||
return extension.plan_append(
|
||||
request_id="req",
|
||||
fence=DuplexFence("sid", incarnation=2, epoch=3),
|
||||
session_config={},
|
||||
runtime_config={
|
||||
"nvc_prompt_token_ids": [0, 42, 1],
|
||||
"nvc_text_pad_id": 12,
|
||||
"nvc_max_model_len": 8192,
|
||||
},
|
||||
seq=input_seq,
|
||||
turn_seq=input_seq,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload=_frame(),
|
||||
final=False,
|
||||
sampling_params=SamplingParams(),
|
||||
)
|
||||
|
||||
|
||||
def test_first_append_prefills_prompt_then_each_append_consumes_one_frame() -> None:
|
||||
extension = NemotronVoiceChatDuplexRuntimeExtension()
|
||||
|
||||
params = extension.configure_sampling_params(
|
||||
runtime_config={}, defaults=(SamplingParams(temperature=0.7, max_tokens=4),)
|
||||
)[0]
|
||||
assert params.temperature == 0.0 and params.max_tokens == 1 and params.top_p == 1.0 and params.top_k == 0
|
||||
|
||||
first = _plan(extension, input_seq=1)
|
||||
later = _plan(extension, input_seq=2)
|
||||
|
||||
assert first.prompt["prompt_token_ids"] == [0, 42, 1, 12]
|
||||
assert later.prompt["prompt_token_ids"] == [12]
|
||||
assert first.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 1
|
||||
assert later.prompt["model_intermediate_buffer"]["duplex"]["source_input_seq"] == 2
|
||||
|
||||
|
||||
def test_append_rejects_stage0_context_overflow() -> None:
|
||||
with pytest.raises(ValueError, match="max_model_len"):
|
||||
_plan(NemotronVoiceChatDuplexRuntimeExtension(), input_seq=8190)
|
||||
|
||||
|
||||
def test_function_output_becomes_versioned_nvidia_channel_tokens() -> None:
|
||||
encoded: list[str] = []
|
||||
adapter = NemotronVoiceChatServingRuntimeAdapter(lambda *_: None)
|
||||
adapter._tokenizer = SimpleNamespace(
|
||||
encode=lambda text, **_kwargs: encoded.append(text) or [31, 32, 33],
|
||||
)
|
||||
|
||||
first = adapter.runtime_config_for_function_output(
|
||||
{"type": "function_call_output", "call_id": "call-1", "output": '{"result":20}'},
|
||||
{},
|
||||
)
|
||||
second = adapter.runtime_config_for_function_output(
|
||||
{"type": "function_call_output", "call_id": "call-2", "output": "plain text"},
|
||||
first,
|
||||
)
|
||||
|
||||
assert _render_tool_response('{"result":20}') == '<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>'
|
||||
assert encoded == [
|
||||
'<TOOL_RESPONSE>[{"result":20}]</TOOL_RESPONSE>',
|
||||
'<TOOL_RESPONSE>["plain text"]</TOOL_RESPONSE>',
|
||||
]
|
||||
assert first["nvc_function_response_generation"] == 1
|
||||
assert second["nvc_function_response_generation"] == 2
|
||||
assert second["nvc_function_response_token_ids"] == [31, 32, 33]
|
||||
assert second["nvc_function_response_call_id"] == "call-2"
|
||||
assert second["nvc_function_response_batches"] == [
|
||||
{"generation": 1, "call_id": "call-1", "token_ids": [31, 32, 33]},
|
||||
{"generation": 2, "call_id": "call-2", "token_ids": [31, 32, 33]},
|
||||
]
|
||||
|
||||
session = {"function_response_generation": 0, "forced_function_tokens": []}
|
||||
NemotronVoiceChatThinkerForConditionalGeneration._sync_forced_function_response(session, second)
|
||||
|
||||
assert session["function_response_generation"] == 2
|
||||
assert session["forced_function_token"] == 31
|
||||
assert session["forced_function_tokens"] == [31, 32, 33, 31, 32, 33]
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.audio import convert_input_audio_with_rate
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_pcm16_input_conversion_is_pure_and_reports_target_rate():
|
||||
pcm16 = np.array([-32768, 0, 32767], dtype="<i2").tobytes()
|
||||
encoded = base64.b64encode(pcm16).decode("ascii")
|
||||
|
||||
converted, fmt, sample_rate = convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
|
||||
assert encoded == base64.b64encode(pcm16).decode("ascii")
|
||||
assert fmt == "pcm_f32le"
|
||||
assert sample_rate == 16_000
|
||||
samples = np.frombuffer(base64.b64decode(converted), dtype="<f4")
|
||||
assert np.allclose(samples, [-1.0, 0.0, 32767 / 32768])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz", [8_000, 192_000])
|
||||
def test_pcm16_input_conversion_accepts_supported_sample_rate_boundaries(sample_rate_hz: int):
|
||||
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
|
||||
|
||||
_, fmt, converted_rate = convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=sample_rate_hz,
|
||||
)
|
||||
|
||||
assert fmt == "pcm_f32le"
|
||||
assert converted_rate == 16_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz", [7_999, 192_001, 10**1000])
|
||||
def test_pcm16_input_conversion_rejects_sample_rates_outside_supported_range(sample_rate_hz: int):
|
||||
encoded = base64.b64encode(np.zeros(8, dtype="<i2").tobytes()).decode("ascii")
|
||||
|
||||
with pytest.raises(ValueError, match="sample_rate_hz"):
|
||||
convert_input_audio_with_rate(
|
||||
encoded,
|
||||
"pcm16",
|
||||
sample_rate_hz=sample_rate_hz,
|
||||
)
|
||||
@@ -1,139 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import (
|
||||
validate_serving_runtime_adapter,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name",
|
||||
[
|
||||
"vllm_omni.experimental.fullduplex.openai.session_runner",
|
||||
"vllm_omni.experimental.fullduplex.openai.runtime_bridge",
|
||||
"vllm_omni.experimental.fullduplex.openai.serving",
|
||||
],
|
||||
)
|
||||
def test_generic_openai_runtime_import_does_not_load_minicpmo45(module_name: str) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = f"""
|
||||
import sys
|
||||
|
||||
import {module_name}
|
||||
|
||||
loaded = sorted(
|
||||
name
|
||||
for name in sys.modules
|
||||
if name == "vllm_omni.experimental.fullduplex.minicpmo45"
|
||||
or name.startswith("vllm_omni.experimental.fullduplex.minicpmo45.")
|
||||
)
|
||||
if loaded:
|
||||
raise SystemExit("generic OpenAI runtime loaded MiniCPM-o: " + ", ".join(loaded))
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_generic_handler_requires_explicit_serving_runtime_adapter() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = """
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.serving import OmniDuplexSessionHandler
|
||||
|
||||
chat_service = SimpleNamespace(engine_client=SimpleNamespace())
|
||||
try:
|
||||
OmniDuplexSessionHandler(chat_service=chat_service)
|
||||
except ValueError as exc:
|
||||
if "serving runtime adapter" not in str(exc).lower():
|
||||
raise
|
||||
else:
|
||||
raise SystemExit("generic handler silently selected a model serving adapter")
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def _valid_runtime_adapter() -> SimpleNamespace:
|
||||
data_plane = SimpleNamespace(
|
||||
begin_request=lambda request_id: None,
|
||||
is_terminal=lambda request_id: False,
|
||||
mark_terminal=lambda request_id: None,
|
||||
close_stream=lambda request_id: None,
|
||||
close_session=lambda session_id, **kwargs: None,
|
||||
project=lambda result, **kwargs: (),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
adapter_id="test-adapter",
|
||||
session_states={},
|
||||
data_plane=data_plane,
|
||||
clean_response_done_prefix="clean",
|
||||
interrupted_tts_prefix="interrupted",
|
||||
private_runtime_config_keys=frozenset({"private_key"}),
|
||||
create_session_state=lambda: object(),
|
||||
session_state=lambda session_id: object(),
|
||||
remove_session_state=lambda session_id: None,
|
||||
is_enabled=lambda config: True,
|
||||
capabilities=lambda **kwargs: object(),
|
||||
validate_client_extra_body=lambda extra_body: None,
|
||||
prepare_runtime_config=lambda config, **kwargs: {},
|
||||
runtime_config_for_update=lambda config, current: {},
|
||||
data_plane_context=lambda **kwargs: object(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "invalid_value"),
|
||||
[
|
||||
("session_states", None),
|
||||
("clean_response_done_prefix", None),
|
||||
("interrupted_tts_prefix", None),
|
||||
("private_runtime_config_keys", {"private_key"}),
|
||||
],
|
||||
)
|
||||
def test_runtime_adapter_validator_rejects_invalid_protocol_attributes(
|
||||
attribute: str,
|
||||
invalid_value: object,
|
||||
) -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
setattr(adapter, attribute, invalid_value)
|
||||
|
||||
with pytest.raises(TypeError, match=attribute):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
|
||||
|
||||
def test_runtime_adapter_validator_rejects_incomplete_data_plane() -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
adapter.data_plane.close_session = None
|
||||
|
||||
with pytest.raises(TypeError, match="data_plane.*close_session"):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Regression tests for duplex ``runtime_control`` payload redaction.
|
||||
|
||||
The redacted result is emitted on the WebSocket (for example
|
||||
``session.created.runtime_control``), so it must always survive
|
||||
``json.dumps``. Control results carry typed values straight from the engine --
|
||||
``fence`` and ``accepted_fence`` are ``DuplexFence`` instances added by
|
||||
``DuplexControlClient.execute`` -- which previously reached
|
||||
``WebSocket.send_json`` unconverted and killed the session at handshake.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.openai.runtime_bridge import NativeRuntimeBridgeMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_redact = NativeRuntimeBridgeMixin._redact_runtime_control_result
|
||||
|
||||
|
||||
def _control_result() -> dict[str, object]:
|
||||
"""The shape ``DuplexControlClient.execute`` returns for an open."""
|
||||
return {
|
||||
"ok": True,
|
||||
"operation": "open",
|
||||
"session_id": "sess-1",
|
||||
"fence": DuplexFence("sess-1", epoch=0, turn_id=0, response_seq=0, incarnation=0),
|
||||
"accepted_fence": DuplexFence("sess-1", epoch=1, turn_id=2, response_seq=3, incarnation=4),
|
||||
"stage_results": [{"stage_id": -1, "replica_id": -1, "result": {"supported": True}}],
|
||||
}
|
||||
|
||||
|
||||
def test_redacted_control_result_is_json_serializable() -> None:
|
||||
"""The whole payload must serialize the way send_json does."""
|
||||
redacted = _redact(_control_result())
|
||||
# Must not raise.
|
||||
json.dumps({"type": "session.created", "runtime_control": redacted})
|
||||
|
||||
|
||||
def test_fences_are_converted_to_plain_dicts() -> None:
|
||||
redacted = _redact(_control_result())
|
||||
assert redacted["fence"] == {
|
||||
"session_id": "sess-1",
|
||||
"epoch": 0,
|
||||
"turn_id": 0,
|
||||
"response_seq": 0,
|
||||
"incarnation": 0,
|
||||
}
|
||||
# Field values are preserved, not just the type.
|
||||
assert redacted["accepted_fence"]["epoch"] == 1
|
||||
assert redacted["accepted_fence"]["turn_id"] == 2
|
||||
assert redacted["accepted_fence"]["response_seq"] == 3
|
||||
assert redacted["accepted_fence"]["incarnation"] == 4
|
||||
|
||||
|
||||
def test_nested_dataclasses_are_converted() -> None:
|
||||
"""Fences can appear inside stage results, not only at the top level."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"stage_results": [
|
||||
{"stage_id": 0, "result": {"fence": DuplexFence("sess-2", epoch=7)}},
|
||||
]
|
||||
}
|
||||
)
|
||||
json.dumps(redacted)
|
||||
assert redacted["stage_results"][0]["result"]["fence"]["epoch"] == 7
|
||||
|
||||
|
||||
def test_enums_are_reduced_to_values() -> None:
|
||||
class _Mode(str, Enum):
|
||||
APPEND = "append_audio_chunk"
|
||||
|
||||
redacted = _redact({"mode": _Mode.APPEND})
|
||||
json.dumps(redacted)
|
||||
assert redacted["mode"] == "append_audio_chunk"
|
||||
|
||||
|
||||
def test_redaction_still_drops_heavy_payload_keys() -> None:
|
||||
"""The pre-existing contract must be preserved."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"ok": True,
|
||||
"stage_handoff": object(),
|
||||
"tts_handoff": object(),
|
||||
"omni_payload": object(),
|
||||
"tts_hidden_states": object(),
|
||||
"tts_token_ids": object(),
|
||||
"traceback": "long trace",
|
||||
}
|
||||
)
|
||||
assert redacted == {"ok": True}
|
||||
|
||||
|
||||
def test_plain_values_pass_through_unchanged() -> None:
|
||||
payload = {"ok": True, "count": 3, "name": "x", "missing": None, "items": [1, "two"]}
|
||||
assert _redact(payload) == payload
|
||||
|
||||
|
||||
def test_dataclass_types_are_not_treated_as_instances() -> None:
|
||||
"""A class object is not a dataclass instance and must not be asdict'd."""
|
||||
|
||||
@dataclass
|
||||
class _Shape:
|
||||
x: int = 1
|
||||
|
||||
assert _redact(_Shape) is _Shape
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.openai.websocket import (
|
||||
DuplexWebSocketActor,
|
||||
normalize_duplex_input_event,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self) -> None:
|
||||
self.sent: list[dict[str, object]] = []
|
||||
|
||||
async def send_json(self, payload: dict[str, object]) -> None:
|
||||
self.sent.append(dict(payload))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event", "expected"),
|
||||
[
|
||||
({"type": "signal_turn", "event": "barge_in"}, {"type": "turn.signal", "event": "barge_in"}),
|
||||
({"type": "close_session"}, {"type": "session.close"}),
|
||||
({"type": "audio.playback_ack", "played_ms": 1}, {"type": "playback.ack", "played_ms": 1}),
|
||||
({"type": "input_text.append", "text": "a"}, {"type": "input.text.append", "text": "a"}),
|
||||
({"type": "push_text", "text": "b"}, {"type": "input.text.append", "text": "b"}),
|
||||
(
|
||||
{"type": "input.audio.append", "audio": "wav"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "wav"},
|
||||
),
|
||||
(
|
||||
{"type": "push_chunk", "audio": "wav", "format": "pcm_f32le"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "pcm_f32le"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_input_aliases_normalize_once_at_mailbox_boundary(event, expected):
|
||||
assert normalize_duplex_input_event(event) == expected
|
||||
|
||||
|
||||
def test_actor_uses_one_fifo_mailbox_for_inbound_events():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
|
||||
assert isinstance(actor.mailbox, asyncio.Queue)
|
||||
assert not isinstance(actor.mailbox, asyncio.PriorityQueue)
|
||||
assert not hasattr(actor, "control_queue")
|
||||
assert not hasattr(actor, "input_queue")
|
||||
assert not hasattr(actor, "event_queue")
|
||||
assert not hasattr(actor, "lifecycle_state")
|
||||
assert not hasattr(actor, "last_response_id")
|
||||
assert not hasattr(actor, "overlap_speech_ms")
|
||||
assert not hasattr(actor, "transition")
|
||||
assert not hasattr(actor, "output_generation_in_flight")
|
||||
assert not hasattr(actor, "session")
|
||||
assert not hasattr(actor, "runtime_opened")
|
||||
assert not hasattr(actor, "runtime_closed")
|
||||
assert not hasattr(actor, "drain_input_queue")
|
||||
assert not hasattr(actor, "control_events_seen")
|
||||
assert not hasattr(actor, "input_events_seen")
|
||||
assert not hasattr(actor, "cancel_count")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_event_does_not_overtake_earlier_audio_input():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": "response.cancel"})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": "response.cancel"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("event_type", ["input.cancel", "session.close", "close_session"])
|
||||
async def test_terminal_control_preserves_wire_order(event_type: str):
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": event_type})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": event_type}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mailbox_delivers_every_enqueued_event_exactly_once():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
events = [
|
||||
{"type": "input_audio_buffer.append", "audio": str(index)}
|
||||
if index % 2
|
||||
else {"type": "response.created", "response_id": f"resp-{index}"}
|
||||
for index in range(20)
|
||||
]
|
||||
|
||||
await asyncio.gather(*(actor.enqueue_event(dict(event)) for event in events))
|
||||
received = [await actor.next_event() for _ in events]
|
||||
|
||||
assert sorted(received, key=repr) == sorted(events, key=repr)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_is_single_owner_of_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "one"})
|
||||
await actor.send_json({"type": "two"})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "one"}, {"type": "two"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_fence_payload_is_dropped_before_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 2)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "response.audio.delta", "epoch": 1})
|
||||
await actor.send_json({"type": "response.audio.delta", "epoch": 2})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "response.audio.delta", "epoch": 2}]
|
||||
assert actor.stale_output_dropped == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_does_not_revoke_accepted_terminal_after_close_starts():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 1)
|
||||
await actor.send_json({"type": "response.done", "epoch": 1, "response_id": "resp-1"})
|
||||
actor.closing = True
|
||||
await actor.close_writer()
|
||||
|
||||
await actor.writer_loop()
|
||||
|
||||
assert websocket.sent == [{"type": "response.done", "epoch": 1, "response_id": "resp-1"}]
|
||||
assert actor.stale_output_dropped == 0
|
||||
@@ -1,30 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def test_batched_runtime_import_does_not_require_optional_opus_dependency() -> None:
|
||||
script = """
|
||||
import builtins
|
||||
|
||||
real_import = builtins.__import__
|
||||
|
||||
def guarded_import(name, *args, **kwargs):
|
||||
if name == "sphn":
|
||||
raise ModuleNotFoundError("blocked optional dependency: sphn")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
builtins.__import__ = guarded_import
|
||||
from vllm_omni.experimental.fullduplex.personaplex.serving.batched import BatchedSessionManager
|
||||
assert BatchedSessionManager is not None
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
@@ -1,303 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import base64
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.experimental.fullduplex.personaplex.policy import (
|
||||
SILENCE_TOKENS,
|
||||
SINE_TOKENS,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.stage0 import (
|
||||
PersonaPlexStage0DuplexRuntime,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_talker import (
|
||||
PersonaPlexTalkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _FakeCodec:
|
||||
def __init__(self) -> None:
|
||||
self.encode_calls = 0
|
||||
self.reset_calls = 0
|
||||
|
||||
def streaming_init(self, batch_size: int) -> None:
|
||||
assert batch_size == 1
|
||||
|
||||
def encode_frame(self, pcm: torch.Tensor) -> torch.Tensor:
|
||||
assert pcm.shape == (1, 1920)
|
||||
self.encode_calls += 1
|
||||
return torch.full((1, 8), self.encode_calls, dtype=torch.long)
|
||||
|
||||
def reset_streaming(self) -> None:
|
||||
self.reset_calls += 1
|
||||
|
||||
|
||||
class _FakeTalker:
|
||||
def __init__(self) -> None:
|
||||
self.device = torch.device("cpu")
|
||||
self.dtype = torch.float32
|
||||
self.frame_calls: list[dict[str, torch.Tensor | None]] = []
|
||||
|
||||
def _build_prefill_embed(
|
||||
self,
|
||||
tokens,
|
||||
offset,
|
||||
span,
|
||||
device,
|
||||
silence=None,
|
||||
user_sine=None,
|
||||
):
|
||||
del offset, silence, user_sine
|
||||
values = tokens[:span].to(device=device, dtype=torch.float32)
|
||||
return values[:, None].expand(-1, 4).contiguous()
|
||||
|
||||
def _build_frame_embed(
|
||||
self,
|
||||
text_token,
|
||||
last_agent,
|
||||
prev_agent,
|
||||
device,
|
||||
user_d0=None,
|
||||
user_d1=None,
|
||||
):
|
||||
self.frame_calls.append(
|
||||
{
|
||||
"text_token": text_token.clone(),
|
||||
"last_agent": None if last_agent is None else last_agent.clone(),
|
||||
"prev_agent": None if prev_agent is None else prev_agent.clone(),
|
||||
"user_d0": None if user_d0 is None else user_d0.clone(),
|
||||
"user_d1": None if user_d1 is None else user_d1.clone(),
|
||||
}
|
||||
)
|
||||
value = float(user_d0[0].item()) if user_d0 is not None else -1.0
|
||||
return torch.full((1, 4), value, device=device)
|
||||
|
||||
|
||||
def _duplex_info(*, seq: int, session_id: str = "session", incarnation: int = 1):
|
||||
pcm = np.zeros(1920, dtype="<f4")
|
||||
return {
|
||||
"data_plane": True,
|
||||
"session_id": session_id,
|
||||
"incarnation": incarnation,
|
||||
"epoch": 0,
|
||||
"seq": seq,
|
||||
"payload": {
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 24000,
|
||||
"audio": base64.b64encode(pcm.tobytes()).decode("ascii"),
|
||||
},
|
||||
"runtime_config": {
|
||||
"personaplex_model_path": "/unused",
|
||||
"personaplex_voice_prompt": "NATF2.pt",
|
||||
"personaplex_persona": "Be concise.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _runtime(*codecs: _FakeCodec) -> PersonaPlexStage0DuplexRuntime:
|
||||
voice_embeddings = torch.arange(8, dtype=torch.float32).reshape(2, 1, 1, 4)
|
||||
codec_args = {"codec": codecs[0]}
|
||||
if len(codecs) > 1:
|
||||
available = iter(codecs)
|
||||
codec_args = {
|
||||
"codec_factory": lambda: next(available),
|
||||
"max_sessions": len(codecs),
|
||||
}
|
||||
return PersonaPlexStage0DuplexRuntime(
|
||||
_FakeTalker(),
|
||||
model_path="/unused",
|
||||
device="cpu",
|
||||
**codec_args,
|
||||
tokenizer=lambda _text: [7, 8, 9],
|
||||
voice_loader=lambda _voice: {
|
||||
"embeddings": voice_embeddings,
|
||||
"cache": torch.zeros((1, 17, 4), dtype=torch.long),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_first_append_prepends_voice_and_persona_once() -> None:
|
||||
codec = _FakeCodec()
|
||||
runtime = _runtime(codec)
|
||||
|
||||
first = runtime.prepare_append(_duplex_info(seq=1), prompt_len=18)
|
||||
second = runtime.prepare_append(_duplex_info(seq=2), prompt_len=18)
|
||||
|
||||
assert first.prefill_applied is True
|
||||
assert first.prompt_offset == 0
|
||||
assert first.user_codes.shape == (1, 8)
|
||||
assert first.inputs_embeds.shape == (18, 4)
|
||||
assert second.prefill_applied is False
|
||||
assert second.prompt_offset == 17
|
||||
assert second.user_codes.shape == (2, 8)
|
||||
assert second.inputs_embeds.shape == (1, 4)
|
||||
assert codec.encode_calls == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("append_count", "expected_tokens", "expected_provided"),
|
||||
[
|
||||
(
|
||||
1,
|
||||
[*SILENCE_TOKENS, 1, *SINE_TOKENS[1:]],
|
||||
[False, *([True] * 15)],
|
||||
),
|
||||
(
|
||||
2,
|
||||
[*SILENCE_TOKENS, 2, *([1] * 7)],
|
||||
[*([False] * 8), *([True] * 8)],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_prepare_append_preserves_native_depformer_teacher_forcing(
|
||||
append_count: int,
|
||||
expected_tokens: list[int],
|
||||
expected_provided: list[bool],
|
||||
) -> None:
|
||||
runtime = _runtime(_FakeCodec())
|
||||
|
||||
result = None
|
||||
for seq in range(1, append_count + 1):
|
||||
result = runtime.prepare_append(_duplex_info(seq=seq), prompt_len=18)
|
||||
|
||||
assert result is not None
|
||||
assert result.info_update["pplex_depformer_audio_tokens"].tolist() == expected_tokens
|
||||
assert result.info_update["pplex_depformer_audio_provided"].tolist() == expected_provided
|
||||
|
||||
|
||||
def test_frame_embed_uses_previous_effective_agent_frame() -> None:
|
||||
fake_talker = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
num_audio_codebooks=16,
|
||||
audio_vocab_size=2048,
|
||||
),
|
||||
input_embeddings=lambda stack: stack,
|
||||
)
|
||||
last_agent = torch.arange(10, 18)
|
||||
|
||||
embeds = PersonaPlexTalkerForConditionalGeneration._build_frame_embed(
|
||||
fake_talker,
|
||||
torch.tensor([3]),
|
||||
last_agent,
|
||||
torch.arange(8),
|
||||
torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert embeds.shape == (1, 17)
|
||||
assert torch.equal(embeds[0, 1:9], last_agent)
|
||||
|
||||
|
||||
def test_repeated_append_identity_does_not_advance_codec() -> None:
|
||||
codec = _FakeCodec()
|
||||
runtime = _runtime(codec)
|
||||
|
||||
info = _duplex_info(seq=1)
|
||||
first = runtime.prepare_append(info, prompt_len=18, request_id="req")
|
||||
retry = runtime.prepare_append(info, prompt_len=18, request_id="req")
|
||||
|
||||
assert codec.encode_calls == 1
|
||||
assert torch.equal(first.inputs_embeds, retry.inputs_embeds)
|
||||
assert torch.equal(first.user_codes, retry.user_codes)
|
||||
|
||||
|
||||
def test_next_append_uses_prior_sample_and_causally_delayed_user_frame() -> None:
|
||||
runtime = _runtime(_FakeCodec())
|
||||
|
||||
runtime.prepare_append(_duplex_info(seq=1), prompt_len=18, request_id="req")
|
||||
first_agent = torch.arange(8, dtype=torch.long)
|
||||
runtime.record_sample(request_id="req", text_token=torch.tensor(101), agent_codes=first_agent)
|
||||
runtime.prepare_append(_duplex_info(seq=2), prompt_len=19, request_id="req")
|
||||
second_agent = torch.arange(10, 18, dtype=torch.long)
|
||||
runtime.record_sample(request_id="req", text_token=torch.tensor(102), agent_codes=second_agent)
|
||||
runtime.prepare_append(_duplex_info(seq=3), prompt_len=20, request_id="req")
|
||||
|
||||
silence = torch.tensor(SILENCE_TOKENS, dtype=torch.long)
|
||||
sine = torch.tensor(SINE_TOKENS, dtype=torch.long)
|
||||
first_call = runtime.stage_model.frame_calls[0]
|
||||
assert all(torch.equal(first_call[key], silence) for key in ("last_agent", "prev_agent"))
|
||||
assert all(torch.equal(first_call[key], sine) for key in ("user_d0", "user_d1"))
|
||||
|
||||
second_call = runtime.stage_model.frame_calls[1]
|
||||
assert second_call["text_token"].tolist() == [101]
|
||||
expected_first_effective = torch.cat([first_agent[:1], silence[1:]])
|
||||
assert torch.equal(second_call["last_agent"], expected_first_effective)
|
||||
assert torch.equal(second_call["prev_agent"], expected_first_effective)
|
||||
assert torch.equal(second_call["user_d0"], torch.full((8,), 1, dtype=torch.long))
|
||||
assert torch.equal(second_call["user_d1"], sine)
|
||||
|
||||
third_call = runtime.stage_model.frame_calls[2]
|
||||
assert third_call["text_token"].tolist() == [102]
|
||||
assert torch.equal(third_call["last_agent"], second_agent)
|
||||
assert torch.equal(third_call["prev_agent"], second_agent)
|
||||
assert torch.equal(third_call["user_d0"], torch.full((8,), 2, dtype=torch.long))
|
||||
assert torch.equal(third_call["user_d1"], torch.full((8,), 1, dtype=torch.long))
|
||||
|
||||
|
||||
def test_decoded_pcm_is_writable_for_torch_zero_copy() -> None:
|
||||
pcm = PersonaPlexStage0DuplexRuntime._decode_pcm(_duplex_info(seq=1)["payload"])
|
||||
|
||||
assert pcm.flags.writeable
|
||||
|
||||
|
||||
def _prepare_two_sessions(
|
||||
runtime: PersonaPlexStage0DuplexRuntime,
|
||||
) -> tuple[object, object, object, object]:
|
||||
first_1 = runtime.prepare_append(_duplex_info(seq=1), prompt_len=18)
|
||||
second_1 = runtime.prepare_append(
|
||||
_duplex_info(seq=1, session_id="other", incarnation=2),
|
||||
prompt_len=18,
|
||||
)
|
||||
first_2 = runtime.prepare_append(_duplex_info(seq=2), prompt_len=19)
|
||||
second_2 = runtime.prepare_append(
|
||||
_duplex_info(seq=2, session_id="other", incarnation=2),
|
||||
prompt_len=19,
|
||||
)
|
||||
return first_1, second_1, first_2, second_2
|
||||
|
||||
|
||||
def test_live_sessions_keep_independent_streaming_encoders() -> None:
|
||||
first_codec = _FakeCodec()
|
||||
second_codec = _FakeCodec()
|
||||
runtime = _runtime(first_codec, second_codec)
|
||||
|
||||
first_1, second_1, first_2, second_2 = _prepare_two_sessions(runtime)
|
||||
|
||||
assert first_1.user_codes[:, 0].tolist() == [1]
|
||||
assert second_1.user_codes[:, 0].tolist() == [1]
|
||||
assert first_2.user_codes[:, 0].tolist() == [1, 2]
|
||||
assert second_2.user_codes[:, 0].tolist() == [1, 2]
|
||||
|
||||
|
||||
def test_stage0_session_capacity_fails_before_codec_state_is_shared() -> None:
|
||||
runtime = _runtime(_FakeCodec(), _FakeCodec())
|
||||
_prepare_two_sessions(runtime)
|
||||
|
||||
with pytest.raises(RuntimeError, match="capacity 2"):
|
||||
runtime.prepare_append(
|
||||
_duplex_info(seq=1, session_id="third", incarnation=3),
|
||||
prompt_len=18,
|
||||
)
|
||||
|
||||
|
||||
def test_close_session_resets_and_reuses_released_codec() -> None:
|
||||
first_codec = _FakeCodec()
|
||||
second_codec = _FakeCodec()
|
||||
runtime = _runtime(first_codec, second_codec)
|
||||
_prepare_two_sessions(runtime)
|
||||
|
||||
runtime.close_session("session", 1)
|
||||
|
||||
assert first_codec.reset_calls == 1
|
||||
assert second_codec.reset_calls == 0
|
||||
replacement = runtime.prepare_append(
|
||||
_duplex_info(seq=1, session_id="replacement", incarnation=4),
|
||||
prompt_len=18,
|
||||
)
|
||||
assert replacement.user_codes[:, 0].tolist() == [3]
|
||||
@@ -1,316 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex import (
|
||||
personaplex_code2wav,
|
||||
personaplex_mimi,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_code2wav import (
|
||||
PersonaPlexCode2Wav,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _FakeStreamingMimi(nn.Module):
|
||||
def __init__(self, stream_value: int = 0) -> None:
|
||||
super().__init__()
|
||||
self.stream_value = stream_value
|
||||
self.decode_frame_calls = 0
|
||||
self.reset_calls = 0
|
||||
|
||||
def decode(self, codes: torch.Tensor, return_dict: bool = True):
|
||||
assert return_dict
|
||||
frames = int(codes.shape[-1])
|
||||
return SimpleNamespace(audio_values=torch.arange(frames * 4, dtype=torch.float32).reshape(1, 1, -1))
|
||||
|
||||
def streaming_init(self, batch_size: int) -> None:
|
||||
assert batch_size == 1
|
||||
|
||||
def decode_frame(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
assert codes.shape == (1, 2)
|
||||
self.decode_frame_calls += 1
|
||||
value = self.stream_value + self.decode_frame_calls
|
||||
return torch.full((1, 4), float(value), dtype=torch.float32)
|
||||
|
||||
def reset_streaming(self) -> None:
|
||||
self.reset_calls += 1
|
||||
|
||||
|
||||
class _FakeChunkedMimi(_FakeStreamingMimi):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.decode_frames_calls = 0
|
||||
self.decode_frames_shapes: list[tuple[int, ...]] = []
|
||||
|
||||
def decode_frames(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
self.decode_frames_calls += 1
|
||||
self.decode_frames_shapes.append(tuple(codes.shape))
|
||||
return torch.arange(codes.shape[-1] * 4, dtype=torch.float32).reshape(1, -1)
|
||||
|
||||
|
||||
class _FakeMimiModel(nn.Module):
|
||||
def __init__(self, _config) -> None:
|
||||
super().__init__()
|
||||
self.encoder_transformer = nn.Linear(1, 1, bias=False)
|
||||
self.decoder_transformer = nn.Linear(1, 1, bias=False)
|
||||
self.quantizer = nn.Linear(1, 1, bias=False)
|
||||
self.encoder = SimpleNamespace(layers=[])
|
||||
self.downsample = SimpleNamespace(conv=nn.Conv1d(1, 1, 1))
|
||||
self.upsample = SimpleNamespace(conv=nn.ConvTranspose1d(1, 1, 1))
|
||||
self.decoder = SimpleNamespace(layers=[])
|
||||
|
||||
def load_state_dict(self, _state_dict, strict: bool = True):
|
||||
assert not strict
|
||||
return SimpleNamespace(
|
||||
missing_keys=[
|
||||
"encoder_transformer.weight",
|
||||
"decoder_transformer.weight",
|
||||
],
|
||||
unexpected_keys=[],
|
||||
)
|
||||
|
||||
|
||||
class _FakeMimiTransformer(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(1))
|
||||
|
||||
def load_weights(self, _state_dict, _prefix: str) -> int:
|
||||
return 80
|
||||
|
||||
|
||||
def test_moshi_checkpoint_mapping_drops_replaced_transformer_weights(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mapper = getattr(personaplex_mimi, "_map_moshi_codec_weights", None)
|
||||
assert callable(mapper), "PersonaPlex Mimi must load its bundled codec weights without Hugging Face"
|
||||
|
||||
source = {
|
||||
"encoder.model.3.conv.conv.weight": torch.ones(1),
|
||||
"downsample.conv.conv.conv.weight": torch.ones(1),
|
||||
"quantizer.rvq_first.vq.layers.0._codebook.embedding_sum": torch.ones(1),
|
||||
"quantizer.rvq_rest.vq.layers.3._codebook._initialized": torch.ones(1),
|
||||
"encoder_transformer.transformer.layers.0.norm1.weight": torch.ones(1),
|
||||
}
|
||||
|
||||
mapped = mapper(source)
|
||||
|
||||
assert set(mapped) == {
|
||||
"encoder.layers.3.conv.weight",
|
||||
"downsample.conv.weight",
|
||||
"quantizer.semantic_residual_vector_quantizer.layers.0.codebook.embed_sum",
|
||||
"quantizer.acoustic_residual_vector_quantizer.layers.3.codebook.initialized",
|
||||
}
|
||||
assert mapped["encoder.layers.3.conv.weight"] is source["encoder.model.3.conv.conv.weight"]
|
||||
monkeypatch.setattr("transformers.MimiConfig", lambda: object())
|
||||
monkeypatch.setattr("transformers.MimiModel", _FakeMimiModel)
|
||||
monkeypatch.setattr("safetensors.torch.load_file", lambda *_args, **_kwargs: {})
|
||||
monkeypatch.setattr(personaplex_mimi, "_MimiStreamingTransformer", _FakeMimiTransformer)
|
||||
|
||||
codec = personaplex_mimi.PersonaPlexMimiCodec(
|
||||
checkpoint="/unused",
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
assert not hasattr(codec.model, "encoder_transformer")
|
||||
assert not hasattr(codec.model, "decoder_transformer")
|
||||
assert not any(
|
||||
name.startswith(("model.encoder_transformer.", "model.decoder_transformer."))
|
||||
for name, _ in codec.named_parameters()
|
||||
)
|
||||
|
||||
|
||||
def test_mimi_full_stream_reset_reuses_all_state_storage_and_clears_offsets() -> None:
|
||||
codec = personaplex_mimi.PersonaPlexMimiCodec.__new__(personaplex_mimi.PersonaPlexMimiCodec)
|
||||
nn.Module.__init__(codec)
|
||||
codec.device = torch.device("cpu")
|
||||
codec.dtype = torch.float32
|
||||
codec._enc_stages = [("conv", personaplex_mimi._StreamConv1d(nn.Conv1d(1, 1, 3)))]
|
||||
codec._dec_stages = []
|
||||
codec._downsample = personaplex_mimi._StreamConv1d(nn.Conv1d(1, 1, 3))
|
||||
codec._upsample = personaplex_mimi._StreamConvTr1d(nn.ConvTranspose1d(1, 1, 3))
|
||||
codec.encoder_transformer = personaplex_mimi._MimiStreamingTransformer(
|
||||
num_layers=1,
|
||||
dim=4,
|
||||
num_heads=1,
|
||||
context=3,
|
||||
)
|
||||
codec.decoder_transformer = personaplex_mimi._MimiStreamingTransformer(
|
||||
num_layers=1,
|
||||
dim=4,
|
||||
num_heads=1,
|
||||
context=3,
|
||||
)
|
||||
codec.streaming_init(batch_size=2)
|
||||
|
||||
conv_states = list(codec._conv_states())
|
||||
conv_buffers = [
|
||||
state.prev if isinstance(state, personaplex_mimi._StreamConv1d) else state.partial for state in conv_states
|
||||
]
|
||||
transformers = [codec.encoder_transformer, codec.decoder_transformer]
|
||||
kv_states = [kv for transformer in transformers for kv in transformer._kv]
|
||||
state_tensors = [
|
||||
*conv_buffers,
|
||||
*(kv.cache for kv in kv_states),
|
||||
*(kv.end_offset for kv in kv_states),
|
||||
*(kv.start_offset for kv in kv_states),
|
||||
*(transformer._offset for transformer in transformers),
|
||||
]
|
||||
for tensor in state_tensors:
|
||||
tensor.fill_(1)
|
||||
for state in conv_states:
|
||||
state._fresh.zero_()
|
||||
storage = [tensor.data_ptr() for tensor in state_tensors]
|
||||
|
||||
codec.reset_streaming()
|
||||
|
||||
assert storage == [tensor.data_ptr() for tensor in state_tensors]
|
||||
cleared = [
|
||||
*conv_buffers,
|
||||
*(kv.end_offset for kv in kv_states),
|
||||
*(kv.start_offset for kv in kv_states),
|
||||
*(transformer._offset for transformer in transformers),
|
||||
]
|
||||
assert all(not tensor.any() for tensor in cleared)
|
||||
assert all(state._fresh.all() for state in conv_states)
|
||||
|
||||
|
||||
def _model(*, max_sessions: int = 1) -> tuple[PersonaPlexCode2Wav, _FakeStreamingMimi]:
|
||||
mimi_config = SimpleNamespace(num_codebooks=2, sample_rate=24000, samples_per_frame=4, mimi_name=None)
|
||||
config = SimpleNamespace(mimi_config=mimi_config, mimi_name=None)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
model="/unused",
|
||||
hf_config=config,
|
||||
duplex_max_sessions=max_sessions,
|
||||
),
|
||||
device_config=SimpleNamespace(device="cpu"),
|
||||
)
|
||||
model = PersonaPlexCode2Wav(vllm_config=vllm_config)
|
||||
mimi = _FakeStreamingMimi()
|
||||
model.mimi = mimi
|
||||
model._mimi_device = torch.device("cpu")
|
||||
return model, mimi
|
||||
|
||||
|
||||
def _codes(frames: int, *, start: int = 0) -> torch.Tensor:
|
||||
return torch.stack([torch.arange(start + offset, start + offset + frames) for offset in (0, 100)]).reshape(-1)
|
||||
|
||||
|
||||
def _audio(output) -> torch.Tensor:
|
||||
return output.multimodal_outputs["model_outputs"][0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("second_codes", [_codes(3), _codes(1, start=100)])
|
||||
def test_resumable_codes_emit_only_new_pcm(second_codes: torch.Tensor) -> None:
|
||||
model, mimi = _model()
|
||||
|
||||
first = model(input_ids=_codes(2), request_ids=["req"])
|
||||
second = model(input_ids=second_codes, request_ids=["req"])
|
||||
|
||||
assert _audio(first).numel() == 8
|
||||
assert _audio(second).numel() == 4
|
||||
assert mimi.decode_frame_calls == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("frames", "expected_shapes"),
|
||||
[
|
||||
(5, [(1, 2, 5)]),
|
||||
(12, [(1, 2, 5), (1, 2, 5), (1, 2, 2)]),
|
||||
],
|
||||
)
|
||||
def test_streaming_decode_uses_bounded_multi_frame_calls(
|
||||
frames: int,
|
||||
expected_shapes: list[tuple[int, ...]],
|
||||
) -> None:
|
||||
model, _ = _model()
|
||||
mimi = _FakeChunkedMimi()
|
||||
model.mimi = mimi
|
||||
|
||||
output = model(input_ids=_codes(frames), request_ids=["req"])
|
||||
|
||||
assert _audio(output).numel() == frames * 4
|
||||
assert mimi.decode_frames_shapes == expected_shapes
|
||||
assert mimi.decode_frame_calls == 0
|
||||
|
||||
|
||||
def test_profile_inputs_skip_decode_while_malformed_online_input_warns(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
model, mimi = _model()
|
||||
runtime_info = [{"meta": {"personaplex_dummy_profile": True}}]
|
||||
warnings: list[tuple] = []
|
||||
monkeypatch.setattr(personaplex_code2wav.logger, "warning", lambda *args: warnings.append(args))
|
||||
|
||||
output = model(
|
||||
input_ids=torch.arange(3),
|
||||
runtime_additional_information=runtime_info,
|
||||
)
|
||||
|
||||
assert _audio(output).numel() == 0
|
||||
assert mimi.decode_frame_calls == 0
|
||||
assert warnings == []
|
||||
|
||||
output = model(input_ids=torch.arange(3))
|
||||
|
||||
assert _audio(output).numel() == 0
|
||||
assert mimi.decode_frame_calls == 0
|
||||
assert len(warnings) == 1
|
||||
assert "not divisible by" in warnings[0][0]
|
||||
assert model.get_dummy_runtime_additional_information(2) == [
|
||||
{"meta": {"personaplex_dummy_profile": True}},
|
||||
{"meta": {"personaplex_dummy_profile": True}},
|
||||
]
|
||||
|
||||
|
||||
def test_request_id_falls_back_to_runtime_information() -> None:
|
||||
model, _ = _model()
|
||||
info = [{"request_id": "runtime-req"}]
|
||||
|
||||
model(input_ids=_codes(2), runtime_additional_information=info)
|
||||
second = model(input_ids=_codes(3), runtime_additional_information=info)
|
||||
|
||||
assert _audio(second).numel() == 4
|
||||
|
||||
|
||||
def test_decoder_slot_lifecycle_isolated_capacity_and_reuse() -> None:
|
||||
model, _ = _model(max_sessions=2)
|
||||
first = _FakeStreamingMimi(stream_value=10)
|
||||
second = _FakeStreamingMimi(stream_value=20)
|
||||
model._set_mimi_codecs([first, second])
|
||||
assert model._max_codec_sessions == 2
|
||||
|
||||
initial = model(
|
||||
input_ids=torch.cat([_codes(1), _codes(1)]),
|
||||
request_ids=["first", "second"],
|
||||
seq_token_counts=[2, 2],
|
||||
)
|
||||
continued = model(
|
||||
input_ids=torch.cat([_codes(1, start=10), _codes(1, start=20)]),
|
||||
request_ids=["first", "second"],
|
||||
seq_token_counts=[2, 2],
|
||||
)
|
||||
|
||||
initial_audio = initial.multimodal_outputs["model_outputs"]
|
||||
continued_audio = continued.multimodal_outputs["model_outputs"]
|
||||
assert initial_audio[0].tolist() == [11.0] * 4
|
||||
assert initial_audio[1].tolist() == [21.0] * 4
|
||||
assert continued_audio[0].tolist() == [12.0] * 4
|
||||
assert continued_audio[1].tolist() == [22.0] * 4
|
||||
assert model._request_codec_slots == {"first": 0, "second": 1}
|
||||
with pytest.raises(RuntimeError, match="decoder capacity 2 is exhausted"):
|
||||
model._codec_for_request("overflow")
|
||||
|
||||
model.on_requests_finished(["first"])
|
||||
assert first.reset_calls == 1
|
||||
assert second.reset_calls == 0
|
||||
replacement = model(input_ids=_codes(1), request_ids=["replacement"])
|
||||
assert _audio(replacement).tolist() == [13.0] * 4
|
||||
@@ -1,415 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.e2e.online_serving import personaplex_realtime_duplex as e2e_driver
|
||||
from vllm_omni.config.stage_config import (
|
||||
load_deploy_config,
|
||||
merge_pipeline_deploy,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.duplex_runtime import (
|
||||
DuplexInputMode,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.openai.runtime_adapter import (
|
||||
ServingRuntimeConfigError,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.config import DEFAULT_PERSONA
|
||||
from vllm_omni.experimental.fullduplex.personaplex.data_plane import (
|
||||
PersonaPlexDataPlaneContext,
|
||||
PersonaPlexDataPlaneSession,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.input import (
|
||||
PersonaPlexPcmAppendBuffer,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.runtime_extension import (
|
||||
PersonaPlexDuplexRuntimeExtension,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import (
|
||||
PersonaPlexServingRuntimeAdapter,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.pipeline import (
|
||||
PERSONAPLEX_PIPELINE,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _pcm_payload(samples: np.ndarray, *, sample_rate_hz: int = 24000) -> dict[str, object]:
|
||||
samples = np.ascontiguousarray(samples, dtype="<f4")
|
||||
return {
|
||||
"type": "audio",
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": sample_rate_hz,
|
||||
"audio": base64.b64encode(samples.tobytes()).decode("ascii"),
|
||||
}
|
||||
|
||||
|
||||
def _audio_client(
|
||||
frame_count: int = 10,
|
||||
*,
|
||||
voiced_frames: int = 0,
|
||||
) -> SimpleNamespace:
|
||||
frames = np.full((frame_count, e2e_driver.FRAME_SAMPLES), 7, dtype="<i2")
|
||||
frames[:voiced_frames] = 1000
|
||||
raw = frames.tobytes()
|
||||
event = {
|
||||
"type": "response.audio.delta",
|
||||
"response_id": "response-1",
|
||||
"sample_rate_hz": e2e_driver.SAMPLE_RATE_HZ,
|
||||
"metadata": {
|
||||
"vllm_omni": {
|
||||
"runtime_impl": "scheduler_data_plane",
|
||||
"uses_model_runner_scheduler": True,
|
||||
"runner_kv_backed": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
events = SimpleNamespace(
|
||||
events=[event],
|
||||
response_audio={"response-1": [raw]},
|
||||
audio_bytes=lambda: raw,
|
||||
)
|
||||
return SimpleNamespace(events=events)
|
||||
|
||||
|
||||
def test_realtime_audio_frame_stats_separate_voiced_and_silent_frames() -> None:
|
||||
silent = np.zeros(e2e_driver.FRAME_SAMPLES, dtype="<i2")
|
||||
voiced = np.full(e2e_driver.FRAME_SAMPLES, 1000, dtype="<i2")
|
||||
|
||||
stats = e2e_driver._audio_frame_stats(
|
||||
silent.tobytes() + voiced.tobytes(),
|
||||
input_frames=3,
|
||||
voiced_frame_rms_threshold=1e-4,
|
||||
)
|
||||
|
||||
assert stats == {
|
||||
"output_frames": 2,
|
||||
"voiced_frames": 1,
|
||||
"silent_frames": 1,
|
||||
"frame_deficit": 1,
|
||||
"frame_coverage_ratio": pytest.approx(2 / 3),
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_audio_frame_stats_reject_partial_codec_frame() -> None:
|
||||
partial_frame = np.zeros(e2e_driver.FRAME_SAMPLES - 1, dtype="<i2")
|
||||
|
||||
with pytest.raises(AssertionError, match="whole 80 ms codec frames"):
|
||||
e2e_driver._audio_frame_stats(
|
||||
partial_frame.tobytes(),
|
||||
input_frames=1,
|
||||
voiced_frame_rms_threshold=1e-4,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("frame_count", [10, 40])
|
||||
def test_e2e_driver_rejects_inaudible_sessions(frame_count: int) -> None:
|
||||
args = SimpleNamespace(
|
||||
max_frame_deficit=4,
|
||||
voiced_frame_rms_threshold=1e-3,
|
||||
min_voiced_frames=5,
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError, match="audible speech"):
|
||||
e2e_driver._session_result(
|
||||
_audio_client(frame_count),
|
||||
input_frames=frame_count,
|
||||
args=args,
|
||||
minimum_chunks=1,
|
||||
)
|
||||
|
||||
|
||||
def test_e2e_driver_uses_absolute_audible_floor() -> None:
|
||||
args = SimpleNamespace(
|
||||
max_frame_deficit=4,
|
||||
voiced_frame_rms_threshold=1e-3,
|
||||
min_voiced_frames=5,
|
||||
)
|
||||
|
||||
e2e_driver._session_result(
|
||||
_audio_client(200, voiced_frames=5),
|
||||
input_frames=200,
|
||||
args=args,
|
||||
minimum_chunks=1,
|
||||
)
|
||||
|
||||
|
||||
def test_e2e_driver_defaults_replacement_to_full_input() -> None:
|
||||
args = e2e_driver.parse_args(
|
||||
[
|
||||
"--model",
|
||||
"/model",
|
||||
"--input-wav",
|
||||
"/input.wav",
|
||||
]
|
||||
)
|
||||
|
||||
assert args.replacement_frames == 0
|
||||
assert args.min_voiced_frames == 5
|
||||
|
||||
|
||||
def test_e2e_driver_records_and_validates_input_sha256(tmp_path: Path) -> None:
|
||||
input_wav = tmp_path / "input.wav"
|
||||
input_wav.write_bytes(b"fixture-wav")
|
||||
|
||||
identity = e2e_driver._input_identity(
|
||||
input_wav,
|
||||
expected_sha256="bade4b3c163edde390ff391207d34a887257d8a9cc3b621cc8c618b6e6761304",
|
||||
)
|
||||
|
||||
assert identity == {
|
||||
"path": str(input_wav.resolve()),
|
||||
"sha256": "bade4b3c163edde390ff391207d34a887257d8a9cc3b621cc8c618b6e6761304",
|
||||
}
|
||||
|
||||
|
||||
def test_e2e_driver_rejects_unexpected_input_sha256(tmp_path: Path) -> None:
|
||||
input_wav = tmp_path / "input.wav"
|
||||
input_wav.write_bytes(b"fixture-wav")
|
||||
|
||||
with pytest.raises(ValueError, match="input WAV SHA-256 mismatch"):
|
||||
e2e_driver._input_identity(input_wav, expected_sha256="0" * 64)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "expected"),
|
||||
[
|
||||
(
|
||||
"duplex_runtime_extension",
|
||||
"vllm_omni.experimental.fullduplex.personaplex.runtime_extension.PersonaPlexDuplexRuntimeExtension",
|
||||
),
|
||||
(
|
||||
"duplex_serving_adapter",
|
||||
"vllm_omni.experimental.fullduplex.personaplex.serving_adapter.PersonaPlexServingRuntimeAdapter",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_personaplex_pipeline_registers_unified_duplex_components(
|
||||
attribute: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert PERSONAPLEX_PIPELINE.duplex_control_enabled is True
|
||||
assert getattr(PERSONAPLEX_PIPELINE, attribute) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("engine_arg", "expected"),
|
||||
[
|
||||
("skip_tokenizer_init", True),
|
||||
("enable_prefix_caching", False),
|
||||
],
|
||||
)
|
||||
def test_personaplex_stage_engine_args(
|
||||
engine_arg: str,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
deploy_path = Path(__file__).parents[5] / "vllm_omni" / "deploy" / "personaplex.yaml"
|
||||
stages = merge_pipeline_deploy(
|
||||
PERSONAPLEX_PIPELINE,
|
||||
load_deploy_config(deploy_path),
|
||||
)
|
||||
|
||||
assert [stage.yaml_engine_args.get(engine_arg) for stage in stages] == [
|
||||
expected,
|
||||
expected,
|
||||
]
|
||||
|
||||
|
||||
def test_personaplex_duplex_capacity_is_propagated_to_all_model_stages() -> None:
|
||||
deploy_path = Path(__file__).parents[5] / "vllm_omni" / "deploy" / "personaplex.yaml"
|
||||
deploy = load_deploy_config(deploy_path)
|
||||
stages = merge_pipeline_deploy(PERSONAPLEX_PIPELINE, deploy)
|
||||
|
||||
assert deploy.duplex_session.max_sessions == 2
|
||||
assert [stage.yaml_engine_args.get("duplex_max_sessions") for stage in stages] == [2, 2]
|
||||
assert "personaplex_codec_max_sessions" not in deploy.connectors["connector_of_shared_memory"]["extra"]
|
||||
|
||||
|
||||
def test_personaplex_capabilities_are_honest() -> None:
|
||||
single = PersonaPlexServingRuntimeAdapter(lambda *_: None).capabilities(max_sessions=1)
|
||||
multi = PersonaPlexServingRuntimeAdapter(lambda *_: None).capabilities(max_sessions=2)
|
||||
|
||||
assert multi.implementation_level == "model_native_duplex"
|
||||
assert multi.input_modes == ["append_audio_chunk"]
|
||||
assert multi.chunk_period_ms == 80
|
||||
assert single.supports_multi_session is False
|
||||
assert multi.supports_multi_session is True
|
||||
assert multi.supports_multi_session_same_replica is True
|
||||
assert multi.supports_barge_in is False
|
||||
|
||||
|
||||
def test_personaplex_runtime_config_update_rejects_changed_persona() -> None:
|
||||
adapter = PersonaPlexServingRuntimeAdapter(lambda *_: None)
|
||||
short_persona = "You are a helpful assistant."
|
||||
longer_persona = (
|
||||
"You are now a swashbuckling pirate captain who speaks entirely in "
|
||||
"nautical slang, references buried treasure constantly, and never "
|
||||
"breaks character no matter what the user asks."
|
||||
)
|
||||
current = {"personaplex_persona": short_persona, "personaplex_voice_prompt": "NATF2.pt"}
|
||||
|
||||
with pytest.raises(ServingRuntimeConfigError, match="persona"):
|
||||
adapter.runtime_config_for_update(
|
||||
SimpleNamespace(instructions=longer_persona, extra_body={}),
|
||||
current,
|
||||
)
|
||||
|
||||
unchanged = adapter.runtime_config_for_update(
|
||||
SimpleNamespace(instructions=short_persona, extra_body={}),
|
||||
current,
|
||||
)
|
||||
assert unchanged["personaplex_persona"] == short_persona
|
||||
|
||||
|
||||
def test_personaplex_runtime_config_update_rejects_changed_voice() -> None:
|
||||
adapter = PersonaPlexServingRuntimeAdapter(lambda *_: None)
|
||||
current = {"personaplex_persona": DEFAULT_PERSONA, "personaplex_voice_prompt": "NATF2.pt"}
|
||||
|
||||
with pytest.raises(ServingRuntimeConfigError, match="voice"):
|
||||
adapter.runtime_config_for_update(
|
||||
SimpleNamespace(instructions=None, voice="OtherVoice.pt", extra_body={}),
|
||||
current,
|
||||
)
|
||||
|
||||
unchanged = adapter.runtime_config_for_update(
|
||||
SimpleNamespace(instructions=None, voice="NATF2.pt", extra_body={}),
|
||||
current,
|
||||
)
|
||||
assert unchanged["personaplex_voice_prompt"] == "NATF2.pt"
|
||||
|
||||
|
||||
def test_personaplex_rejects_minicpmo_only_opt_out_flag() -> None:
|
||||
adapter = PersonaPlexServingRuntimeAdapter(lambda *_: None)
|
||||
|
||||
with pytest.raises(ServingRuntimeConfigError, match="minicpmo45_native_duplex"):
|
||||
adapter.validate_client_extra_body({"minicpmo45_native_duplex": False})
|
||||
|
||||
|
||||
def test_pcm_buffer_emits_one_80ms_frame_transactionally() -> None:
|
||||
buffer = PersonaPlexPcmAppendBuffer()
|
||||
reservation = buffer.prepare_append(
|
||||
_pcm_payload(np.arange(1920, dtype=np.float32)),
|
||||
operation_id="op-1",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=True,
|
||||
)
|
||||
|
||||
assert reservation is not None
|
||||
assert reservation.byte_count == 1920 * 4
|
||||
assert buffer.pending_byte_count == 0
|
||||
reservation.rollback()
|
||||
assert buffer.pending_byte_count == 1920 * 4
|
||||
|
||||
|
||||
def test_pcm_buffer_rejects_changed_sample_rate() -> None:
|
||||
buffer = PersonaPlexPcmAppendBuffer()
|
||||
assert (
|
||||
buffer.prepare_append(
|
||||
_pcm_payload(np.zeros(100, np.float32)),
|
||||
operation_id="op-1",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="sample_rate"):
|
||||
buffer.prepare_append(
|
||||
_pcm_payload(np.zeros(100, np.float32), sample_rate_hz=16000),
|
||||
operation_id="op-2",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=False,
|
||||
)
|
||||
|
||||
|
||||
def test_pcm_buffer_rejects_non_finite_samples() -> None:
|
||||
buffer = PersonaPlexPcmAppendBuffer()
|
||||
|
||||
with pytest.raises(ValueError, match="finite"):
|
||||
buffer.prepare_append(
|
||||
_pcm_payload(np.array([0.0, np.nan], np.float32)),
|
||||
operation_id="op-1",
|
||||
chunk_period_ms=80,
|
||||
allow_emit=False,
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_extension_builds_one_frame_scheduler_append() -> None:
|
||||
extension = PersonaPlexDuplexRuntimeExtension()
|
||||
defaults = (
|
||||
SamplingParams(temperature=0.8, top_k=10, max_tokens=10),
|
||||
SamplingParams(max_tokens=1024),
|
||||
)
|
||||
|
||||
configured = extension.configure_sampling_params(runtime_config={}, defaults=defaults)
|
||||
assert configured[0].temperature == 0.0
|
||||
assert configured[0].top_k == 1
|
||||
assert configured[0].max_tokens == 1
|
||||
assert configured[1] is defaults[1]
|
||||
|
||||
plan = extension.plan_append(
|
||||
request_id="req",
|
||||
fence=DuplexFence("session", incarnation=2, epoch=3),
|
||||
session_config={"instructions": "Be concise."},
|
||||
runtime_config={"personaplex_prefill_slots": 4},
|
||||
seq=1,
|
||||
turn_seq=1,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload=_pcm_payload(np.zeros(1920, np.float32)),
|
||||
final=False,
|
||||
sampling_params=configured[0],
|
||||
)
|
||||
assert len(plan.prompt["prompt_token_ids"]) == 5
|
||||
duplex = plan.prompt["model_intermediate_buffer"]["duplex"]
|
||||
assert duplex["session_id"] == "session"
|
||||
assert duplex["incarnation"] == 2
|
||||
assert duplex["epoch"] == 3
|
||||
assert duplex["seq"] == 1
|
||||
assert duplex["data_plane"] is True
|
||||
|
||||
|
||||
def test_projector_emits_only_cumulative_audio_and_text_suffixes() -> None:
|
||||
encoded_sizes: list[int] = []
|
||||
|
||||
def encode_audio(audio, _sample_rate, _response_format, _speed):
|
||||
size = int(np.asarray(audio, dtype=np.float32).size)
|
||||
encoded_sizes.append(size)
|
||||
return f"audio-{size}"
|
||||
|
||||
projector = PersonaPlexDataPlaneSession(encode_audio)
|
||||
projector.begin_request("req")
|
||||
context = PersonaPlexDataPlaneContext(response_format="wav")
|
||||
|
||||
first = SimpleNamespace(
|
||||
request_id="req",
|
||||
outputs=[SimpleNamespace(text="he", multimodal_output={})],
|
||||
multimodal_output={"model_outputs": np.arange(4, dtype=np.float32), "sr": 24000},
|
||||
finished=False,
|
||||
)
|
||||
second = SimpleNamespace(
|
||||
request_id="req",
|
||||
outputs=[SimpleNamespace(text="hello", multimodal_output={})],
|
||||
multimodal_output={"model_outputs": np.arange(6, dtype=np.float32), "sr": 24000},
|
||||
finished=False,
|
||||
)
|
||||
|
||||
projected = list(
|
||||
projector.project(
|
||||
{"data_plane_outputs": [first, second]},
|
||||
context=context,
|
||||
)
|
||||
)
|
||||
|
||||
assert encoded_sizes == [4, 2]
|
||||
assert [item["text"] for item in projected] == ["he", "llo"]
|
||||
assert [item["audio_data"] for item in projected] == ["audio-4", "audio-2"]
|
||||
assert all(item["sample_rate_hz"] == 24000 for item in projected)
|
||||
assert all(item["end_of_turn"] is False for item in projected)
|
||||
@@ -15,20 +15,20 @@ from vllm_omni.experimental.fullduplex.client import (
|
||||
read_pcm16_wav,
|
||||
write_pcm16_wav,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.minicpmo45.policy import (
|
||||
MiniCPMO45DuplexPolicy,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.video_stacking import (
|
||||
concat_frames,
|
||||
unit_subframe_offsets,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
|
||||
MiniCPMO45DuplexPolicy,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_realtime_client_builds_explicit_native_duplex_url():
|
||||
url = build_realtime_url(
|
||||
"ws://localhost:8099/v1/realtime?custom=1&duplex=0&model=stale&minicpmo45_native_duplex=0&session_id=stale",
|
||||
"ws://localhost:8099/v1/realtime?custom=1&duplex=0&model=stale&native_duplex=0&session_id=stale",
|
||||
"openbmb/MiniCPM-o-4_5",
|
||||
session_id="session-a",
|
||||
)
|
||||
@@ -38,7 +38,7 @@ def test_realtime_client_builds_explicit_native_duplex_url():
|
||||
"custom": ["1"],
|
||||
"duplex": ["1"],
|
||||
"model": ["openbmb/MiniCPM-o-4_5"],
|
||||
"minicpmo45_native_duplex": ["1"],
|
||||
"native_duplex": ["1"],
|
||||
"session_id": ["session-a"],
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ def test_realtime_client_builds_resume_only_url_when_autostart_disabled():
|
||||
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
assert query["autostart"] == ["0"]
|
||||
assert query["minicpmo45_native_duplex"] == ["1"]
|
||||
assert query["native_duplex"] == ["1"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -232,7 +232,7 @@ async def test_realtime_client_configure_explicit_tts_opts_out_of_native_duplex(
|
||||
session_extra_body = client.sent[0]["session"]["extra_body"]
|
||||
assert session_extra_body == {
|
||||
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||
"minicpmo45_native_duplex": False,
|
||||
"native_duplex": False,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Contract tests for the fullduplex core (PR #3907 alignment).
|
||||
|
||||
Covers the DuplexFence identity value, the session identity rules (epoch only
|
||||
@@ -14,6 +14,7 @@ import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.core import protocol as ev
|
||||
from vllm_omni.experimental.fullduplex.core.adapter import (
|
||||
DuplexAdapter,
|
||||
@@ -26,7 +27,6 @@ from vllm_omni.experimental.fullduplex.core.session import (
|
||||
DuplexSessionConfig,
|
||||
DuplexState,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PersonaPlex adapter driven through the model-owned lockstep runtime.
|
||||
|
||||
PersonaPlexDuplexRuntime starts ONE response for the whole session, streams one
|
||||
agent frame per user frame, and drains on close (core stays verbatim upstream;
|
||||
the lockstep lifecycle is model policy) — all with a GPU-free stub stepper.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.core import protocol as ev
|
||||
from vllm_omni.experimental.fullduplex.core.session import (
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
DuplexState,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.adapter import (
|
||||
PersonaPlexDuplexAdapter,
|
||||
PersonaPlexDuplexRuntime,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.personaplex.config import FRAME_SIZE
|
||||
from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class StubStepper:
|
||||
"""Deterministic FrameStepper: frame i -> audio filled with i, text 't{i}' on odds."""
|
||||
|
||||
sample_rate = 24000
|
||||
frame_size = FRAME_SIZE
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.opened: tuple[str | None, str | None] | None = None
|
||||
self.steps = 0
|
||||
|
||||
def open_session(self, voice_prompt=None, persona=None) -> None:
|
||||
self.opened = (voice_prompt, persona)
|
||||
self.steps = 0
|
||||
|
||||
def step(self, user_pcm):
|
||||
assert user_pcm.shape[0] == self.frame_size
|
||||
self.steps += 1
|
||||
i = self.steps
|
||||
return FrameOutput(
|
||||
audio=np.full(self.frame_size, float(i), dtype=np.float32),
|
||||
text=(f"t{i}" if i % 2 else None),
|
||||
)
|
||||
|
||||
|
||||
class RecordingAdapter(PersonaPlexDuplexAdapter):
|
||||
def __init__(self, stepper: StubStepper) -> None:
|
||||
super().__init__(stepper)
|
||||
self.closed = False
|
||||
|
||||
async def on_close(self, session: DuplexSession) -> None:
|
||||
self.closed = True
|
||||
await super().on_close(session)
|
||||
|
||||
|
||||
def _collector():
|
||||
out: list[dict] = []
|
||||
|
||||
async def emit(event: dict) -> None:
|
||||
out.append(event)
|
||||
|
||||
return out, emit
|
||||
|
||||
|
||||
async def _feed(events):
|
||||
for e in events:
|
||||
yield e
|
||||
|
||||
|
||||
async def _raising_feed(frame):
|
||||
yield {"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}
|
||||
await asyncio.sleep(0)
|
||||
raise RuntimeError("input failed")
|
||||
|
||||
|
||||
def _respond_tasks() -> list[asyncio.Task]:
|
||||
current = asyncio.current_task()
|
||||
return [
|
||||
task
|
||||
for task in asyncio.all_tasks()
|
||||
if task is not current and "_respond_once" in getattr(task.get_coro(), "__qualname__", "")
|
||||
]
|
||||
|
||||
|
||||
def _session():
|
||||
cfg = DuplexSessionConfig(
|
||||
input_modalities=("audio",),
|
||||
output_modalities=("audio", "text"),
|
||||
)
|
||||
return DuplexSession("s", cfg)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lockstep_one_agent_frame_per_user_frame():
|
||||
stub = StubStepper()
|
||||
adapter = PersonaPlexDuplexAdapter(stub, voice_prompt="NATF2.pt", persona="be terse")
|
||||
rt = PersonaPlexDuplexRuntime(_session(), adapter)
|
||||
out, emit = _collector()
|
||||
|
||||
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
|
||||
events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame} for _ in range(3)]
|
||||
events.append({"type": ev.CLOSE})
|
||||
await rt.run(_feed(events), emit)
|
||||
|
||||
types = [e["type"] for e in out]
|
||||
# exactly one response for the whole session, drained cleanly on close
|
||||
assert types.count(ev.RESPONSE_CREATED) == 1
|
||||
assert types.count(ev.RESPONSE_DONE) == 1
|
||||
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
|
||||
text = [e["data"] for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "text"]
|
||||
assert len(audio) == 3 # one agent frame per user frame
|
||||
assert [int(e["data"][0]) for e in audio] == [1, 2, 3] # lockstep order
|
||||
assert text == ["t1", "t3"]
|
||||
assert stub.steps == 3
|
||||
assert stub.opened == ("NATF2.pt", "be terse")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unaligned_chunks_are_reframed_to_80ms():
|
||||
stub = StubStepper()
|
||||
adapter = PersonaPlexDuplexAdapter(stub)
|
||||
rt = PersonaPlexDuplexRuntime(_session(), adapter)
|
||||
out, emit = _collector()
|
||||
|
||||
# 2.5 frames in one chunk + 0.5 frame in another => 3 whole frames
|
||||
events = [
|
||||
{"type": ev.INPUT_APPEND, "modality": "audio", "data": np.zeros(FRAME_SIZE * 5 // 2, dtype=np.float32)},
|
||||
{"type": ev.INPUT_APPEND, "modality": "audio", "data": np.zeros(FRAME_SIZE // 2, dtype=np.float32)},
|
||||
{"type": ev.CLOSE},
|
||||
]
|
||||
await rt.run(_feed(events), emit)
|
||||
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
|
||||
assert len(audio) == 3
|
||||
assert stub.steps == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_without_input_is_clean():
|
||||
adapter = PersonaPlexDuplexAdapter(StubStepper())
|
||||
rt = PersonaPlexDuplexRuntime(_session(), adapter)
|
||||
out, emit = _collector()
|
||||
await rt.run(_feed([{"type": ev.CLOSE}]), emit)
|
||||
# no input -> response never started -> no created/done, no error
|
||||
assert not any(e["type"] == ev.ERROR for e in out)
|
||||
assert not any(e["type"] == ev.RESPONSE_CREATED for e in out)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_iterator_eof_without_close_drains_and_stops():
|
||||
# Transport drop: the input iterator ends WITHOUT an explicit CLOSE event.
|
||||
# The runtime must still signal on_close so the eternal response drains
|
||||
# instead of blocking on its inbox forever.
|
||||
stub = StubStepper()
|
||||
adapter = PersonaPlexDuplexAdapter(stub)
|
||||
rt = PersonaPlexDuplexRuntime(_session(), adapter)
|
||||
out, emit = _collector()
|
||||
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
|
||||
events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}]
|
||||
await asyncio.wait_for(rt.run(_feed(events), emit), timeout=5.0)
|
||||
types = [e["type"] for e in out]
|
||||
assert types.count(ev.RESPONSE_DONE) == 1
|
||||
assert stub.steps == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_exception_closes_response_and_runtime() -> None:
|
||||
stub = StubStepper()
|
||||
adapter = RecordingAdapter(stub)
|
||||
rt = PersonaPlexDuplexRuntime(_session(), adapter)
|
||||
out, emit = _collector()
|
||||
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
|
||||
|
||||
with pytest.raises(RuntimeError, match="input failed"):
|
||||
await rt.run(_raising_feed(frame), emit)
|
||||
|
||||
leaked = _respond_tasks()
|
||||
for task in leaked:
|
||||
task.cancel()
|
||||
if leaked:
|
||||
await asyncio.gather(*leaked, return_exceptions=True)
|
||||
|
||||
assert adapter.closed is True
|
||||
assert rt.session.state is DuplexState.CLOSED
|
||||
assert not leaked
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_session_config_gets_lockstep_lifecycle():
|
||||
# The lifecycle comes from the model-owned runtime class, not a session
|
||||
# flag: a plain default-constructed session config runs lockstep.
|
||||
stub = StubStepper()
|
||||
adapter = PersonaPlexDuplexAdapter(stub)
|
||||
session = DuplexSession(
|
||||
"s",
|
||||
DuplexSessionConfig(input_modalities=("audio",), output_modalities=("audio", "text")),
|
||||
)
|
||||
rt = PersonaPlexDuplexRuntime(session, adapter)
|
||||
out, emit = _collector()
|
||||
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
|
||||
events = [{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame}, {"type": ev.CLOSE}]
|
||||
await rt.run(_feed(events), emit)
|
||||
types = [e["type"] for e in out]
|
||||
assert types.count(ev.RESPONSE_CREATED) == 1
|
||||
assert types.count(ev.RESPONSE_DONE) == 1
|
||||
assert stub.steps == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_tail_is_padded_and_stepped_on_close():
|
||||
# A trailing sub-frame chunk is zero-padded and stepped at close (same
|
||||
# contract as PersonaPlexSession.flush) rather than silently dropped.
|
||||
stub = StubStepper()
|
||||
adapter = PersonaPlexDuplexAdapter(stub)
|
||||
rt = PersonaPlexDuplexRuntime(_session(), adapter)
|
||||
out, emit = _collector()
|
||||
events = [
|
||||
{"type": ev.INPUT_APPEND, "modality": "audio", "data": np.ones(FRAME_SIZE // 2, dtype=np.float32)},
|
||||
{"type": ev.CLOSE},
|
||||
]
|
||||
await rt.run(_feed(events), emit)
|
||||
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
|
||||
assert len(audio) == 1
|
||||
assert stub.steps == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_eternal_response_across_barge_in():
|
||||
# Moved from the core contract tests when the lockstep lifecycle became
|
||||
# model-owned: ONE response for the whole session, RESPONSE_CANCEL bumps the
|
||||
# epoch fence (playback cursor) but the stream keeps flowing, and close
|
||||
# drains via on_close.
|
||||
stub = StubStepper()
|
||||
adapter = PersonaPlexDuplexAdapter(stub)
|
||||
session = _session()
|
||||
rt = PersonaPlexDuplexRuntime(session, adapter)
|
||||
out, emit = _collector()
|
||||
|
||||
frame = np.zeros(FRAME_SIZE, dtype=np.float32)
|
||||
events = [
|
||||
{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame},
|
||||
{"type": ev.RESPONSE_CANCEL}, # barge-in: epoch bumps, stream continues
|
||||
{"type": ev.INPUT_APPEND, "modality": "audio", "data": frame},
|
||||
{"type": ev.CLOSE},
|
||||
]
|
||||
await rt.run(_feed(events), emit)
|
||||
|
||||
types = [e["type"] for e in out]
|
||||
assert types.count(ev.RESPONSE_CREATED) == 1, "exactly one eternal response"
|
||||
assert types.count(ev.RESPONSE_DONE) == 1
|
||||
audio = [e for e in out if e["type"] == ev.RESPONSE_DELTA and e["modality"] == "audio"]
|
||||
assert len(audio) == 2, "stream keeps flowing across barge-in"
|
||||
assert session.epoch == 1, "cancel still advanced the epoch fence"
|
||||
assert stub.steps == 2
|
||||
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU-free tests for the batched PersonaPlex slot manager (stub engine)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, PrefillStep
|
||||
from vllm_omni.experimental.fullduplex.personaplex.serving.batched import (
|
||||
BatchedSessionManager,
|
||||
SlotPhase,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
FRAME = 4
|
||||
|
||||
|
||||
class StubEngine:
|
||||
"""Records the slot-API call sequence; echoes each row's pcm as its output."""
|
||||
|
||||
frame_size = FRAME
|
||||
|
||||
def __init__(self, batch_size: int = 4, prefill_len: int = 2) -> None:
|
||||
self.config = SimpleNamespace(batch_size=batch_size)
|
||||
self.prefill_len = prefill_len
|
||||
self.recycled: list[int] = []
|
||||
self.voice_ended: list[int] = []
|
||||
self.prefill_done: list[int] = []
|
||||
self.steps: list[tuple[np.ndarray, dict | None]] = []
|
||||
|
||||
def prefill_steps(self, persona=None):
|
||||
steps = [PrefillStep("sacrifice"), PrefillStep("voice", embedding=None), PrefillStep("voice_end")]
|
||||
steps += [PrefillStep("tokens", moshi_tokens=None, text_token=3, user_sine=True)] * self.prefill_len
|
||||
return steps
|
||||
|
||||
def recycle_slot(self, b):
|
||||
self.recycled.append(b)
|
||||
|
||||
def voice_end(self, b):
|
||||
self.voice_ended.append(b)
|
||||
|
||||
def end_prefill(self, b):
|
||||
self.prefill_done.append(b)
|
||||
|
||||
def step_batch(self, pcm, prefill=None):
|
||||
self.steps.append((pcm.copy(), dict(prefill) if prefill else None))
|
||||
return [FrameOutput(audio=pcm[b].copy(), text=None) for b in range(self.config.batch_size)]
|
||||
|
||||
|
||||
def test_all_idle_does_not_step():
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng)
|
||||
assert mgr.tick() is False
|
||||
assert eng.steps == []
|
||||
|
||||
|
||||
def test_pristine_slot_is_live_immediately():
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng)
|
||||
slot, ready_now = mgr.acquire()
|
||||
assert ready_now is True
|
||||
assert slot.phase is SlotPhase.LIVE
|
||||
assert eng.recycled == []
|
||||
|
||||
|
||||
def test_custom_persona_prefills_and_recycles_on_tick_thread():
|
||||
eng = StubEngine(prefill_len=2)
|
||||
mgr = BatchedSessionManager(eng)
|
||||
slot, ready_now = mgr.acquire(persona="custom")
|
||||
assert ready_now is False
|
||||
assert slot.phase is SlotPhase.PREFILL
|
||||
assert eng.recycled == [] # deferred: reset must run between engine steps
|
||||
|
||||
ready = []
|
||||
slot.on_ready = lambda: ready.append(True)
|
||||
|
||||
mgr.tick() # recycle + sacrifice
|
||||
assert eng.recycled == [slot.index]
|
||||
assert eng.steps[-1][1][slot.index].kind == "sacrifice"
|
||||
|
||||
mgr.tick() # voice frame
|
||||
assert eng.steps[-1][1][slot.index].kind == "voice"
|
||||
assert eng.voice_ended == []
|
||||
|
||||
mgr.tick() # voice_end executed BETWEEN ticks, then first tokens frame
|
||||
assert eng.voice_ended == [slot.index]
|
||||
assert eng.steps[-1][1][slot.index].kind == "tokens"
|
||||
|
||||
mgr.tick() # second tokens frame
|
||||
assert not ready
|
||||
mgr.tick() # prefill exhausted -> live
|
||||
assert eng.prefill_done == [slot.index]
|
||||
assert slot.phase is SlotPhase.LIVE
|
||||
assert ready == [True]
|
||||
|
||||
|
||||
def test_pristine_slot_after_ticking_requires_recycle():
|
||||
# step_batch advances EVERY row each tick, so a pristine idle row that has
|
||||
# lived through ticks has drifted from its boot prefill and must be
|
||||
# recycled before a client may join it.
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng)
|
||||
first, ready_now = mgr.acquire()
|
||||
assert ready_now is True # claimed before any tick: truly pristine
|
||||
mgr.tick()
|
||||
second, ready_now2 = mgr.acquire()
|
||||
assert second.index != first.index
|
||||
assert ready_now2 is False
|
||||
assert second.phase is SlotPhase.PREFILL
|
||||
|
||||
|
||||
def test_used_slot_requires_recycle_even_with_default_persona():
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng)
|
||||
slot, ready_now = mgr.acquire()
|
||||
assert ready_now
|
||||
mgr.release(slot)
|
||||
slot2, ready_now2 = mgr.acquire()
|
||||
assert slot2.index == slot.index
|
||||
assert ready_now2 is False
|
||||
assert slot2.phase is SlotPhase.PREFILL
|
||||
|
||||
|
||||
def test_acquire_returns_none_when_full():
|
||||
eng = StubEngine(batch_size=2)
|
||||
mgr = BatchedSessionManager(eng)
|
||||
assert mgr.acquire() is not None
|
||||
assert mgr.acquire() is not None
|
||||
assert mgr.acquire() is None
|
||||
|
||||
|
||||
def test_feed_frames_epoch_guard_and_backlog_cap():
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng, max_buffered_frames=3)
|
||||
slot, _ = mgr.acquire()
|
||||
mgr.feed(slot, np.arange(FRAME * 2, dtype=np.float32), slot.epoch)
|
||||
assert len(slot.inq) == 2
|
||||
mgr.feed(slot, np.zeros(3, dtype=np.float32), slot.epoch) # partial tail buffered
|
||||
assert len(slot.inq) == 2 and slot.pending.shape[0] == 3
|
||||
mgr.feed(slot, np.zeros(FRAME * 5, dtype=np.float32), slot.epoch)
|
||||
assert len(slot.inq) == 3 # oldest dropped beyond the cap
|
||||
mgr.feed(slot, np.zeros(FRAME, dtype=np.float32), slot.epoch - 1) # stale epoch ignored
|
||||
assert len(slot.inq) == 3
|
||||
|
||||
|
||||
def test_live_slot_consumes_input_and_emits_output():
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng)
|
||||
slot, _ = mgr.acquire()
|
||||
got = []
|
||||
slot.on_output = got.append
|
||||
frame = np.full(FRAME, 0.5, dtype=np.float32)
|
||||
mgr.feed(slot, frame, slot.epoch)
|
||||
mgr.tick()
|
||||
assert len(got) == 1
|
||||
np.testing.assert_array_equal(got[0].audio, frame)
|
||||
# queue drained: next tick feeds zeros
|
||||
mgr.tick()
|
||||
np.testing.assert_array_equal(got[1].audio, np.zeros(FRAME, dtype=np.float32))
|
||||
|
||||
|
||||
def test_release_mid_flight_stops_emission():
|
||||
eng = StubEngine()
|
||||
mgr = BatchedSessionManager(eng)
|
||||
slot, _ = mgr.acquire()
|
||||
got = []
|
||||
slot.on_output = got.append
|
||||
mgr.tick()
|
||||
assert len(got) == 1
|
||||
mgr.release(slot)
|
||||
# another slot keeps the batch non-idle
|
||||
other, _ = mgr.acquire() # same slot index, now PREFILL (used)
|
||||
assert other.phase is SlotPhase.PREFILL
|
||||
mgr.tick()
|
||||
assert len(got) == 1 # no further output for the released epoch
|
||||
@@ -1,245 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""GPU e2e for PersonaPlex elastic batching (bit-exact isolation + recycle).
|
||||
|
||||
Greedy decoding makes every cross-slot leak observable as a bit difference:
|
||||
|
||||
1. Isolation (bit-exact): recycling one slot mid-flight leaves every other
|
||||
slot's output bit-identical to a run that never recycled.
|
||||
2. Replay exactness: a recycled slot's opening live ticks match a pristine boot
|
||||
at numerical-noise level (a broken KV mask / staging roll garbles frame 0).
|
||||
3. Coherence (behavioral): a recycled slot fed a real spoken question produces
|
||||
a real reply in its inner-monologue text stream.
|
||||
|
||||
Run (moshi-free native stack; ~30 GB GPU + HF access to the gated repo; set
|
||||
PPLEX_QUESTION_WAV to a spoken-question wav to enable the coherence test):
|
||||
pytest tests/e2e/features/fullduplex/test_personaplex_elastic.py -v -s
|
||||
|
||||
Bench (B sweep, ms/tick vs the 80 ms budget):
|
||||
python tests/e2e/features/fullduplex/test_personaplex_elastic.py --bench 2 4 8 16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import hardware_marks
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
# slow + H100 + omni + cards_1 routes this module into weekly
|
||||
# "Omni · H100 · Single-GPU" (test-weekly.yml).
|
||||
pytestmark = [
|
||||
pytest.mark.omni,
|
||||
pytest.mark.slow,
|
||||
*hardware_marks(res={"cuda": "H100"}),
|
||||
]
|
||||
|
||||
if not torch.cuda.is_available(): # noqa: E402
|
||||
pytest.skip("needs CUDA", allow_module_level=True)
|
||||
|
||||
from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig # noqa: E402
|
||||
from vllm_omni.experimental.fullduplex.personaplex.runtime import ( # noqa: E402
|
||||
PersonaPlexEngine,
|
||||
)
|
||||
|
||||
B = 4
|
||||
LIVE_TICKS = 40 # live frames compared per claim
|
||||
RECYCLE_AT = 10
|
||||
|
||||
|
||||
def _make_engine() -> PersonaPlexEngine:
|
||||
cfg = PersonaPlexConfig(batch_size=B) # native stepper is greedy-only
|
||||
try:
|
||||
return PersonaPlexEngine(cfg).load()
|
||||
except Exception as exc: # gated-repo access is environment-dependent
|
||||
name = type(exc).__name__
|
||||
if "Gated" in name or "401" in str(exc) or "403" in str(exc):
|
||||
pytest.skip(f"no access to the gated nvidia/personaplex-7b-v1 repo: {name}")
|
||||
raise
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def engine() -> PersonaPlexEngine:
|
||||
return _make_engine()
|
||||
|
||||
|
||||
def _slot_stream(slot: int, tick: int, frame: int) -> np.ndarray:
|
||||
"""Deterministic per-slot input: distinct low-amplitude tones."""
|
||||
sr = 24000
|
||||
freq = 180.0 + 90.0 * slot
|
||||
t0 = tick * frame
|
||||
t = (np.arange(frame) + t0) / sr
|
||||
return (0.12 * np.sin(2 * np.pi * freq * t)).astype(np.float32)
|
||||
|
||||
|
||||
def _run(engine, total_ticks: int, recycle: bool):
|
||||
"""Run the lockstep loop; optionally recycle slot 1 at RECYCLE_AT.
|
||||
|
||||
Returns per-slot lists of (audio | None) per live tick index, where the
|
||||
recycled slot's list restarts its input stream at its post-prefill tick.
|
||||
"""
|
||||
frame = engine.frame_size
|
||||
outs: list[list[np.ndarray | None]] = [[] for _ in range(B)]
|
||||
stream_tick = [0] * B # per-slot conversation clock driving its input
|
||||
prefill = None
|
||||
tick = 0
|
||||
while tick < total_ticks:
|
||||
if recycle and tick == RECYCLE_AT and prefill is None:
|
||||
engine.recycle_slot(1)
|
||||
prefill = list(engine.prefill_steps(None))
|
||||
pcm = np.zeros((B, frame), dtype=np.float32)
|
||||
step_for_1 = None
|
||||
if prefill:
|
||||
while prefill and prefill[0].kind == "voice_end":
|
||||
engine.voice_end(1)
|
||||
prefill.pop(0)
|
||||
if prefill:
|
||||
step_for_1 = prefill.pop(0)
|
||||
else:
|
||||
engine.end_prefill(1)
|
||||
prefill = None
|
||||
stream_tick[1] = 0 # restart slot 1's input stream from its own t=0
|
||||
for b in range(B):
|
||||
if b == 1 and step_for_1 is not None:
|
||||
continue # engine overrides the user channel for prefill rows
|
||||
pcm[b] = _slot_stream(b, stream_tick[b], frame)
|
||||
stream_tick[b] += 1
|
||||
frame_outs = engine.step_batch(pcm, {1: step_for_1} if step_for_1 is not None else None)
|
||||
for b in range(B):
|
||||
if b == 1 and step_for_1 is not None:
|
||||
continue # prefill tick: nothing conversational to record
|
||||
frame_out = frame_outs[b]
|
||||
outs[b].append(None if frame_out is None else frame_out.audio)
|
||||
tick += 1
|
||||
return outs
|
||||
|
||||
|
||||
def test_recycle_isolation_and_fresh_equivalence(engine):
|
||||
engine.open_batch()
|
||||
prefill_len = sum(1 for s in engine.prefill_steps(None) if s.kind != "voice_end")
|
||||
total = RECYCLE_AT + prefill_len + LIVE_TICKS
|
||||
|
||||
base = _run(engine, total, recycle=False)
|
||||
|
||||
engine.open_batch() # deterministic re-boot (greedy, fixed voice/persona)
|
||||
rec = _run(engine, total, recycle=True)
|
||||
|
||||
# Claim 1 -- isolation: untouched slots bit-identical to the no-recycle run.
|
||||
for b in (0, 2, 3):
|
||||
assert len(base[b]) == total and len(rec[b]) == total
|
||||
for i, (x, y) in enumerate(zip(base[b], rec[b])):
|
||||
assert (x is None) == (y is None), f"slot {b} tick {i} validity diverged"
|
||||
if x is not None:
|
||||
assert np.array_equal(x, y), f"slot {b} tick {i} audio diverged (recycle leaked)"
|
||||
|
||||
# Claim 2 -- replay exactness: the recycled slot's opening live ticks match a
|
||||
# pristine boot at numerical-noise level. A broken KV mask / staging-ring
|
||||
# roll / Mimi reset garbles the audio from the very first frame (diffs of
|
||||
# 0.1-1.0); a correct replay differs only by decoder-state rounding (~1e-4).
|
||||
#
|
||||
# Full-length bitwise identity is NOT expected, for two structural reasons:
|
||||
# the replayed system prompt sits at a different ABSOLUTE RoPE offset (bf16
|
||||
# rounds differently there), and the recycled Mimi ENCODER cannot restore a
|
||||
# fresh stream's start padding (a batch-shared scalar), so from the first
|
||||
# non-silent user frame it emits different-but-equally-valid codes for the
|
||||
# same audio. Once any near-tie flips a token, the two equally valid
|
||||
# trajectories legitimately diverge; coherence of the recycled slot is
|
||||
# asserted behaviorally in test_recycled_slot_converses.
|
||||
fresh = [a for a in base[1] if a is not None][:LIVE_TICKS]
|
||||
reborn = [a for a in rec[1][RECYCLE_AT:] if a is not None][:LIVE_TICKS]
|
||||
assert len(reborn) >= LIVE_TICKS // 2, "recycled slot produced too few live frames"
|
||||
diffs = [float(np.max(np.abs(x - y))) for x, y in zip(fresh, reborn)]
|
||||
first_flip = next((i for i, d in enumerate(diffs) if d > 5e-2), len(diffs))
|
||||
print(
|
||||
f"[elastic] replay exactness: {first_flip}/{len(diffs)} opening ticks at noise level "
|
||||
f"(head diffs: {[f'{d:.1e}' for d in diffs[:6]]})"
|
||||
)
|
||||
assert first_flip >= 4, (
|
||||
f"recycled slot diverged from fresh behavior at tick {first_flip} "
|
||||
f"(diffs {diffs[: first_flip + 2]}); the system-prompt replay is broken"
|
||||
)
|
||||
assert max(diffs[:4]) < 2e-2, f"reborn audio not at noise level: {max(diffs[:4])}"
|
||||
|
||||
|
||||
def test_recycled_slot_converses(engine):
|
||||
"""Behavioral guarantee: a recycled slot holds a coherent conversation.
|
||||
|
||||
Feed a real spoken question into a slot that was recycled mid-flight (while
|
||||
other slots keep running); the model's own inner-monologue text stream must
|
||||
produce a real reply. This is the end-user property the encoder-state
|
||||
caveats above cannot break.
|
||||
"""
|
||||
import os
|
||||
|
||||
import sphn
|
||||
|
||||
wav_path = os.environ.get("PPLEX_QUESTION_WAV")
|
||||
if not wav_path:
|
||||
pytest.skip("set PPLEX_QUESTION_WAV to a spoken-question wav to run")
|
||||
wav, _ = sphn.read(wav_path, sample_rate=24000)
|
||||
wav = wav[0].astype(np.float32)
|
||||
frame = engine.frame_size
|
||||
pad = (-len(wav)) % frame
|
||||
wav = np.concatenate([wav, np.zeros(pad, dtype=np.float32)])
|
||||
q_frames = [wav[i : i + frame] for i in range(0, len(wav), frame)]
|
||||
|
||||
engine.open_batch()
|
||||
pcm = np.zeros((B, frame), dtype=np.float32)
|
||||
for _ in range(5): # everyone live briefly
|
||||
engine.step_batch(pcm)
|
||||
engine.recycle_slot(1)
|
||||
prefill = list(engine.prefill_steps(None))
|
||||
while prefill:
|
||||
while prefill and prefill[0].kind == "voice_end":
|
||||
engine.voice_end(1)
|
||||
prefill.pop(0)
|
||||
if not prefill:
|
||||
break
|
||||
engine.step_batch(pcm, {1: prefill.pop(0)})
|
||||
engine.end_prefill(1)
|
||||
|
||||
text = ""
|
||||
total = len(q_frames) + 120 # question + ~10 s to answer
|
||||
for i in range(total):
|
||||
pcm = np.zeros((B, frame), dtype=np.float32)
|
||||
if i < len(q_frames):
|
||||
pcm[1] = q_frames[i]
|
||||
outs = engine.step_batch(pcm)
|
||||
if outs[1] is not None and outs[1].text:
|
||||
text += outs[1].text
|
||||
alpha = sum(c.isalpha() for c in text)
|
||||
print(f"[elastic] recycled slot inner monologue: {text!r}")
|
||||
assert alpha >= 15, f"recycled slot produced no coherent reply: {text!r}"
|
||||
|
||||
|
||||
def _bench(batch_sizes: list[int]) -> None:
|
||||
import time
|
||||
|
||||
for bs in batch_sizes:
|
||||
cfg = PersonaPlexConfig(batch_size=max(bs, 2))
|
||||
eng = PersonaPlexEngine(cfg).load()
|
||||
eng.open_batch()
|
||||
frame = eng.frame_size
|
||||
pcm = np.zeros((cfg.batch_size, frame), dtype=np.float32)
|
||||
for _ in range(10): # warm
|
||||
eng.step_batch(pcm)
|
||||
torch.cuda.synchronize()
|
||||
t0 = time.monotonic()
|
||||
n = 100
|
||||
for _ in range(n):
|
||||
eng.step_batch(pcm)
|
||||
torch.cuda.synchronize()
|
||||
ms = (time.monotonic() - t0) / n * 1000
|
||||
mem = torch.cuda.max_memory_allocated() / 2**30
|
||||
print(f"B={cfg.batch_size:3d} {ms:6.1f} ms/tick RTF/stream {ms / 80:.3f} {mem:.1f} GiB")
|
||||
del eng
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sizes = [int(x) for x in sys.argv[2:]] if len(sys.argv) > 2 and sys.argv[1] == "--bench" else [2, 4, 8]
|
||||
_bench(sizes)
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Single-session lease teardown: a cancelled engine call must be drained.
|
||||
|
||||
Cancelling an asyncio task blocked on a worker-thread engine call returns
|
||||
immediately while the thread keeps running (executor futures cannot be
|
||||
interrupted). The server therefore tracks the in-flight future on the session
|
||||
state and waits it out before releasing the capacity-one lease; otherwise a
|
||||
fast reconnect could call ``open()`` on the shared engine while the previous
|
||||
connection's final ``step()`` is still mutating it.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
try:
|
||||
import sphn # noqa: F401
|
||||
except ImportError:
|
||||
# These lease tests never encode/decode Opus. Keep the optional serving
|
||||
# dependency from hiding dependency-free teardown regressions.
|
||||
sys.modules["sphn"] = types.ModuleType("sphn")
|
||||
|
||||
from vllm_omni.experimental.fullduplex.personaplex.config import PersonaPlexConfig
|
||||
from vllm_omni.experimental.fullduplex.personaplex.serving.server import DuplexServer
|
||||
from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexServingSessionState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_engine_call_is_drained_before_lease_release():
|
||||
server = DuplexServer(PersonaPlexConfig()) # engine never loaded: stub calls only
|
||||
state = PersonaPlexServingSessionState()
|
||||
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
finished = threading.Event()
|
||||
|
||||
def slow_engine_op():
|
||||
started.set()
|
||||
release.wait(timeout=5.0)
|
||||
finished.set()
|
||||
return []
|
||||
|
||||
task = asyncio.create_task(server._engine_call(state, slow_engine_op))
|
||||
await asyncio.to_thread(started.wait, 5.0)
|
||||
|
||||
# Cancel while the worker thread is mid-call: the await returns at once,
|
||||
# the thread keeps running, and its completion event stays tracked on the
|
||||
# state (the asyncio future itself reads as cancelled/done here, which is
|
||||
# exactly why the drain must not rely on it).
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
assert state.inflight is not None
|
||||
assert not state.inflight.is_set()
|
||||
|
||||
# The drain must genuinely wait for the worker, not just return.
|
||||
drain = asyncio.create_task(server._drain_inflight(state))
|
||||
await asyncio.sleep(0.05)
|
||||
assert not drain.done()
|
||||
|
||||
release.set()
|
||||
await asyncio.wait_for(drain, timeout=5.0)
|
||||
assert finished.is_set()
|
||||
assert state.inflight.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_engine_call_clears_inflight():
|
||||
server = DuplexServer(PersonaPlexConfig())
|
||||
state = PersonaPlexServingSessionState()
|
||||
result = await server._engine_call(state, lambda: "ok")
|
||||
assert result == "ok"
|
||||
assert state.inflight is None
|
||||
await server._drain_inflight(state) # no-op on a clean state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_timeout_keeps_lease_until_worker_finishes():
|
||||
server = DuplexServer(PersonaPlexConfig())
|
||||
server._active = True
|
||||
state = PersonaPlexServingSessionState()
|
||||
state.inflight = threading.Event()
|
||||
|
||||
drained = await server._release_lease_after_drain(state, timeout=0.01)
|
||||
|
||||
assert drained is False
|
||||
assert server._active is True
|
||||
|
||||
state.inflight.set()
|
||||
for _ in range(100):
|
||||
if not server._active:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert server._active is False
|
||||
@@ -1,97 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lockstep-contract tests for the PersonaPlex session driver (no GPU / no moshi).
|
||||
|
||||
A stub :class:`FrameStepper` stands in for the real Moshi backend so we can assert
|
||||
the framing + 1-frame-in / 1-frame-out invariants without weights.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.experimental.fullduplex.personaplex.config import FRAME_SIZE, PersonaPlexConfig
|
||||
from vllm_omni.experimental.fullduplex.personaplex.engine import FrameOutput, FrameStepper
|
||||
from vllm_omni.experimental.fullduplex.personaplex.session import PersonaPlexSession
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class StubStepper:
|
||||
"""Deterministic FrameStepper: frame i -> audio filled with i, text 't{i}' on odds."""
|
||||
|
||||
sample_rate = 24000
|
||||
frame_size = FRAME_SIZE
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.opened: tuple[str | None, str | None] | None = None
|
||||
self.steps = 0
|
||||
|
||||
def open_session(self, voice_prompt=None, persona=None) -> None:
|
||||
self.opened = (voice_prompt, persona)
|
||||
self.steps = 0
|
||||
|
||||
def step(self, user_pcm):
|
||||
assert user_pcm.shape[0] == self.frame_size
|
||||
self.steps += 1
|
||||
i = self.steps
|
||||
return FrameOutput(
|
||||
audio=np.full(self.frame_size, float(i), dtype=np.float32),
|
||||
text=(f"t{i}" if i % 2 else None),
|
||||
)
|
||||
|
||||
|
||||
def test_stub_satisfies_protocol():
|
||||
assert isinstance(StubStepper(), FrameStepper)
|
||||
|
||||
|
||||
def test_feed_emits_one_output_per_complete_frame():
|
||||
s = PersonaPlexSession(StubStepper(), PersonaPlexConfig())
|
||||
s.open(voice_prompt="NATF2.pt", persona="be terse")
|
||||
out = s.feed(np.zeros(FRAME_SIZE * 3, dtype=np.float32))
|
||||
assert len(out) == 3
|
||||
# lockstep order preserved: frame i -> audio filled with i
|
||||
assert [int(o.audio[0]) for o in out] == [1, 2, 3]
|
||||
assert [o.text for o in out] == ["t1", None, "t3"]
|
||||
|
||||
|
||||
def test_partial_frames_are_buffered_until_complete():
|
||||
stub = StubStepper()
|
||||
s = PersonaPlexSession(stub, PersonaPlexConfig())
|
||||
s.open()
|
||||
assert s.feed(np.zeros(FRAME_SIZE // 2, dtype=np.float32)) == [] # nothing yet
|
||||
assert stub.steps == 0
|
||||
out = s.feed(np.zeros(FRAME_SIZE, dtype=np.float32)) # 1.5 frames total -> 1 frame
|
||||
assert len(out) == 1
|
||||
assert stub.steps == 1
|
||||
|
||||
|
||||
def test_flush_pads_the_tail():
|
||||
stub = StubStepper()
|
||||
s = PersonaPlexSession(stub, PersonaPlexConfig())
|
||||
s.open()
|
||||
s.feed(np.ones(FRAME_SIZE + 10, dtype=np.float32)) # 1 frame + 10-sample tail
|
||||
assert stub.steps == 1
|
||||
tail = s.flush()
|
||||
assert len(tail) == 1 and stub.steps == 2
|
||||
assert s.flush() == [] # nothing left
|
||||
|
||||
|
||||
def test_run_concatenates_audio_and_joins_text():
|
||||
s = PersonaPlexSession(StubStepper(), PersonaPlexConfig())
|
||||
s.open()
|
||||
audio, text = s.run(np.zeros(FRAME_SIZE * 4, dtype=np.float32))
|
||||
assert audio.shape[0] == FRAME_SIZE * 4
|
||||
assert text == "t1t3" # frames 1..4, text on odds
|
||||
|
||||
|
||||
def test_open_forwards_voice_and_persona():
|
||||
stub = StubStepper()
|
||||
s = PersonaPlexSession(stub, PersonaPlexConfig())
|
||||
s.open(voice_prompt="NATM1.pt", persona="pirate")
|
||||
assert stub.opened == ("NATM1.pt", "pirate")
|
||||
|
||||
|
||||
def test_feed_before_open_raises():
|
||||
s = PersonaPlexSession(StubStepper(), PersonaPlexConfig())
|
||||
with pytest.raises(RuntimeError):
|
||||
s.feed(np.zeros(FRAME_SIZE, dtype=np.float32))
|
||||
Reference in new issue
Block a user