[Bugfix] Reject instructions changes on locked native duplex sessions (#6318)
Signed-off-by: ANURAG KANADE <anuragkanade6@gmail.com>
This commit is contained in:
1 parent
926e7fbfa2
commit
91dd7fcaec
8 files changed
+309
-7
No files matched your search
@@ -49,6 +49,9 @@ from vllm_omni.experimental.fullduplex.openai.serving import (
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.openai.websocket import DuplexWebSocketActor
|
||||
from vllm_omni.experimental.fullduplex.output import attach_duplex_output_decision
|
||||
from vllm_omni.experimental.fullduplex.personaplex.serving_adapter import (
|
||||
PersonaPlexServingRuntimeAdapter,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -991,6 +994,178 @@ async def test_minicpmo_native_session_update_requires_ref_audio_before_enabling
|
||||
assert ("sid-runtime-update-ref-required", "session.update") not in engine.signals
|
||||
|
||||
|
||||
def test_personaplex_candidate_update_does_not_leak_minicpmo_ref_audio_check():
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=SimpleNamespace(engine_client=SimpleNamespace()),
|
||||
serving_runtime_adapter=PersonaPlexServingRuntimeAdapter(lambda *_: None),
|
||||
)
|
||||
session = DuplexSession(
|
||||
session_id="sid-personaplex-voice-change",
|
||||
config=DuplexSessionConfig(modalities=["audio"], voice="NATF2.pt"),
|
||||
capabilities=PersonaPlexServingRuntimeAdapter.capabilities(max_sessions=1),
|
||||
)
|
||||
candidate_config = DuplexSessionConfig(modalities=["audio"], voice="OtherVoice.pt")
|
||||
|
||||
error = handler._runtime_session_candidate_update_error(session, candidate_config)
|
||||
|
||||
assert error is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_session_update_rejects_instructions_after_native_context_locked():
|
||||
engine = FakeEngineClient()
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
ws = TimedWebSocket()
|
||||
ws.put(_native_session_create("sid-instructions-locked", modalities=["text"]))
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(16000, value=0.05),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
"duration_ms": 1000,
|
||||
"is_speech": True,
|
||||
}
|
||||
)
|
||||
ws.put(
|
||||
{
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {"instructions": "You are now a pirate."},
|
||||
}
|
||||
)
|
||||
ws.put({"type": "session.close"})
|
||||
|
||||
await handler.handle_session(ws)
|
||||
|
||||
error = next(message for message in ws.sent if message.get("type") == "error")
|
||||
assert error["code"] == "instructions_update_unsupported"
|
||||
assert "session.updated" not in ws.sent_types()
|
||||
assert ("sid-instructions-locked", "session.update") not in engine.signals
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_session_update_allows_unchanged_instructions_after_lock():
|
||||
engine = FakeEngineClient()
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
ws = TimedWebSocket()
|
||||
ws.put(_native_session_create("sid-instructions-unchanged", modalities=["text"]))
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(16000, value=0.05),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
"duration_ms": 1000,
|
||||
"is_speech": True,
|
||||
}
|
||||
)
|
||||
ws.put(
|
||||
{
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {"instructions": "You are a concise assistant.", "temperature": 0.3},
|
||||
}
|
||||
)
|
||||
ws.put({"type": "session.close"})
|
||||
|
||||
await handler.handle_session(ws)
|
||||
|
||||
assert "session.updated" in ws.sent_types()
|
||||
assert not any(
|
||||
message.get("type") == "error" and message.get("code") == "instructions_update_unsupported"
|
||||
for message in ws.sent
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_session_update_rejects_instructions_before_first_real_append():
|
||||
engine = FakeEngineClient()
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
ws = TimedWebSocket()
|
||||
ws.put(_native_session_create("sid-buffered-before-lock", modalities=["text"]))
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(3200, value=0.05),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
"duration_ms": 200,
|
||||
"is_speech": True,
|
||||
}
|
||||
)
|
||||
ws.put(
|
||||
{
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {
|
||||
"instructions": (
|
||||
"You are now a swashbuckling pirate captain who speaks entirely in "
|
||||
"nautical slang, references buried treasure constantly, and never "
|
||||
"breaks character no matter what the user asks."
|
||||
)
|
||||
},
|
||||
}
|
||||
)
|
||||
ws.put({"type": "input_audio_buffer.commit", "final": True, "response_create": True})
|
||||
ws.put({"type": "session.close"})
|
||||
|
||||
await handler.handle_session(ws)
|
||||
|
||||
error = next(message for message in ws.sent if message.get("type") == "error")
|
||||
assert error["code"] == "instructions_update_unsupported"
|
||||
assert "session.updated" not in ws.sent_types()
|
||||
assert engine.appended, "the commit must still flush the buffered chunk despite the rejected update"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_session_update_rejects_native_duplex_mode_flip():
|
||||
engine = FakeEngineClient()
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
ws = TimedWebSocket()
|
||||
ws.put(_native_session_create("sid-native-duplex-flag-flip", modalities=["text"]))
|
||||
ws.put(
|
||||
{
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {"extra_body": {"minicpmo45_native_duplex": False}},
|
||||
}
|
||||
)
|
||||
ws.put(
|
||||
{
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {"instructions": "You are now a pirate."},
|
||||
}
|
||||
)
|
||||
ws.put({"type": "session.close"})
|
||||
|
||||
await handler.handle_session(ws)
|
||||
|
||||
errors = [message for message in ws.sent if message.get("type") == "error"]
|
||||
assert [error["code"] for error in errors] == [
|
||||
"native_duplex_mode_update_unsupported",
|
||||
"instructions_update_unsupported",
|
||||
]
|
||||
assert "session.updated" not in ws.sent_types()
|
||||
|
||||
|
||||
def test_native_realtime_protocol_audio_delta_preserves_sample_rate_hz():
|
||||
ws = TimedWebSocket()
|
||||
protocol = NativeRealtimeSessionProtocol(ws) # type: ignore[arg-type]
|
||||
|
||||
Reference in new issue
Block a user