[Core] Graduate MiniCPM-o 4.5 and PersonaPlex full-duplex serving out of experimental (#6196)
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
179 files changed
+6462
-3832
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.commit_policy import (
|
||||
CommitAction,
|
||||
CommitSnapshot,
|
||||
decide_commit_action,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "snapshot", "expected"),
|
||||
[
|
||||
(
|
||||
"first idle commit",
|
||||
CommitSnapshot(
|
||||
auto_responds=True,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=False,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.START_AUTO_RESPONSE,
|
||||
),
|
||||
(
|
||||
"native auto-response remains model-owned during playback",
|
||||
CommitSnapshot(
|
||||
auto_responds=True,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=True,
|
||||
playback_active=True,
|
||||
),
|
||||
CommitAction.START_AUTO_RESPONSE,
|
||||
),
|
||||
(
|
||||
"native auto-response commit without speech",
|
||||
CommitSnapshot(
|
||||
auto_responds=True,
|
||||
speech_since_commit=False,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=True,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.COMMIT_ONLY,
|
||||
),
|
||||
(
|
||||
"non-auto response overlap",
|
||||
CommitSnapshot(
|
||||
auto_responds=False,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=True,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.DEFER_ACTIVE_RESPONSE,
|
||||
),
|
||||
(
|
||||
"non-auto playback overlap",
|
||||
CommitSnapshot(
|
||||
auto_responds=False,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=False,
|
||||
playback_active=True,
|
||||
),
|
||||
CommitAction.DEFER_ACTIVE_RESPONSE,
|
||||
),
|
||||
(
|
||||
"non-auto commit without overlap",
|
||||
CommitSnapshot(
|
||||
auto_responds=False,
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=False,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.COMMIT_ONLY,
|
||||
),
|
||||
],
|
||||
ids=lambda value: value if isinstance(value, str) else None,
|
||||
)
|
||||
def test_decide_commit_action_covers_realtime_lifecycle_sequences(
|
||||
case: str,
|
||||
snapshot: CommitSnapshot,
|
||||
expected: CommitAction,
|
||||
) -> None:
|
||||
del case
|
||||
|
||||
assert decide_commit_action(snapshot) is expected
|
||||
@@ -0,0 +1,179 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.input import MiniCPMO45PcmAppendBuffer
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def pcm_payload(samples: int, *, speech: bool = True) -> dict[str, object]:
|
||||
audio = np.ones(samples, dtype=np.float32).tobytes()
|
||||
return {
|
||||
"type": "audio",
|
||||
"audio": base64.b64encode(audio).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16_000,
|
||||
"is_speech": speech,
|
||||
}
|
||||
|
||||
|
||||
def test_commit_does_not_add_silence_after_incremental_audio_was_drained():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
|
||||
emitted = buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
|
||||
committed = buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
assert emitted is not None
|
||||
assert not buffer.has_pending()
|
||||
assert committed is None
|
||||
|
||||
|
||||
def test_append_emits_one_model_unit_when_multiple_units_are_buffered():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
|
||||
emitted = buffer.append(pcm_payload(32_000), chunk_period_ms=1_000)
|
||||
|
||||
assert emitted is not None
|
||||
assert len(base64.b64decode(emitted["audio"])) == 16_000 * 4
|
||||
assert buffer.pending_byte_count == 16_000 * 4
|
||||
|
||||
|
||||
def test_speech_marker_does_not_leak_across_irregular_chunk_boundaries():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
|
||||
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
|
||||
speech_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
|
||||
silence_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
|
||||
|
||||
assert speech_unit is not None
|
||||
assert speech_unit["is_speech"] is True
|
||||
assert silence_unit is not None
|
||||
assert silence_unit["is_speech"] is False
|
||||
|
||||
|
||||
def test_force_listen_marker_does_not_leak_across_irregular_chunk_boundaries():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
forced = pcm_payload(15_000, speech=False)
|
||||
forced["force_listen"] = True
|
||||
|
||||
assert buffer.append(forced, chunk_period_ms=1_000) is None
|
||||
forced_unit = buffer.append(pcm_payload(2_000, speech=False), chunk_period_ms=1_000)
|
||||
unforced_unit = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
|
||||
|
||||
assert forced_unit is not None
|
||||
assert forced_unit["force_listen"] is True
|
||||
assert unforced_unit is not None
|
||||
assert unforced_unit["force_listen"] is False
|
||||
|
||||
|
||||
def test_pcm_append_rollback_restores_per_span_speech_markers():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
assert buffer.append(pcm_payload(15_000), chunk_period_ms=1_000) is None
|
||||
reservation = buffer.prepare_append(
|
||||
pcm_payload(2_000, speech=False),
|
||||
operation_id="mixed-unit",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert reservation is not None
|
||||
reservation.rollback()
|
||||
first = buffer.append(pcm_payload(15_000, speech=False), chunk_period_ms=1_000)
|
||||
second = buffer.flush(chunk_period_ms=1_000)
|
||||
|
||||
assert first is not None
|
||||
assert first["is_speech"] is True
|
||||
assert second is not None
|
||||
assert second["is_speech"] is False
|
||||
|
||||
|
||||
def test_commit_without_speech_does_not_synthesize_terminal_audio():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
buffer.append(pcm_payload(8_000, speech=False), chunk_period_ms=1_000)
|
||||
|
||||
committed = buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
assert committed is None
|
||||
|
||||
|
||||
def test_commit_resets_cumulative_turn_accounting():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
buffer.append(pcm_payload(16_000), chunk_period_ms=1_000)
|
||||
buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
empty = buffer.commit(chunk_period_ms=1_000)
|
||||
|
||||
assert empty is None
|
||||
|
||||
|
||||
def test_pcm_append_reservation_rollback_restores_emitted_audio():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
original = pcm_payload(16_000)
|
||||
|
||||
reservation = buffer.prepare_append(
|
||||
original,
|
||||
operation_id="append-1",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert reservation is not None
|
||||
assert not buffer.has_pending()
|
||||
reservation.rollback()
|
||||
assert buffer.has_pending()
|
||||
|
||||
retried = buffer.flush(chunk_period_ms=1_000)
|
||||
assert retried is not None
|
||||
assert base64.b64decode(retried["audio"]) == base64.b64decode(original["audio"])
|
||||
|
||||
|
||||
def test_pcm_commit_keeps_prior_append_reservation_active():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
append_reservation = buffer.prepare_append(
|
||||
pcm_payload(16_000),
|
||||
operation_id="append-before-commit",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
commit_reservation = buffer.prepare_commit(
|
||||
operation_id="commit-after-append",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert append_reservation is not None
|
||||
assert append_reservation.active
|
||||
assert commit_reservation.payload is None
|
||||
append_reservation.commit()
|
||||
commit_reservation.commit()
|
||||
|
||||
|
||||
def test_pcm_commit_reservation_rollback_restores_residual_audio():
|
||||
buffer = MiniCPMO45PcmAppendBuffer()
|
||||
original = pcm_payload(8_000)
|
||||
assert (
|
||||
buffer.prepare_append(
|
||||
original,
|
||||
operation_id="buffer-half-chunk",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
reservation = buffer.prepare_commit(
|
||||
operation_id="final-half-chunk",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
|
||||
assert reservation.payload is not None
|
||||
assert reservation.payload["final"] is True
|
||||
reservation.rollback()
|
||||
|
||||
retried = buffer.prepare_commit(
|
||||
operation_id="retry-final-half-chunk",
|
||||
chunk_period_ms=1_000,
|
||||
)
|
||||
assert retried.payload is not None
|
||||
assert base64.b64decode(retried.payload["audio"]) == (base64.b64decode(original["audio"]) + b"\x00" * (8_000 * 4))
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.policy import (
|
||||
MiniCPMO45DuplexPolicy,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_seed_tts_initial_text_is_part_of_native_duplex_context():
|
||||
prefix, suffix = MiniCPMO45DuplexPolicy.session_context_texts(
|
||||
"Speak exactly.",
|
||||
True,
|
||||
"The quick brown fox.",
|
||||
)
|
||||
|
||||
assert prefix == "<|im_start|>system\nSpeak exactly.\n<|audio_start|>"
|
||||
assert suffix == (
|
||||
"<|audio_end|><|im_end|>\n<|im_start|>user\nThe quick brown fox.<|im_end|>\n<|im_start|>assistant\n"
|
||||
)
|
||||
Reference in new issue
Block a user