[Perf][PersonaPlex] Batch live duplex appends and replay Mimi codec frames from CUDA graphs (#8356)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-10-07 00:05:08 +00:00
1 parent e61cd4290f
commit ed6bc5ae0c
14 files changed
+1640 -116

No files matched your search

@@ -17,6 +17,7 @@ from vllm.sampling_params import SamplingParams
from tests.e2e.online_serving import personaplex_realtime_duplex as e2e_driver
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
from vllm_omni.engine.arg_utils import OmniEngineArgs
from vllm_omni.engine.duplex.config import DuplexSessionConfig
from vllm_omni.engine.duplex.contracts import DuplexFence
from vllm_omni.engine.duplex.plugin import (
@@ -26,6 +27,7 @@ from vllm_omni.engine.duplex.plugin import (
load_duplex_plugin,
validate_duplex_plugin_sampling,
)
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import PersonaPlexConfig
from vllm_omni.model_executor.models.personaplex.duplex import stage0
from vllm_omni.model_executor.models.personaplex.duplex.config import DEFAULT_PERSONA, FRAME_SIZE, SAMPLE_RATE
from vllm_omni.model_executor.models.personaplex.duplex.data_plane import PersonaPlexDataPlaneSession
@@ -106,6 +108,36 @@ def test_personaplex_deploy_is_duplex_and_propagates_capacity_to_all_model_stage
assert "personaplex_codec_max_sessions" not in deploy.connectors["connector_of_shared_memory"]["extra"]
def test_personaplex_deploy_enables_mimi_cuda_graphs_on_both_stage_configs(tmp_path: Path) -> None:
# The checkpoint ships an empty config.json; each stage's engine args build
# its PersonaPlexConfig from that plus the deploy YAML's hf_overrides.
(tmp_path / "config.json").write_text("{}")
stages = merge_pipeline_deploy(PERSONAPLEX_PIPELINE, load_deploy_config(DEPLOY_PATH))
def resolve(model_arch: str, hf_overrides: dict[str, object] | None) -> PersonaPlexConfig:
engine_args = OmniEngineArgs(
model=str(tmp_path),
model_arch=model_arch,
hf_overrides=hf_overrides,
skip_tokenizer_init=True,
trust_remote_code=True,
)
config = engine_args.create_model_config().hf_config
assert isinstance(config, PersonaPlexConfig)
return config
assert [stage.yaml_engine_args["model_arch"] for stage in stages] == [
"PersonaPlexTalkerForConditionalGeneration",
"PersonaPlexCode2Wav",
]
for stage in stages:
model_arch = stage.yaml_engine_args["model_arch"]
assert stage.yaml_engine_args["hf_overrides"] == {"mimi_cuda_graphs": True}
assert resolve(model_arch, dict(stage.yaml_engine_args["hf_overrides"])).mimi_cuda_graphs is True
# Without the deploy override the flag keeps its default.
assert resolve(model_arch, None).mimi_cuda_graphs is False
# --------------------------------------------------------------------------- #
# Session policy #
# --------------------------------------------------------------------------- #
@@ -0,0 +1,312 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Batched live-append preparation must match the per-request path exactly."""
import base64
from dataclasses import dataclass
import numpy as np
import pytest
import torch
from vllm_omni.model_executor.models.personaplex.duplex.policy import (
SILENCE_TOKENS,
SINE_TOKENS,
)
from vllm_omni.model_executor.models.personaplex.duplex.stage0 import (
PersonaPlexStage0DuplexRuntime,
)
from vllm_omni.model_executor.models.personaplex.personaplex_talker import (
PersonaPlexTalkerForConditionalGeneration,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_WIDTH = 17 # text row + 8 agent rows + 8 user rows
_PREFILL_LEN = 18 # 2 voice rows + 2 x 6 silence frames + 3 persona tokens + the live frame
class _FakeCodec:
"""Shared streaming encoder whose codes encode the row's frame count."""
def __init__(self) -> None:
self.encode_calls = 0
self.frames: list[int] = []
def streaming_init(self, batch_size: int) -> None:
self.frames = [0] * batch_size
def encode_frame(self, pcm: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
del pcm
self.encode_calls += 1
codes = torch.zeros((len(self.frames), 8), dtype=torch.long)
for row, is_active in enumerate(active.tolist()):
if is_active:
self.frames[row] += 1
codes[row] = self.frames[row] * 16 + torch.arange(8) + 1
return codes
def reset_slot(self, row: int) -> None:
self.frames[row] = 0
@dataclass(frozen=True)
class _TalkerConfig:
num_audio_codebooks: int = 16
audio_vocab_size: int = 2048
class _EmbeddingTalker:
"""Stage model whose input embedding is the token stack itself, so embeds expose the layout."""
def __init__(self) -> None:
self.device = torch.device("cpu")
self.dtype = torch.float32
self.config = _TalkerConfig()
def input_embeddings(self, stack: torch.Tensor) -> torch.Tensor:
return stack.squeeze(-1).to(torch.float32)
def _build_prefill_embed(self, tokens, offset, span, device, silence=None, user_sine=None):
del offset, silence, user_sine
values = tokens[:span].to(device=device, dtype=torch.float32)
return values[:, None].expand(-1, _WIDTH).contiguous()
_build_frame_embed = PersonaPlexTalkerForConditionalGeneration._build_frame_embed
def _depformer(text_token, hidden, *, audio_tokens, audio_provided, num_steps):
"""Deterministic stand-in whose codes depend on every input it is given."""
del hidden
assert audio_provided.dtype == torch.bool
base = audio_tokens[:, 8 : 8 + num_steps] * 7 + audio_provided[:, :num_steps].long()
return (base + text_token[:, None] + torch.arange(num_steps)) % 2048
def _talker(max_sessions: int) -> PersonaPlexTalkerForConditionalGeneration:
runtime = PersonaPlexStage0DuplexRuntime(
_EmbeddingTalker(),
model_path="/unused",
device="cpu",
codec=_FakeCodec(),
max_sessions=max_sessions,
tokenizer=lambda _text: [7, 8, 9],
voice_loader=lambda _voice: {"embeddings": torch.arange(2 * _WIDTH, dtype=torch.float32).reshape(2, _WIDTH)},
)
talker = PersonaPlexTalkerForConditionalGeneration.__new__(PersonaPlexTalkerForConditionalGeneration)
torch.nn.Module.__init__(talker)
talker._personaplex_duplex_stage0_runtime = runtime
talker._dtype = torch.float32
talker.mtp_hidden_size = _WIDTH
talker.num_active_codebooks = 8
talker.depformer = _depformer
return talker
def _append(session_id: str, seq: int, epoch: int = 0) -> dict:
pcm = np.zeros(1920, dtype="<f4")
return {
"data_plane": True,
"session_id": session_id,
"epoch": epoch,
"seq": seq,
"payload": {
"format": "pcm_f32le",
"sample_rate_hz": 24000,
"audio": base64.b64encode(pcm.tobytes()).decode("ascii"),
},
"runtime_config": {"personaplex_voice_prompt": "NATF2.pt", "personaplex_persona": "Be concise."},
}
@dataclass
class _StepResult:
embeds: dict[str, torch.Tensor]
audio_tokens: torch.Tensor
audio_provided: torch.Tensor
codes: torch.Tensor
agents: dict[tuple[str, int], torch.Tensor]
def _run_step(
talker: PersonaPlexTalkerForConditionalGeneration,
step: list[tuple[str, dict, int]],
*,
batched: bool,
text_base: int,
) -> _StepResult:
"""One scheduler step: batch preprocess, per-request preprocess, then depformer sampling.
``step`` holds ``(request_id, append, prompt_len)``; a prompt length of
``_PREFILL_LEN`` marks a first append scheduled as one full prefill span.
"""
runtime = talker._personaplex_duplex_stage0_runtime
infos = {}
spans = {}
for request_id, duplex, prompt_len in step:
spans[request_id] = prompt_len if prompt_len == _PREFILL_LEN else 1
infos[request_id] = {
"duplex": duplex,
"request_id": request_id,
"duplex_prompt_len": prompt_len,
"duplex_token_offset": prompt_len - spans[request_id],
}
if batched:
talker.preprocess_batch(req_ids=list(infos), model_intermediate_buffer=infos, device=torch.device("cpu"))
else:
runtime.encode_appends([info["duplex"] for info in infos.values()])
embeds, req_infos = {}, []
for request_id, info in infos.items():
input_ids = torch.zeros(spans[request_id], dtype=torch.long)
_, embed, update = talker.preprocess(input_ids, None, _omni_is_prefill=True, **info)
embeds[request_id] = embed
req_infos.append(update)
received = {}
def depformer(text_token, hidden, **kwargs):
received.update(kwargs)
return _depformer(text_token, hidden, **kwargs)
talker.depformer = depformer
req_ids = list(infos)
codes = talker.post_sample_talker_mtp(
input_ids=torch.arange(len(req_ids)) + text_base,
hidden_states=torch.zeros((len(req_ids), _WIDTH)),
req_ids=req_ids,
req_infos=req_infos,
)
return _StepResult(
embeds=embeds,
audio_tokens=received["audio_tokens"],
audio_provided=received["audio_provided"],
codes=codes,
agents={key: state.last_agent_codes.clone() for key, state in runtime.sessions.items()},
)
def _assert_same(batched: _StepResult, reference: _StepResult) -> None:
assert batched.embeds.keys() == reference.embeds.keys()
for request_id, embed in batched.embeds.items():
assert torch.equal(embed, reference.embeds[request_id]), request_id
assert torch.equal(batched.audio_tokens, reference.audio_tokens)
assert torch.equal(batched.audio_provided, reference.audio_provided)
assert torch.equal(batched.codes, reference.codes)
assert batched.agents.keys() == reference.agents.keys()
for key, agent in batched.agents.items():
assert torch.equal(agent, reference.agents[key]), key
def _device_prepared(talker, key: tuple[str, int]) -> bool:
state = talker._personaplex_duplex_stage0_runtime.sessions[key]
return state.prepared.depformer_audio_tokens_device is not None
def test_mixed_prefill_and_live_steps_match_the_per_request_path() -> None:
steps = [
[("a", _append("a", 1), 18), ("b", _append("b", 1), 18)],
# A first append joins a step of live appends.
[("a", _append("a", 2), 19), ("b", _append("b", 2), 19), ("c", _append("c", 1), 18)],
[("a", _append("a", 3), 20), ("b", _append("b", 3), 20), ("c", _append("c", 2), 19)],
# Only one session appends; the others keep their state.
[("c", _append("c", 3), 20)],
[("a", _append("a", 4), 21), ("c", _append("c", 4), 21)],
]
batched, reference = _talker(max_sessions=4), _talker(max_sessions=4)
for index, step in enumerate(steps):
text_base = 100 + 10 * index
result = _run_step(batched, step, batched=True, text_base=text_base)
_assert_same(result, _run_step(reference, step, batched=False, text_base=text_base))
assert _device_prepared(batched, ("a", 0)) and _device_prepared(batched, ("c", 0))
assert not _device_prepared(reference, ("a", 0))
# Batched live appends publish no host teacher-forcing copies; the per-request path still does.
host_fields = {"pplex_depformer_audio_tokens", "pplex_depformer_audio_provided"}
batched_info = batched._personaplex_duplex_stage0_runtime.sessions[("a", 0)].prepared.info_update
reference_info = reference._personaplex_duplex_stage0_runtime.sessions[("a", 0)].prepared.info_update
assert host_fields.isdisjoint(batched_info)
assert host_fields <= reference_info.keys()
runtime = batched._personaplex_duplex_stage0_runtime
assert runtime._shared_codec().encode_calls == len(steps)
assert len(runtime.sessions[("a", 0)].user_history_device) == 2
# Live embeds carry the causal user delay: cb0 from the previous frame, cb1..7 from the one before.
a4 = result.embeds["a"][0]
assert a4[9].item() == 3 * 16 + 1
assert a4[10:17].tolist() == [2 * 16 + k + 1 for k in range(1, 8)]
def test_second_append_reads_sine_for_the_missing_user_history() -> None:
talker = _talker(max_sessions=1)
def state_of():
return talker._personaplex_duplex_stage0_runtime.sessions[("a", 0)]
first = _run_step(talker, [("a", _append("a", 1), 18)], batched=True, text_base=100)
# The first append forces agent cb1..7 to silence; cb0 is sampled.
assert state_of().last_agent_codes.tolist() == [first.codes[0, 0].item(), *SILENCE_TOKENS[1:]]
result = _run_step(talker, [("a", _append("a", 2), 19)], batched=True, text_base=101)
# Live appends force only the user codebooks, so the sampled agent frame is kept as is.
assert torch.equal(state_of().last_agent_codes, result.codes[0])
embed = result.embeds["a"][0]
assert embed[0].item() == 100
assert embed[9].item() == 16 + 1
assert embed[10:17].tolist() == list(SINE_TOKENS[1:])
assert result.audio_tokens[0].tolist() == [*SILENCE_TOKENS, 2 * 16 + 1, *(16 + k + 1 for k in range(1, 8))]
assert result.audio_provided[0].tolist() == [False] * 8 + [True] * 8
def test_cancel_restart_and_recycled_slot_match_the_per_request_path() -> None:
steps = [
[("a0", _append("a", 1), 18), ("b", _append("b", 1), 18)],
[("a0", _append("a", 2), 19), ("b", _append("b", 2), 19)],
# A cancel restarts session a as epoch 1 while epoch 0's aborted request
# still has an append in this step, next to b's live append.
[("a0", _append("a", 3), 20), ("a1", _append("a", 1, epoch=1), 18), ("b", _append("b", 3), 20)],
[("a1", _append("a", 2, epoch=1), 19), ("b", _append("b", 4), 21)],
]
batched, reference = _talker(max_sessions=2), _talker(max_sessions=2)
for index, step in enumerate(steps):
text_base = 100 + 10 * index
result = _run_step(batched, step, batched=True, text_base=text_base)
_assert_same(result, _run_step(reference, step, batched=False, text_base=text_base))
runtime = batched._personaplex_duplex_stage0_runtime
assert sorted(runtime.sessions) == [("a", 1), ("b", 0)]
assert _device_prepared(batched, ("a", 1))
# The restarted epoch starts from an empty history on its recycled row.
assert result.embeds["a1"][0][10:17].tolist() == list(SINE_TOKENS[1:])
# A new session on the row of a closed one also starts from a clean history.
for talker in (batched, reference):
talker.on_requests_finished({"b"})
followups = [
[("c", _append("c", 1), 18)],
[("c", _append("c", 2), 19), ("a1", _append("a", 3, epoch=1), 20)],
]
for index, step in enumerate(followups):
text_base = 200 + 10 * index
result = _run_step(batched, step, batched=True, text_base=text_base)
_assert_same(result, _run_step(reference, step, batched=False, text_base=text_base))
assert result.embeds["c"][0][10:17].tolist() == list(SINE_TOKENS[1:])
def test_prepare_live_appends_ignores_empty_first_and_unencoded_appends() -> None:
talker = _talker(max_sessions=2)
runtime = talker._personaplex_duplex_stage0_runtime
assert runtime.prepare_live_appends([]) == 0
talker.preprocess_batch(req_ids=["x"], model_intermediate_buffer={"x": {}}, device=torch.device("cpu"))
assert runtime.sessions == {}
runtime.encode_appends([_append("a", 1)])
assert runtime.prepare_live_appends([_append("a", 1)]) == 0 # the first append keeps the prefill path
_run_step(talker, [("a", _append("a", 1), 18)], batched=False, text_base=100)
# Not encoded in this step: left to prepare_append.
assert runtime.prepare_live_appends([_append("a", 2)]) == 0
runtime.encode_appends([_append("a", 2)])
assert runtime.prepare_live_appends([_append("a", 2), _append("a", 2)]) == 1
# Already prepared: a repeat in a later step is a no-op.
assert runtime.prepare_live_appends([_append("a", 2)]) == 0
assert runtime.prepare_append(_append("a", 2), prompt_len=19, request_id="a").prompt_offset == 18
@@ -245,23 +245,23 @@ def test_repeated_append_identity_does_not_advance_codec() -> None:
def test_next_append_uses_prior_sample_and_causally_delayed_user_frame() -> None:
runtime = _runtime(_FakeCodec())
runtime.prepare_append(_duplex_info(seq=1), prompt_len=18, request_id="req")
first_agent = torch.arange(8, dtype=torch.long)
runtime.record_sample(request_id="req", text_token=torch.tensor(101), agent_codes=first_agent)
runtime.prepare_append(_duplex_info(seq=2), prompt_len=19, request_id="req")
second_agent = torch.arange(10, 18, dtype=torch.long)
runtime.record_sample(request_id="req", text_token=torch.tensor(102), agent_codes=second_agent)
runtime.prepare_append(_duplex_info(seq=3), prompt_len=20, request_id="req")
silence = torch.tensor(SILENCE_TOKENS, dtype=torch.long)
sine = torch.tensor(SINE_TOKENS, dtype=torch.long)
runtime.prepare_append(_duplex_info(seq=1), prompt_len=18, request_id="req")
# The first append teacher-forces agent cb1..7 to silence.
expected_first_effective = torch.cat([torch.tensor([0]), silence[1:]])
runtime.record_sample(request_id="req", text_token=torch.tensor(101), effective_codes=expected_first_effective)
runtime.prepare_append(_duplex_info(seq=2), prompt_len=19, request_id="req")
second_agent = torch.arange(10, 18, dtype=torch.long)
runtime.record_sample(request_id="req", text_token=torch.tensor(102), effective_codes=second_agent)
runtime.prepare_append(_duplex_info(seq=3), prompt_len=20, request_id="req")
first_call = runtime.stage_model.frame_calls[0]
assert all(torch.equal(first_call[key], silence) for key in ("last_agent", "prev_agent"))
assert all(torch.equal(first_call[key], sine) for key in ("user_d0", "user_d1"))
second_call = runtime.stage_model.frame_calls[1]
assert second_call["text_token"].tolist() == [101]
expected_first_effective = torch.cat([first_agent[:1], silence[1:]])
assert torch.equal(second_call["last_agent"], expected_first_effective)
assert torch.equal(second_call["prev_agent"], expected_first_effective)
assert torch.equal(second_call["user_d0"], torch.full((8,), 1, dtype=torch.long))
@@ -468,7 +468,7 @@ def test_cancel_overlap_full_processing_order_never_re_leases_the_aborted_epoch(
else:
restarted = runtime.prepare_append(duplex, prompt_len=prompt_len, request_id=request_id)
for request_id in ("req-e0", "req-e1"):
runtime.record_sample(request_id=request_id, text_token=5, agent_codes=list(range(8)))
runtime.record_sample(request_id=request_id, text_token=5, effective_codes=list(range(8)))
runtime.close_request("req-e0") # the engine's late finish of the aborted request
assert list(runtime.sessions) == [("session", 1)]