[Perf][PersonaPlex] Batch live duplex appends and replay Mimi codec frames from CUDA graphs (#8356)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-10-07 00:05:08 +00:00
1 parent e61cd4290f
commit ed6bc5ae0c
14 files changed
+1640 -116

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Configuration for PersonaPlex (a Moshi finetune; 2-stage audio->audio pipeline).
PersonaPlex is a staged AR speech model composed of:
@@ -203,6 +203,10 @@ class PersonaPlexConfig(PretrainedConfig):
mimi_name (`str`, *optional*):
Convenience mirror of ``mimi_config.mimi_name``; if set, it
overrides the value carried inside ``mimi_config``.
mimi_cuda_graphs (`bool`, *optional*, defaults to `False`):
Replay the streaming Mimi per-frame encode (Stage 0) and decode
(Stage 1) from model-local CUDA graphs. Can be enabled through
``hf_overrides``.
"""
model_type = "personaplex"
@@ -222,6 +226,7 @@ class PersonaPlexConfig(PretrainedConfig):
audio_vocab_size: int = 2048,
num_audio_codebooks: int = 16,
mimi_name: str | None = None,
mimi_cuda_graphs: bool = False,
**kwargs: Any,
) -> None:
if temporal_config is None:
@@ -257,6 +262,7 @@ class PersonaPlexConfig(PretrainedConfig):
if mimi_name is not None:
self.mimi_config.mimi_name = mimi_name
self.mimi_name = self.mimi_config.mimi_name
self.mimi_cuda_graphs = mimi_cuda_graphs
@staticmethod
def _coerce(value: Any, config_cls: type[PretrainedConfig]) -> PretrainedConfig:
@@ -9,9 +9,10 @@ from collections.abc import Callable
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
from typing import Any
from typing import TYPE_CHECKING, Any
import numpy as np
from vllm.logger import init_logger
from vllm_omni.model_executor.common.audio.pcm import pcm_f32le_samples
from vllm_omni.model_executor.common.duplex.payload import decode_pcm_f32le_payload
@@ -24,8 +25,13 @@ from vllm_omni.model_executor.models.personaplex.duplex.policy import (
wrap_with_system_tags,
)
if TYPE_CHECKING:
import torch
_FRAME_SAMPLES = FRAME_SIZE
logger = init_logger(__name__)
class PersonaPlexStage0CapacityError(RuntimeError):
"""Every streaming encoder row is leased by a live session."""
@@ -43,6 +49,9 @@ class PersonaPlexStage0PreparedAppend:
info_update: dict[str, Any]
prefill_applied: bool
prompt_offset: int
# Device copies of the depformer teacher-forcing state (batched path only).
depformer_audio_tokens_device: torch.Tensor | None = None
depformer_audio_provided_device: torch.Tensor | None = None
@dataclass(slots=True)
@@ -63,6 +72,10 @@ class PersonaPlexStage0SessionState:
slot: int | None = None
encoded_identity: tuple[int, int] | None = None
encoded_frame: Any | None = None
# Device copies of the encoded frame and of the last two consumed user
# frames (newest last), so live appends are built without host round trips.
encoded_frame_device: torch.Tensor | None = None
user_history_device: list[torch.Tensor] = field(default_factory=list)
def _tokenizer_path(model_path: str) -> Path:
@@ -151,6 +164,7 @@ class PersonaPlexStage0DuplexRuntime:
codec: Any | None = None,
codec_factory: Callable[[], Any] | None = None,
max_sessions: int = 1,
codec_cuda_graphs: bool = False,
tokenizer=None,
voice_loader=None,
) -> None:
@@ -161,6 +175,7 @@ class PersonaPlexStage0DuplexRuntime:
self.device = device
self.max_sessions = max_sessions
self._codec_factory = codec_factory
self._codec_cuda_graphs = codec_cuda_graphs
self._codec: Any | None = None
self._free_slots: list[int] = list(reversed(range(max_sessions)))
self._tokenizer = tokenizer
@@ -170,6 +185,7 @@ class PersonaPlexStage0DuplexRuntime:
# Requests of a superseded epoch that still reached this step; they are
# finished by the engine and must not lease a row or record a sample.
self._stale_requests: set[str] = set()
self._constants_cache: dict[str, torch.Tensor] | None = None
if codec is not None:
codec.streaming_init(max_sessions)
self._codec = codec
@@ -239,13 +255,22 @@ class PersonaPlexStage0DuplexRuntime:
state.request_ids.add(request_id)
self.request_sessions[request_id] = key
if state.prepared_identity == identity and state.prepared is not None:
return state.prepared
prepared = state.prepared
if prepared.prompt_offset < 0:
prepared.prompt_offset = int(prompt_len) - int(prepared.inputs_embeds.shape[0])
if prepared.prompt_offset < 0:
raise ValueError(
"PersonaPlex scheduler prompt reservation mismatch: "
f"reserved={prompt_len}, prepared={prepared.inputs_embeds.shape[0]}"
)
return prepared
if seq <= state.last_seq:
raise ValueError(f"PersonaPlex duplex append seq must increase: last={state.last_seq}, got={seq}")
if state.encoded_identity != identity:
self._encode_rows([(state, identity, self._decode_pcm(duplex.get("payload")))])
user_frame = state.encoded_frame
self._push_user_history(state)
state.encoded_identity = None
state.encoded_frame = None
state.user_codes = user_frame if state.user_codes is None else torch.cat([state.user_codes, user_frame], dim=0)
@@ -344,44 +369,188 @@ class PersonaPlexStage0DuplexRuntime:
f"PersonaPlex scheduler prompt reservation mismatch: reserved={prompt_len}, prepared={prepared_len}"
)
input_ids = torch.zeros((prepared_len,), dtype=torch.long, device=device)
info_update = {
info_update = self._prepared_info_update(
state, seq, silence=silence.detach().cpu(), prefill_applied=first_append
)
# This path has no device teacher-forcing copies, so the talker reads
# these host tensors from the request information.
info_update["pplex_depformer_audio_tokens"] = depformer_audio_tokens.detach().cpu()
info_update["pplex_depformer_audio_provided"] = depformer_audio_provided.detach().cpu()
return self._store_prepared(
state,
seq,
PersonaPlexStage0PreparedAppend(
input_ids=input_ids,
inputs_embeds=full_embeds,
user_codes=state.user_codes,
info_update=info_update,
prefill_applied=first_append,
prompt_offset=prompt_offset,
),
)
@staticmethod
def _prepared_info_update(
state: PersonaPlexStage0SessionState,
seq: int,
*,
silence: torch.Tensor,
prefill_applied: bool,
) -> dict[str, Any]:
"""Request-information update shared by the per-request and batched append paths."""
return {
"pplex_user_codes": state.user_codes,
"pplex_silence_codes": silence.detach().cpu(),
"pplex_depformer_audio_tokens": depformer_audio_tokens.detach().cpu(),
"pplex_depformer_audio_provided": depformer_audio_provided.detach().cpu(),
"pplex_silence_codes": silence,
"meta": {
"pplex_frame": state.prefill_slots + int(state.user_codes.shape[0]),
"pplex_prefill_len": state.prefill_slots,
},
"duplex": {
"stage0_prepared": True,
"prefill_applied": first_append,
"session_id": session_id,
"epoch": epoch,
"prefill_applied": prefill_applied,
"session_id": state.session_id,
"epoch": state.epoch,
"seq": seq,
},
}
prepared = PersonaPlexStage0PreparedAppend(
input_ids=input_ids,
inputs_embeds=full_embeds,
user_codes=state.user_codes,
info_update=info_update,
prefill_applied=first_append,
prompt_offset=prompt_offset,
)
state.prepared_identity = identity
@staticmethod
def _store_prepared(
state: PersonaPlexStage0SessionState,
seq: int,
prepared: PersonaPlexStage0PreparedAppend,
) -> PersonaPlexStage0PreparedAppend:
"""Record ``prepared`` as the session's current append and advance its sequence."""
state.prepared_identity = (state.epoch, seq)
state.prepared = prepared
state.last_seq = seq
return prepared
@staticmethod
def _push_user_history(state: PersonaPlexStage0SessionState) -> None:
"""Move the consumed encoded frame into the device history (last two frames)."""
if state.encoded_frame_device is None:
raise RuntimeError("PersonaPlex Stage 0 append has no encoded user frame")
state.user_history_device.append(state.encoded_frame_device)
del state.user_history_device[:-2]
state.encoded_frame_device = None
def _constants(self) -> dict[str, torch.Tensor]:
"""Live-append constant frames on the device (plus host silence), built once."""
if self._constants_cache is not None:
return self._constants_cache
import torch
device, _ = self._model_device_dtype()
silence = torch.tensor(SILENCE_TOKENS, dtype=torch.long)
self._constants_cache = {
"silence": silence,
"silence_device": silence.to(device),
"sine_device": torch.tensor(SINE_TOKENS, dtype=torch.long, device=device),
"live_provided_device": torch.tensor([False] * 8 + [True] * 8, dtype=torch.bool, device=device),
"zero_text_device": torch.tensor([ZERO_TEXT_TOKEN], dtype=torch.long, device=device),
}
return self._constants_cache
def prepare_live_appends(self, appends: list[dict[str, Any]]) -> int:
"""Prepare every one-frame live append of this step with one batched embedding call.
Only appends whose frame was encoded this step and that are not the
first append of their epoch (which carries the voice/persona prefill)
take this path; the rest are left to :meth:`prepare_append`. Results are
stored exactly as :meth:`prepare_append` stores them, so its per-request
call returns the cached preparation. Returns the number prepared.
"""
import torch
batch: dict[tuple[str, int], tuple[PersonaPlexStage0SessionState, int]] = {}
for duplex in appends:
try:
session_id, epoch, seq = _append_identity(duplex)
except ValueError:
continue
key = (session_id, epoch)
state = self.sessions.get(key)
# Only an append encoded in this step and not yet consumed: its seq
# is past last_seq, so it has not been prepared either.
if state is None or state.last_seq == 0 or seq <= state.last_seq or key in batch:
continue
if state.encoded_identity != (epoch, seq):
continue
batch[key] = (state, seq)
if not batch:
return 0
c = self._constants()
device, dtype = self._model_device_dtype()
sine_d, silence_d = c["sine_device"], c["silence_device"]
texts, agents, d0s, d1s, currents = [], [], [], [], []
for state, _ in batch.values():
history = state.user_history_device # consumed frames, newest last
texts.append(
state.last_text_token.reshape(1) if state.last_text_token is not None else c["zero_text_device"]
)
agents.append(state.last_agent_codes[:8] if state.last_agent_codes is not None else silence_d)
# The frame being appended is not in the history yet: user cb0 reads
# the previous frame and user cb1..7 the one before it.
d0s.append(history[-1] if len(history) >= 1 else sine_d)
d1s.append(history[-2] if len(history) >= 2 else sine_d)
currents.append(state.encoded_frame_device)
n = len(batch)
text = torch.stack(texts)
agent = torch.stack(agents)
user_d0 = torch.stack(d0s)
user_d1 = torch.stack(d1s)
current = torch.stack(currents)
stack = torch.cat([text, agent, user_d0[:, :1], user_d1[:, 1:8]], dim=1).unsqueeze(-1)
embeds = self.stage_model.input_embeddings(stack).reshape(n, 1, -1).to(dtype=dtype)
dep_tokens_device = torch.cat([silence_d.expand(n, 8), current[:, :1], user_d0[:, 1:8]], dim=1)
input_ids = torch.zeros((n, 1), dtype=torch.long, device=device)
for row, (state, seq) in enumerate(batch.values()):
user_frame = state.encoded_frame
self._push_user_history(state)
state.encoded_identity = None
state.encoded_frame = None
state.user_codes = torch.cat([state.user_codes, user_frame], dim=0)
# The talker reads teacher forcing from the device copies below, so
# no host tokens or mask are built for these appends.
self._store_prepared(
state,
seq,
PersonaPlexStage0PreparedAppend(
input_ids=input_ids[row],
inputs_embeds=embeds[row],
user_codes=state.user_codes,
info_update=self._prepared_info_update(state, seq, silence=c["silence"], prefill_applied=False),
prefill_applied=False,
# Resolved by prepare_append from the scheduler's prompt length.
prompt_offset=-1,
depformer_audio_tokens_device=dep_tokens_device[row],
depformer_audio_provided_device=c["live_provided_device"],
),
)
return n
def prepared_depformer_state(self, request_id: str) -> tuple[torch.Tensor, torch.Tensor] | None:
"""Device teacher-forcing tensors of the request's current append, when prepared on the device."""
key = self.request_sessions.get(request_id)
state = self.sessions.get(key) if key is not None else None
prepared = state.prepared if state is not None else None
if prepared is None or prepared.depformer_audio_tokens_device is None:
return None
return prepared.depformer_audio_tokens_device, prepared.depformer_audio_provided_device
def record_sample(
self,
*,
request_id: str,
text_token: Any,
agent_codes: Any,
effective_codes: Any,
) -> None:
"""Commit one sampled temporal frame for the next live append."""
"""Commit one sampled temporal frame for the next live append.
``effective_codes`` are the frame's eight agent codes with the append's
depformer teacher forcing already applied, which the next append reads.
"""
import torch
if request_id in self._stale_requests:
@@ -396,26 +565,12 @@ class PersonaPlexStage0DuplexRuntime:
return
text = torch.as_tensor(text_token, device=self._model_device_dtype()[0], dtype=torch.long).reshape(-1)
codes = torch.as_tensor(agent_codes, device=text.device, dtype=torch.long).reshape(-1)
codes = torch.as_tensor(effective_codes, device=text.device, dtype=torch.long).reshape(-1)
if text.numel() != 1:
raise ValueError(f"PersonaPlex Stage 0 expected one sampled text token, got {text.numel()}")
if codes.numel() < 8:
raise ValueError(f"PersonaPlex Stage 0 expected at least 8 agent codes, got {codes.numel()}")
effective_codes = codes[:8].clone()
prepared = state.prepared
if prepared is None:
raise RuntimeError(f"PersonaPlex Stage 0 request has no prepared payload: {request_id}")
target = prepared.info_update.get("pplex_depformer_audio_tokens")
provided = prepared.info_update.get("pplex_depformer_audio_provided")
if not isinstance(target, torch.Tensor) or not isinstance(provided, torch.Tensor):
raise RuntimeError("PersonaPlex Stage 0 prepared payload has no depformer teacher-forcing state")
target = target.reshape(-1).to(device=effective_codes.device, dtype=torch.long)
provided = provided.reshape(-1).to(device=effective_codes.device, dtype=torch.bool)
if target.numel() < 8 or provided.numel() < 8:
raise RuntimeError("PersonaPlex Stage 0 depformer teacher-forcing state is shorter than one agent frame")
effective_codes = torch.where(provided[:8], target[:8], effective_codes)
state.last_agent_codes = effective_codes.detach()
if codes.numel() != 8:
raise ValueError(f"PersonaPlex Stage 0 expected 8 effective agent codes, got {codes.numel()}")
state.last_agent_codes = codes.detach()
state.last_text_token = text.detach()
state.sampled_identity = state.prepared_identity
@@ -482,12 +637,14 @@ class PersonaPlexStage0DuplexRuntime:
pcm[state.slot] = torch.from_numpy(samples)
active[state.slot] = True
encoded = codec.encode_frame(pcm, active)
codes_device = encoded.detach()[:, :8].to(dtype=torch.long)
# One device-to-host copy per step, whatever the number of sessions.
codes = encoded.detach().to(dtype=torch.long, device="cpu")
if codes.shape[-1] < 8:
raise RuntimeError(f"PersonaPlex Mimi encoder returned {codes.shape[-1]} codebooks, expected at least 8")
for state, identity, _ in rows:
state.encoded_frame = codes[state.slot : state.slot + 1, :8].clone()
state.encoded_frame_device = codes_device[state.slot]
state.encoded_identity = identity
def _shared_codec(self):
@@ -506,6 +663,15 @@ class PersonaPlexStage0DuplexRuntime:
device=self.device,
)
codec.streaming_init(self.max_sessions)
if self._codec_cuda_graphs:
# Rows are leased and recycled in place, so one graph at
# max_sessions with the active mask as input serves every step.
captured = codec.capture_cuda_graphs(encode=True, decode_frame_counts=())
logger.info(
"PersonaPlex Stage 0 Mimi encoder %s at %d rows",
"replays a CUDA graph" if captured else "runs eagerly",
self.max_sessions,
)
self._codec = codec
return codec
@@ -406,6 +406,28 @@ class PersonaPlexCode2Wav(nn.Module):
).eval()
for _ in range(self._max_codec_sessions)
]
if getattr(self.config, "mimi_cuda_graphs", False):
# Each decoder is leased per request and reset in place, so its B=1
# graphs stay valid. Deltas arrive as the first single frame and then
# full chunks, so those two frame counts are recorded; other counts
# run eagerly. The graphs replay one at a time on this stage's stream
# and copy their output, so they share one pool.
pool = torch.cuda.graph_pool_handle() if torch.device(str(device)).type == "cuda" else None
captured = 0
for codec in codecs:
codec.streaming_init(1)
captured += bool(
codec.capture_cuda_graphs(
encode=False,
decode_frame_counts=(1, _MIMI_DECODE_BATCH_FRAMES),
pool=pool,
)
)
logger.info(
"PersonaPlex Code2Wav replays Mimi decode from CUDA graphs on %d/%d stream(s)",
captured,
len(codecs),
)
self._set_mimi_codecs(codecs)
self._mimi_device = torch.device(str(device))
reported_sr = getattr(codecs[0].model.config, "sampling_rate", None)
@@ -29,9 +29,12 @@ codec composes with elastic slot recycling in batched duplex serving.
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm_omni.model_executor.models.personaplex.personaplex_temporal import (
_apply_rope,
@@ -42,13 +45,17 @@ DEFAULT_HF_REPO = "kyutai/mimi"
FRAME_SIZE = 1920
CODEBOOKS = 8
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm_omni.model_executor.models.personaplex.personaplex_mimi_cudagraph import MimiFrameGraph
def _normalize_active(active: torch.Tensor | None, all_active: torch.Tensor) -> torch.Tensor:
# Stage 0 and Code2Wav currently use one B=1 codec per session and omit
# `active`, so preserve that API by treating None as all rows active. When
# they switch to shared B>1 codecs, their batch builders must pass bool[B]:
# Stage 0 shares one codec with a row per session slot and passes bool[B]:
# True advances that row's streaming state; False keeps an absent or padded
# row's offsets and convolution carries unchanged.
# row's offsets and convolution carries unchanged. Code2Wav leases one B=1
# codec per request and omits `active`, which means all rows are active.
if active is None:
return all_active
if active.shape != all_active.shape:
@@ -107,6 +114,10 @@ class _StreamConv1d:
``pad_mode`` controls the stream-start left padding: SEANet convs use zeros
(``constant``); the down/upsample resamplers use ``replicate`` (the first
real sample), marked per row so elastic slot recycling re-primes correctly.
The carry and the fresh-row flags are allocated once in ``reset`` and only
ever updated in place, so a CUDA graph recorded over ``__call__`` keeps
reading and writing the live state across frames, row resets and sessions.
"""
def __init__(self, conv: nn.Conv1d, pad_mode: str = "constant") -> None:
@@ -122,29 +133,36 @@ class _StreamConv1d:
self.prev = torch.zeros(batch_size, self.conv.in_channels, pad, device=device, dtype=dtype)
self._fresh = torch.ones(batch_size, dtype=torch.bool, device=device)
def reset_all(self) -> None:
self.prev.zero_()
self._fresh.fill_(True)
def reset_slot(self, b: int) -> None:
self.prev[b].zero_()
self._fresh[b] = True
def __call__(self, x: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
active_view = active.view(-1, 1, 1)
if self.pad_mode == "replicate":
pad = self.prev.shape[-1]
edge = x[..., 0:1].expand(-1, -1, pad)
fresh = (self._fresh & active).view(-1, 1, 1)
self.prev = torch.where(fresh, edge.to(self.prev.dtype), self.prev)
self._fresh[active] = False
self.prev.copy_(torch.where(fresh, edge.to(self.prev.dtype), self.prev))
self._fresh.logical_and_(~active)
x = torch.cat([self.prev, x], dim=-1)
t = x.shape[-1]
num_frames = max(0, (t - self.kernel) // self.stride + 1)
prev = x[..., num_frames * self.stride :]
self.prev = torch.where(active.view(-1, 1, 1), prev, self.prev)
self.prev.copy_(torch.where(active_view, x[..., num_frames * self.stride :], self.prev))
if num_frames == 0:
return x.new_zeros(x.shape[0], self.conv.out_channels, 0)
return self.conv(x[..., : (num_frames - 1) * self.stride + self.kernel])
class _StreamConvTr1d:
"""Moshi ``RawStreamingConvTranspose1d``: overlap-add tail carry per call."""
"""Moshi ``RawStreamingConvTranspose1d``: overlap-add tail carry per call.
Like ``_StreamConv1d``, the tail carry is updated in place only.
"""
def __init__(self, conv: nn.ConvTranspose1d) -> None:
self.conv = conv
@@ -159,6 +177,10 @@ class _StreamConvTr1d:
)
self._fresh = torch.ones(batch_size, dtype=torch.bool, device=device)
def reset_all(self) -> None:
self.partial.zero_()
self._fresh.fill_(True)
def reset_slot(self, b: int) -> None:
self.partial[b].zero_()
self._fresh[b] = True
@@ -168,17 +190,17 @@ class _StreamConvTr1d:
length = out.shape[-1]
tail = self.kernel - self.stride
pt = self.partial.shape[-1]
merge = self.partial.clone()
merge = self.partial
if self.conv.bias is not None:
# The carried tail already includes the bias; the fresh output adds
# it again, so subtract one copy -- except on a row's very first
# frame, where the carry is zeros by construction.
merge = merge - self.conv.bias[:, None]
merge[self._fresh & active] = 0.0
self._fresh[active] = False
first = (self._fresh & active).view(-1, 1, 1)
merge = torch.where(first, torch.zeros_like(merge), merge)
self._fresh.logical_and_(~active)
out[..., :pt] += merge
partial = out[..., length - tail :].clone()
self.partial = torch.where(active.view(-1, 1, 1), partial, self.partial)
self.partial.copy_(torch.where(active.view(-1, 1, 1), out[..., length - tail :], self.partial))
return out[..., : length - tail]
@@ -358,13 +380,16 @@ class PersonaPlexMimiCodec(nn.Module):
n_dec = self.decoder_transformer.load_weights(sd, "decoder_transformer")
assert n_enc == n_dec == 80, (n_enc, n_dec)
del sd
self._init_streaming_stages()
def _init_streaming_stages(self) -> None:
m = self.model
self._enc_stages = _walk_seanet(m.encoder.layers)
self._downsample = _StreamConv1d(m.downsample.conv, pad_mode="replicate")
self._upsample = _StreamConvTr1d(m.upsample.conv)
self._dec_stages = _walk_seanet(m.decoder.layers)
self._batch_size: int | None = None
self._cuda_graphs: dict[str, MimiFrameGraph] = {}
self._all_active: torch.Tensor
# -- streaming state ------------------------------------------------------
@@ -380,7 +405,22 @@ class PersonaPlexMimiCodec(nn.Module):
yield self._upsample
def streaming_init(self, batch_size: int) -> None:
# Re-initializing at the same batch size resets in place: the state
# tensors keep their storage, so captured frame graphs stay valid.
if getattr(self, "_batch_size", None) == batch_size:
self.reset_streaming()
return
if getattr(self, "_cuda_graphs", None):
# The state is reallocated below, so graphs recorded at the old
# batch size would read stale tensors; drop them and say so.
logger.warning(
"PersonaPlex Mimi stream resized from %s to %d rows; dropping its CUDA graphs (%s)",
self._batch_size,
batch_size,
"/".join(sorted(self._cuda_graphs)),
)
self._batch_size = batch_size
self._cuda_graphs = {}
self._all_active = torch.ones(batch_size, dtype=torch.bool, device=self.device)
for s in self._conv_states():
s.reset(batch_size, self.device, self.dtype)
@@ -390,8 +430,7 @@ class PersonaPlexMimiCodec(nn.Module):
def reset_streaming(self) -> None:
assert self._batch_size is not None
for state in self._conv_states():
for b in range(self._batch_size):
state.reset_slot(b)
state.reset_all()
self.encoder_transformer.reset_streaming()
self.decoder_transformer.reset_streaming()
@@ -401,6 +440,42 @@ class PersonaPlexMimiCodec(nn.Module):
self.encoder_transformer.reset_slot(b)
self.decoder_transformer.reset_slot(b)
def capture_cuda_graphs(
self,
*,
encode: bool = True,
decode_frame_counts: tuple[int, ...] = (1,),
warmup_iters: int = 2,
pool: tuple[int, int] | None = None,
) -> list[str]:
"""Record the per-frame codec steps at the streaming batch size.
``encode`` records ``encode_frame``; each ``F`` in ``decode_frame_counts``
records ``decode_frames`` over ``F`` frames (``decode_frame`` is ``F=1``).
Every row is recorded active; the ``active`` mask is a graph input, so
one graph serves any subset of live rows. Warmup frames run through the
real streaming state, which is then reset in place, so the codec is
fresh afterwards. ``pool`` is a CUDA graph memory pool to share with
other codecs replayed on the same stream; by default the codec gets a
private one. Returns the captured graph names; on a non-CUDA device the
codec stays eager, and so it does after a recoverable capture failure,
which is logged with its traceback. Warmup, device execution and reset
failures propagate instead of leaving an apparently usable codec.
"""
from vllm_omni.model_executor.models.personaplex.personaplex_mimi_cudagraph import (
capture_mimi_frame_graphs,
)
assert self._batch_size is not None, "call streaming_init first"
self._cuda_graphs = capture_mimi_frame_graphs(
self,
encode=encode,
decode_frame_counts=decode_frame_counts,
warmup_iters=warmup_iters,
pool=pool,
)
return sorted(self._cuda_graphs)
# -- per-frame codec -------------------------------------------------------
@staticmethod
@@ -413,32 +488,68 @@ class PersonaPlexMimiCodec(nn.Module):
x = stage(x, active) if kind in {"conv", "convtr"} else stage(x)
return x
def _frame_graph(self, name: str) -> MimiFrameGraph | None:
graph = self._cuda_graphs.get(name)
if graph is None or torch.cuda.is_current_stream_capturing():
return None
return graph
@torch.no_grad()
def encode_frame(self, pcm: torch.Tensor, active: torch.Tensor | None = None) -> torch.Tensor:
"""``[B, frame_size]`` float PCM -> ``[B, 8]`` codes."""
x = pcm.to(self.device, self.dtype).view(-1, 1, FRAME_SIZE)
active = _normalize_active(active, self._all_active)
graph = self._frame_graph("encode")
if graph is not None:
return graph.replay(pcm.reshape(-1, FRAME_SIZE), active)
return self._encode_frame_eager(pcm, active)
def _encode_frame_eager(self, pcm: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
x = pcm.to(self.device, self.dtype).view(-1, 1, FRAME_SIZE)
x = self._run_stages(x, self._enc_stages, active)
x = self.encoder_transformer.step(x.transpose(1, 2), active).transpose(1, 2)
x = self._downsample(x, active)
codes = self.model.quantizer.encode(x, num_quantizers=CODEBOOKS) # [Q, B, T]
return codes[:CODEBOOKS, :, 0].transpose(0, 1).contiguous()
def _quantizer_decode(self, codes: torch.Tensor) -> torch.Tensor:
"""``MimiSplitResidualVectorQuantizer.decode`` without its host-side scalar.
The Hugging Face RVQ decode seeds its sum with ``torch.tensor(0.0)``, a
host-to-device copy that cannot be recorded in a CUDA graph. This
seeds it with an on-device zero instead; the arithmetic is unchanged.
"""
quantizer = self.model.quantizer
split = quantizer.num_semantic_quantizers
def rvq_decode(rvq, rvq_codes: torch.Tensor) -> torch.Tensor:
out = torch.zeros((), device=rvq_codes.device)
for layer, indices in zip(rvq.layers, rvq_codes.transpose(0, 1)):
out = out + layer.decode(indices)
if rvq.output_proj is not None:
out = rvq.output_proj(out)
return out
out = rvq_decode(quantizer.semantic_residual_vector_quantizer, codes[:, :split])
if codes.shape[1] > split:
out += rvq_decode(quantizer.acoustic_residual_vector_quantizer, codes[:, split:])
return out
@torch.no_grad()
def decode_frame(self, codes: torch.Tensor, active: torch.Tensor | None = None) -> torch.Tensor:
"""``[B, 8]`` codes -> ``[B, frame_size]`` float PCM."""
emb = self.model.quantizer.decode(codes.to(self.device).view(-1, CODEBOOKS, 1))
active = _normalize_active(active, self._all_active)
emb = self._upsample(emb, active)
emb = self.decoder_transformer.step(emb.transpose(1, 2), active).transpose(1, 2)
x = self._run_stages(emb, self._dec_stages, active)
return x[:, 0, :]
return self.decode_frames(codes.reshape(-1, CODEBOOKS, 1), active)
@torch.no_grad()
def decode_frames(self, codes: torch.Tensor, active: torch.Tensor | None = None) -> torch.Tensor:
"""``[B, 8, F]`` codes -> ``[B, F * frame_size]`` float PCM."""
emb = self.model.quantizer.decode(codes.to(self.device))
active = _normalize_active(active, self._all_active)
graph = self._frame_graph(f"decode_f{codes.shape[-1]}")
if graph is not None:
return graph.replay(codes, active)
return self._decode_frames_eager(codes, active)
def _decode_frames_eager(self, codes: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
emb = self._quantizer_decode(codes.to(self.device))
emb = self._upsample(emb, active)
emb = self.decoder_transformer.step(emb.transpose(1, 2), active).transpose(1, 2)
x = self._run_stages(emb, self._dec_stages, active)
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Replay the PersonaPlex streaming Mimi frame steps from CUDA graphs.
One 80 ms Mimi encode or decode frame is a few hundred small kernels (SEANet
convs, an 8-layer streaming transformer and the RVQ), so eager execution is
launch bound. Every streaming state tensor of ``PersonaPlexMimiCodec`` (conv
carries, fresh-row flags, ring KV caches and offsets) is allocated once by
``streaming_init`` and only updated in place, and row resets are in place too,
so a graph recorded at the streaming batch size stays valid across frames,
session slot recycling and full stream resets.
The ``active`` row mask is a graph input: one graph at the shared batch size
serves any subset of live rows, and inactive rows keep their state exactly as
in eager execution.
"""
from __future__ import annotations
import traceback
from collections.abc import Callable
from contextlib import ExitStack
from typing import TYPE_CHECKING
import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm_omni.model_executor.models.personaplex.personaplex_mimi import PersonaPlexMimiCodec
logger = init_logger(__name__)
__all__ = ["MimiFrameGraph", "capture_mimi_frame_graphs"]
def _is_recoverable_capture_error(error: RuntimeError) -> bool:
"""Allow only known capture limitations, never an unexplained CUDA error."""
pending: list[BaseException] = [error]
seen: set[int] = set()
known_failure = False
while pending:
current = pending.pop()
if id(current) in seen:
continue
seen.add(id(current))
message = str(current).split("\n", 1)[0]
if isinstance(current, torch.cuda.OutOfMemoryError) or (
isinstance(current, RuntimeError)
and (
message.startswith("CUDA out of memory.")
or message
in {
"CUDA error: out of memory",
"CUDA error: operation not permitted when stream is capturing",
}
)
):
known_failure = True
elif not (
isinstance(current, RuntimeError)
and message == "CUDA error: operation failed due to a previous error during capture"
):
return False
# capture_end can replace a body exception with capture-invalidated.
# The original failure must also be recoverable, even if suppressed.
pending.extend(exc for exc in (current.__cause__, current.__context__) if exc is not None)
return known_failure
def _capture_error_details(error: RuntimeError) -> str:
"""Preserve diagnostics without retaining capture frames or graph pools."""
details = "".join(traceback.format_exception(error))
pending: list[BaseException] = [error]
seen: set[int] = set()
while pending:
current = pending.pop()
if id(current) in seen:
continue
seen.add(id(current))
if current.__traceback__ is not None:
# ExitStack frames can retain their own traceback in exception tuples.
traceback.clear_frames(current.__traceback__)
current.__traceback__ = None
pending.extend(exc for exc in (current.__cause__, current.__context__) if exc is not None)
return details
class MimiFrameGraph:
"""A recorded ``(frame, active) -> output`` codec step with static buffers."""
def __init__(
self,
graph: torch.cuda.CUDAGraph,
static_input: torch.Tensor,
static_active: torch.Tensor,
static_output: torch.Tensor,
eager: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],
) -> None:
self.graph = graph
self.static_input = static_input
self.static_active = static_active
self.static_output = static_output
self.eager = eager
self.replays = 0
self._warned = False
def replay(self, frame: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
if frame.shape != self.static_input.shape or active.shape != self.static_active.shape:
if not self._warned:
self._warned = True
logger.warning(
"PersonaPlex Mimi graph input shape %s does not match capture shape %s; running eagerly",
tuple(frame.shape),
tuple(self.static_input.shape),
)
return self.eager(frame, active)
self.static_input.copy_(frame)
self.static_active.copy_(active)
self.graph.replay()
self.replays += 1
# The static output is overwritten by the next replay.
return self.static_output.clone()
def capture_mimi_frame_graphs(
codec: PersonaPlexMimiCodec,
*,
encode: bool,
decode_frame_counts: tuple[int, ...],
warmup_iters: int = 2,
pool: tuple[int, int] | None = None,
) -> dict[str, MimiFrameGraph]:
"""Capture the codec's encode and ``F``-frame decode steps.
Returns an empty dict off CUDA or after a recoverable capture failure.
Warmup, device execution and stream-reset failures always propagate.
"""
from vllm_omni.model_executor.models.personaplex.personaplex_mimi import CODEBOOKS, FRAME_SIZE
device = torch.device(codec.device)
if device.type != "cuda" or not torch.cuda.is_available():
logger.info("PersonaPlex Mimi CUDA graphs skipped on device %s", device)
return {}
batch_size = codec._batch_size
specs: list[tuple[str, Callable, torch.Tensor]] = []
if encode:
specs.append(
(
"encode",
codec._encode_frame_eager,
torch.zeros(batch_size, FRAME_SIZE, device=device, dtype=codec.dtype),
)
)
for frames in sorted(set(decode_frame_counts)):
specs.append(
(
f"decode_f{frames}",
codec._decode_frames_eager,
torch.zeros(batch_size, CODEBOOKS, frames, device=device, dtype=torch.long),
)
)
# A private pool by default: these graphs are captured outside the model
# runner's own capture sequence, so they must not reuse its pool blocks.
if pool is None:
pool = torch.cuda.graph_pool_handle()
graphs: dict[str, MimiFrameGraph] = {}
warmed_specs: list[tuple[str, Callable, torch.Tensor, torch.Tensor]] = []
capture_error_details: str | None = None
try:
with torch.no_grad():
# Validate every eager step before capture can select the fallback.
for name, eager, static_input in specs:
static_active = torch.ones(batch_size, dtype=torch.bool, device=device)
for _ in range(max(warmup_iters, 1)):
eager(static_input, static_active)
torch.accelerator.synchronize(device)
warmed_specs.append((name, eager, static_input, static_active))
for name, eager, static_input, static_active in warmed_specs:
# capture_end may raise before torch.cuda.graph restores its
# stream. Always restore our caller's stream on that path.
with torch.cuda.stream(torch.cuda.current_stream(device)):
graph = torch.cuda.CUDAGraph()
# __enter__ synchronizes and clears the CUDA cache. A
# failure there is a setup failure, not a capture fallback.
capture = torch.cuda.graph(graph, pool=pool, capture_error_mode="thread_local")
capture.__enter__()
try:
# Other worker threads may issue CUDA calls while this records.
with ExitStack() as stack:
stack.push(capture)
static_output = eager(static_input, static_active)
except RuntimeError as error:
if not _is_recoverable_capture_error(error):
raise
capture_error_details = _capture_error_details(error)
graphs.clear()
# Release partial graphs and graph-owned output tensors
# before reset, including the last loop iteration.
graph = static_output = capture = None
break
# A device execution failure is not an optional capture failure.
torch.accelerator.synchronize(device)
graphs[name] = MimiFrameGraph(graph, static_input, static_active, static_output, eager)
finally:
# Warmup frames advanced the real streaming state; restore a fresh
# stream without reallocating the tensors the graphs point at.
codec.reset_streaming()
# Reset enqueues device work too. Do not report usable graphs or an eager
# fallback unless that work completed successfully.
torch.accelerator.synchronize(device)
if capture_error_details is not None:
logger.warning(
"PersonaPlex Mimi CUDA graphs were requested (%s at batch size %d) but capture failed; "
"the codec runs eagerly, which lowers realtime session capacity\n%s",
"/".join(name for name, _, _ in specs),
batch_size,
capture_error_details,
)
if graphs:
logger.debug("Captured PersonaPlex Mimi %s graph(s) at batch size %d", "/".join(graphs), batch_size)
return graphs
@@ -30,7 +30,7 @@ encode of the input WAV (built in ``preprocess``); live duplex is Phase 2.
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
from typing import TYPE_CHECKING, Any
import torch
from torch import nn
@@ -54,6 +54,11 @@ from vllm_omni.model_executor.models.personaplex.personaplex_embeddings import (
PersonaPlexInputEmbeddings,
)
if TYPE_CHECKING:
from vllm_omni.model_executor.models.personaplex.duplex.stage0 import (
PersonaPlexStage0DuplexRuntime,
)
__all__ = ["PersonaPlexTalkerForConditionalGeneration"]
@@ -437,6 +442,7 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
model_path=model_path,
device=device,
max_sessions=int(getattr(self.vllm_config.model_config, "duplex_max_sessions", 1)),
codec_cuda_graphs=bool(getattr(self.config, "mimi_cuda_graphs", False)),
)
self._personaplex_duplex_stage0_runtime = runtime
return runtime
@@ -485,7 +491,9 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
if isinstance(duplex, dict) and duplex.get("data_plane") is True:
appends.append(duplex)
if appends:
self._duplex_stage0_runtime().encode_appends(appends)
runtime = self._duplex_stage0_runtime()
runtime.encode_appends(appends)
runtime.prepare_live_appends(appends)
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None)
@@ -551,9 +559,48 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
)
text_token = input_ids.reshape(bsz).to(torch.long)
hidden = hidden_states.reshape(bsz, 1, -1).to(self._dtype)
audio_tokens: list[torch.Tensor] = []
audio_provided: list[torch.Tensor] = []
for info in req_infos:
runtime = self._duplex_stage0_runtime()
audio_tokens, audio_provided = self._depformer_teacher_forcing(runtime, req_ids, req_infos, hidden.device)
codes = self.depformer(
text_token,
hidden,
audio_tokens=audio_tokens,
audio_provided=audio_provided,
num_steps=self.num_active_codebooks,
).to(torch.long)
# The next append reads the agent frame with the teacher-forced
# codebooks applied; merge it once for the whole batch.
effective = torch.where(audio_provided[:, :8], audio_tokens[:, :8], codes[:, :8])
for row, request_id in enumerate(req_ids):
runtime.record_sample(
request_id=request_id,
text_token=text_token[row],
effective_codes=effective[row],
)
return codes
@staticmethod
def _depformer_teacher_forcing(
runtime: PersonaPlexStage0DuplexRuntime,
req_ids: list[str],
req_infos: list[dict[str, Any]],
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Stack each request's depformer teacher-forcing tokens and mask on ``device``.
Live appends batched in ``preprocess_batch`` already hold device copies.
The rest (first appends, aborted epochs) carry host tensors in their
request information and are moved in one copy.
"""
tokens_rows: list[torch.Tensor] = []
provided_rows: list[torch.Tensor] = []
host_rows: list[int] = []
for row, (request_id, info) in enumerate(zip(req_ids, req_infos)):
prepared = runtime.prepared_depformer_state(request_id)
if prepared is not None:
tokens_rows.append(prepared[0])
provided_rows.append(prepared[1])
continue
tokens = info.get("pplex_depformer_audio_tokens")
provided = info.get("pplex_depformer_audio_provided")
if not isinstance(tokens, torch.Tensor) or not isinstance(provided, torch.Tensor):
@@ -565,29 +612,18 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
"PersonaPlex depformer teacher-forcing token/mask shapes differ: "
f"{tuple(tokens.shape)} != {tuple(provided.shape)}"
)
audio_tokens.append(tokens)
audio_provided.append(provided)
codes = self.depformer(
text_token,
hidden,
audio_tokens=torch.stack(audio_tokens).to(
device=hidden.device,
dtype=torch.long,
),
audio_provided=torch.stack(audio_provided).to(
device=hidden.device,
dtype=torch.bool,
),
num_steps=self.num_active_codebooks,
).to(torch.long)
runtime = self._duplex_stage0_runtime()
for row, request_id in enumerate(req_ids):
runtime.record_sample(
request_id=request_id,
text_token=text_token[row],
agent_codes=codes[row],
)
return codes
host_rows.append(row)
tokens_rows.append(tokens)
provided_rows.append(provided)
if host_rows:
host_tokens = torch.stack([tokens_rows[row] for row in host_rows]).to(device=device, dtype=torch.long)
host_provided = torch.stack([provided_rows[row] for row in host_rows]).to(device=device, dtype=torch.bool)
if len(host_rows) == len(req_ids):
return host_tokens, host_provided
for index, row in enumerate(host_rows):
tokens_rows[row] = host_tokens[index]
provided_rows[row] = host_provided[index]
return torch.stack(tokens_rows), torch.stack(provided_rows)
# ------------------------------------------------------------------
# Weight loading