[Perf][PersonaPlex] Batch live duplex appends and replay Mimi codec frames from CUDA graphs (#8356)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
1 parent
e61cd4290f
commit
ed6bc5ae0c
14 files changed
+1640
-116
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Configuration for PersonaPlex (a Moshi finetune; 2-stage audio->audio pipeline).
|
||||
|
||||
PersonaPlex is a staged AR speech model composed of:
|
||||
@@ -203,6 +203,10 @@ class PersonaPlexConfig(PretrainedConfig):
|
||||
mimi_name (`str`, *optional*):
|
||||
Convenience mirror of ``mimi_config.mimi_name``; if set, it
|
||||
overrides the value carried inside ``mimi_config``.
|
||||
mimi_cuda_graphs (`bool`, *optional*, defaults to `False`):
|
||||
Replay the streaming Mimi per-frame encode (Stage 0) and decode
|
||||
(Stage 1) from model-local CUDA graphs. Can be enabled through
|
||||
``hf_overrides``.
|
||||
"""
|
||||
|
||||
model_type = "personaplex"
|
||||
@@ -222,6 +226,7 @@ class PersonaPlexConfig(PretrainedConfig):
|
||||
audio_vocab_size: int = 2048,
|
||||
num_audio_codebooks: int = 16,
|
||||
mimi_name: str | None = None,
|
||||
mimi_cuda_graphs: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if temporal_config is None:
|
||||
@@ -257,6 +262,7 @@ class PersonaPlexConfig(PretrainedConfig):
|
||||
if mimi_name is not None:
|
||||
self.mimi_config.mimi_name = mimi_name
|
||||
self.mimi_name = self.mimi_config.mimi_name
|
||||
self.mimi_cuda_graphs = mimi_cuda_graphs
|
||||
|
||||
@staticmethod
|
||||
def _coerce(value: Any, config_cls: type[PretrainedConfig]) -> PretrainedConfig:
|
||||
|
||||
@@ -9,9 +9,10 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import numpy as np
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.common.audio.pcm import pcm_f32le_samples
|
||||
from vllm_omni.model_executor.common.duplex.payload import decode_pcm_f32le_payload
|
||||
@@ -24,8 +25,13 @@ from vllm_omni.model_executor.models.personaplex.duplex.policy import (
|
||||
wrap_with_system_tags,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
_FRAME_SAMPLES = FRAME_SIZE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PersonaPlexStage0CapacityError(RuntimeError):
|
||||
"""Every streaming encoder row is leased by a live session."""
|
||||
@@ -43,6 +49,9 @@ class PersonaPlexStage0PreparedAppend:
|
||||
info_update: dict[str, Any]
|
||||
prefill_applied: bool
|
||||
prompt_offset: int
|
||||
# Device copies of the depformer teacher-forcing state (batched path only).
|
||||
depformer_audio_tokens_device: torch.Tensor | None = None
|
||||
depformer_audio_provided_device: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -63,6 +72,10 @@ class PersonaPlexStage0SessionState:
|
||||
slot: int | None = None
|
||||
encoded_identity: tuple[int, int] | None = None
|
||||
encoded_frame: Any | None = None
|
||||
# Device copies of the encoded frame and of the last two consumed user
|
||||
# frames (newest last), so live appends are built without host round trips.
|
||||
encoded_frame_device: torch.Tensor | None = None
|
||||
user_history_device: list[torch.Tensor] = field(default_factory=list)
|
||||
|
||||
|
||||
def _tokenizer_path(model_path: str) -> Path:
|
||||
@@ -151,6 +164,7 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
codec: Any | None = None,
|
||||
codec_factory: Callable[[], Any] | None = None,
|
||||
max_sessions: int = 1,
|
||||
codec_cuda_graphs: bool = False,
|
||||
tokenizer=None,
|
||||
voice_loader=None,
|
||||
) -> None:
|
||||
@@ -161,6 +175,7 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
self.device = device
|
||||
self.max_sessions = max_sessions
|
||||
self._codec_factory = codec_factory
|
||||
self._codec_cuda_graphs = codec_cuda_graphs
|
||||
self._codec: Any | None = None
|
||||
self._free_slots: list[int] = list(reversed(range(max_sessions)))
|
||||
self._tokenizer = tokenizer
|
||||
@@ -170,6 +185,7 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
# Requests of a superseded epoch that still reached this step; they are
|
||||
# finished by the engine and must not lease a row or record a sample.
|
||||
self._stale_requests: set[str] = set()
|
||||
self._constants_cache: dict[str, torch.Tensor] | None = None
|
||||
if codec is not None:
|
||||
codec.streaming_init(max_sessions)
|
||||
self._codec = codec
|
||||
@@ -239,13 +255,22 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
state.request_ids.add(request_id)
|
||||
self.request_sessions[request_id] = key
|
||||
if state.prepared_identity == identity and state.prepared is not None:
|
||||
return state.prepared
|
||||
prepared = state.prepared
|
||||
if prepared.prompt_offset < 0:
|
||||
prepared.prompt_offset = int(prompt_len) - int(prepared.inputs_embeds.shape[0])
|
||||
if prepared.prompt_offset < 0:
|
||||
raise ValueError(
|
||||
"PersonaPlex scheduler prompt reservation mismatch: "
|
||||
f"reserved={prompt_len}, prepared={prepared.inputs_embeds.shape[0]}"
|
||||
)
|
||||
return prepared
|
||||
if seq <= state.last_seq:
|
||||
raise ValueError(f"PersonaPlex duplex append seq must increase: last={state.last_seq}, got={seq}")
|
||||
|
||||
if state.encoded_identity != identity:
|
||||
self._encode_rows([(state, identity, self._decode_pcm(duplex.get("payload")))])
|
||||
user_frame = state.encoded_frame
|
||||
self._push_user_history(state)
|
||||
state.encoded_identity = None
|
||||
state.encoded_frame = None
|
||||
state.user_codes = user_frame if state.user_codes is None else torch.cat([state.user_codes, user_frame], dim=0)
|
||||
@@ -344,44 +369,188 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
f"PersonaPlex scheduler prompt reservation mismatch: reserved={prompt_len}, prepared={prepared_len}"
|
||||
)
|
||||
input_ids = torch.zeros((prepared_len,), dtype=torch.long, device=device)
|
||||
info_update = {
|
||||
info_update = self._prepared_info_update(
|
||||
state, seq, silence=silence.detach().cpu(), prefill_applied=first_append
|
||||
)
|
||||
# This path has no device teacher-forcing copies, so the talker reads
|
||||
# these host tensors from the request information.
|
||||
info_update["pplex_depformer_audio_tokens"] = depformer_audio_tokens.detach().cpu()
|
||||
info_update["pplex_depformer_audio_provided"] = depformer_audio_provided.detach().cpu()
|
||||
return self._store_prepared(
|
||||
state,
|
||||
seq,
|
||||
PersonaPlexStage0PreparedAppend(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=full_embeds,
|
||||
user_codes=state.user_codes,
|
||||
info_update=info_update,
|
||||
prefill_applied=first_append,
|
||||
prompt_offset=prompt_offset,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _prepared_info_update(
|
||||
state: PersonaPlexStage0SessionState,
|
||||
seq: int,
|
||||
*,
|
||||
silence: torch.Tensor,
|
||||
prefill_applied: bool,
|
||||
) -> dict[str, Any]:
|
||||
"""Request-information update shared by the per-request and batched append paths."""
|
||||
return {
|
||||
"pplex_user_codes": state.user_codes,
|
||||
"pplex_silence_codes": silence.detach().cpu(),
|
||||
"pplex_depformer_audio_tokens": depformer_audio_tokens.detach().cpu(),
|
||||
"pplex_depformer_audio_provided": depformer_audio_provided.detach().cpu(),
|
||||
"pplex_silence_codes": silence,
|
||||
"meta": {
|
||||
"pplex_frame": state.prefill_slots + int(state.user_codes.shape[0]),
|
||||
"pplex_prefill_len": state.prefill_slots,
|
||||
},
|
||||
"duplex": {
|
||||
"stage0_prepared": True,
|
||||
"prefill_applied": first_append,
|
||||
"session_id": session_id,
|
||||
"epoch": epoch,
|
||||
"prefill_applied": prefill_applied,
|
||||
"session_id": state.session_id,
|
||||
"epoch": state.epoch,
|
||||
"seq": seq,
|
||||
},
|
||||
}
|
||||
prepared = PersonaPlexStage0PreparedAppend(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=full_embeds,
|
||||
user_codes=state.user_codes,
|
||||
info_update=info_update,
|
||||
prefill_applied=first_append,
|
||||
prompt_offset=prompt_offset,
|
||||
)
|
||||
state.prepared_identity = identity
|
||||
|
||||
@staticmethod
|
||||
def _store_prepared(
|
||||
state: PersonaPlexStage0SessionState,
|
||||
seq: int,
|
||||
prepared: PersonaPlexStage0PreparedAppend,
|
||||
) -> PersonaPlexStage0PreparedAppend:
|
||||
"""Record ``prepared`` as the session's current append and advance its sequence."""
|
||||
state.prepared_identity = (state.epoch, seq)
|
||||
state.prepared = prepared
|
||||
state.last_seq = seq
|
||||
return prepared
|
||||
|
||||
@staticmethod
|
||||
def _push_user_history(state: PersonaPlexStage0SessionState) -> None:
|
||||
"""Move the consumed encoded frame into the device history (last two frames)."""
|
||||
if state.encoded_frame_device is None:
|
||||
raise RuntimeError("PersonaPlex Stage 0 append has no encoded user frame")
|
||||
state.user_history_device.append(state.encoded_frame_device)
|
||||
del state.user_history_device[:-2]
|
||||
state.encoded_frame_device = None
|
||||
|
||||
def _constants(self) -> dict[str, torch.Tensor]:
|
||||
"""Live-append constant frames on the device (plus host silence), built once."""
|
||||
if self._constants_cache is not None:
|
||||
return self._constants_cache
|
||||
import torch
|
||||
|
||||
device, _ = self._model_device_dtype()
|
||||
silence = torch.tensor(SILENCE_TOKENS, dtype=torch.long)
|
||||
self._constants_cache = {
|
||||
"silence": silence,
|
||||
"silence_device": silence.to(device),
|
||||
"sine_device": torch.tensor(SINE_TOKENS, dtype=torch.long, device=device),
|
||||
"live_provided_device": torch.tensor([False] * 8 + [True] * 8, dtype=torch.bool, device=device),
|
||||
"zero_text_device": torch.tensor([ZERO_TEXT_TOKEN], dtype=torch.long, device=device),
|
||||
}
|
||||
return self._constants_cache
|
||||
|
||||
def prepare_live_appends(self, appends: list[dict[str, Any]]) -> int:
|
||||
"""Prepare every one-frame live append of this step with one batched embedding call.
|
||||
|
||||
Only appends whose frame was encoded this step and that are not the
|
||||
first append of their epoch (which carries the voice/persona prefill)
|
||||
take this path; the rest are left to :meth:`prepare_append`. Results are
|
||||
stored exactly as :meth:`prepare_append` stores them, so its per-request
|
||||
call returns the cached preparation. Returns the number prepared.
|
||||
"""
|
||||
import torch
|
||||
|
||||
batch: dict[tuple[str, int], tuple[PersonaPlexStage0SessionState, int]] = {}
|
||||
for duplex in appends:
|
||||
try:
|
||||
session_id, epoch, seq = _append_identity(duplex)
|
||||
except ValueError:
|
||||
continue
|
||||
key = (session_id, epoch)
|
||||
state = self.sessions.get(key)
|
||||
# Only an append encoded in this step and not yet consumed: its seq
|
||||
# is past last_seq, so it has not been prepared either.
|
||||
if state is None or state.last_seq == 0 or seq <= state.last_seq or key in batch:
|
||||
continue
|
||||
if state.encoded_identity != (epoch, seq):
|
||||
continue
|
||||
batch[key] = (state, seq)
|
||||
if not batch:
|
||||
return 0
|
||||
c = self._constants()
|
||||
device, dtype = self._model_device_dtype()
|
||||
sine_d, silence_d = c["sine_device"], c["silence_device"]
|
||||
texts, agents, d0s, d1s, currents = [], [], [], [], []
|
||||
for state, _ in batch.values():
|
||||
history = state.user_history_device # consumed frames, newest last
|
||||
texts.append(
|
||||
state.last_text_token.reshape(1) if state.last_text_token is not None else c["zero_text_device"]
|
||||
)
|
||||
agents.append(state.last_agent_codes[:8] if state.last_agent_codes is not None else silence_d)
|
||||
# The frame being appended is not in the history yet: user cb0 reads
|
||||
# the previous frame and user cb1..7 the one before it.
|
||||
d0s.append(history[-1] if len(history) >= 1 else sine_d)
|
||||
d1s.append(history[-2] if len(history) >= 2 else sine_d)
|
||||
currents.append(state.encoded_frame_device)
|
||||
n = len(batch)
|
||||
text = torch.stack(texts)
|
||||
agent = torch.stack(agents)
|
||||
user_d0 = torch.stack(d0s)
|
||||
user_d1 = torch.stack(d1s)
|
||||
current = torch.stack(currents)
|
||||
stack = torch.cat([text, agent, user_d0[:, :1], user_d1[:, 1:8]], dim=1).unsqueeze(-1)
|
||||
embeds = self.stage_model.input_embeddings(stack).reshape(n, 1, -1).to(dtype=dtype)
|
||||
dep_tokens_device = torch.cat([silence_d.expand(n, 8), current[:, :1], user_d0[:, 1:8]], dim=1)
|
||||
input_ids = torch.zeros((n, 1), dtype=torch.long, device=device)
|
||||
for row, (state, seq) in enumerate(batch.values()):
|
||||
user_frame = state.encoded_frame
|
||||
self._push_user_history(state)
|
||||
state.encoded_identity = None
|
||||
state.encoded_frame = None
|
||||
state.user_codes = torch.cat([state.user_codes, user_frame], dim=0)
|
||||
# The talker reads teacher forcing from the device copies below, so
|
||||
# no host tokens or mask are built for these appends.
|
||||
self._store_prepared(
|
||||
state,
|
||||
seq,
|
||||
PersonaPlexStage0PreparedAppend(
|
||||
input_ids=input_ids[row],
|
||||
inputs_embeds=embeds[row],
|
||||
user_codes=state.user_codes,
|
||||
info_update=self._prepared_info_update(state, seq, silence=c["silence"], prefill_applied=False),
|
||||
prefill_applied=False,
|
||||
# Resolved by prepare_append from the scheduler's prompt length.
|
||||
prompt_offset=-1,
|
||||
depformer_audio_tokens_device=dep_tokens_device[row],
|
||||
depformer_audio_provided_device=c["live_provided_device"],
|
||||
),
|
||||
)
|
||||
return n
|
||||
|
||||
def prepared_depformer_state(self, request_id: str) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""Device teacher-forcing tensors of the request's current append, when prepared on the device."""
|
||||
key = self.request_sessions.get(request_id)
|
||||
state = self.sessions.get(key) if key is not None else None
|
||||
prepared = state.prepared if state is not None else None
|
||||
if prepared is None or prepared.depformer_audio_tokens_device is None:
|
||||
return None
|
||||
return prepared.depformer_audio_tokens_device, prepared.depformer_audio_provided_device
|
||||
|
||||
def record_sample(
|
||||
self,
|
||||
*,
|
||||
request_id: str,
|
||||
text_token: Any,
|
||||
agent_codes: Any,
|
||||
effective_codes: Any,
|
||||
) -> None:
|
||||
"""Commit one sampled temporal frame for the next live append."""
|
||||
"""Commit one sampled temporal frame for the next live append.
|
||||
|
||||
``effective_codes`` are the frame's eight agent codes with the append's
|
||||
depformer teacher forcing already applied, which the next append reads.
|
||||
"""
|
||||
import torch
|
||||
|
||||
if request_id in self._stale_requests:
|
||||
@@ -396,26 +565,12 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
return
|
||||
|
||||
text = torch.as_tensor(text_token, device=self._model_device_dtype()[0], dtype=torch.long).reshape(-1)
|
||||
codes = torch.as_tensor(agent_codes, device=text.device, dtype=torch.long).reshape(-1)
|
||||
codes = torch.as_tensor(effective_codes, device=text.device, dtype=torch.long).reshape(-1)
|
||||
if text.numel() != 1:
|
||||
raise ValueError(f"PersonaPlex Stage 0 expected one sampled text token, got {text.numel()}")
|
||||
if codes.numel() < 8:
|
||||
raise ValueError(f"PersonaPlex Stage 0 expected at least 8 agent codes, got {codes.numel()}")
|
||||
|
||||
effective_codes = codes[:8].clone()
|
||||
prepared = state.prepared
|
||||
if prepared is None:
|
||||
raise RuntimeError(f"PersonaPlex Stage 0 request has no prepared payload: {request_id}")
|
||||
target = prepared.info_update.get("pplex_depformer_audio_tokens")
|
||||
provided = prepared.info_update.get("pplex_depformer_audio_provided")
|
||||
if not isinstance(target, torch.Tensor) or not isinstance(provided, torch.Tensor):
|
||||
raise RuntimeError("PersonaPlex Stage 0 prepared payload has no depformer teacher-forcing state")
|
||||
target = target.reshape(-1).to(device=effective_codes.device, dtype=torch.long)
|
||||
provided = provided.reshape(-1).to(device=effective_codes.device, dtype=torch.bool)
|
||||
if target.numel() < 8 or provided.numel() < 8:
|
||||
raise RuntimeError("PersonaPlex Stage 0 depformer teacher-forcing state is shorter than one agent frame")
|
||||
effective_codes = torch.where(provided[:8], target[:8], effective_codes)
|
||||
state.last_agent_codes = effective_codes.detach()
|
||||
if codes.numel() != 8:
|
||||
raise ValueError(f"PersonaPlex Stage 0 expected 8 effective agent codes, got {codes.numel()}")
|
||||
state.last_agent_codes = codes.detach()
|
||||
state.last_text_token = text.detach()
|
||||
state.sampled_identity = state.prepared_identity
|
||||
|
||||
@@ -482,12 +637,14 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
pcm[state.slot] = torch.from_numpy(samples)
|
||||
active[state.slot] = True
|
||||
encoded = codec.encode_frame(pcm, active)
|
||||
codes_device = encoded.detach()[:, :8].to(dtype=torch.long)
|
||||
# One device-to-host copy per step, whatever the number of sessions.
|
||||
codes = encoded.detach().to(dtype=torch.long, device="cpu")
|
||||
if codes.shape[-1] < 8:
|
||||
raise RuntimeError(f"PersonaPlex Mimi encoder returned {codes.shape[-1]} codebooks, expected at least 8")
|
||||
for state, identity, _ in rows:
|
||||
state.encoded_frame = codes[state.slot : state.slot + 1, :8].clone()
|
||||
state.encoded_frame_device = codes_device[state.slot]
|
||||
state.encoded_identity = identity
|
||||
|
||||
def _shared_codec(self):
|
||||
@@ -506,6 +663,15 @@ class PersonaPlexStage0DuplexRuntime:
|
||||
device=self.device,
|
||||
)
|
||||
codec.streaming_init(self.max_sessions)
|
||||
if self._codec_cuda_graphs:
|
||||
# Rows are leased and recycled in place, so one graph at
|
||||
# max_sessions with the active mask as input serves every step.
|
||||
captured = codec.capture_cuda_graphs(encode=True, decode_frame_counts=())
|
||||
logger.info(
|
||||
"PersonaPlex Stage 0 Mimi encoder %s at %d rows",
|
||||
"replays a CUDA graph" if captured else "runs eagerly",
|
||||
self.max_sessions,
|
||||
)
|
||||
self._codec = codec
|
||||
return codec
|
||||
|
||||
|
||||
@@ -406,6 +406,28 @@ class PersonaPlexCode2Wav(nn.Module):
|
||||
).eval()
|
||||
for _ in range(self._max_codec_sessions)
|
||||
]
|
||||
if getattr(self.config, "mimi_cuda_graphs", False):
|
||||
# Each decoder is leased per request and reset in place, so its B=1
|
||||
# graphs stay valid. Deltas arrive as the first single frame and then
|
||||
# full chunks, so those two frame counts are recorded; other counts
|
||||
# run eagerly. The graphs replay one at a time on this stage's stream
|
||||
# and copy their output, so they share one pool.
|
||||
pool = torch.cuda.graph_pool_handle() if torch.device(str(device)).type == "cuda" else None
|
||||
captured = 0
|
||||
for codec in codecs:
|
||||
codec.streaming_init(1)
|
||||
captured += bool(
|
||||
codec.capture_cuda_graphs(
|
||||
encode=False,
|
||||
decode_frame_counts=(1, _MIMI_DECODE_BATCH_FRAMES),
|
||||
pool=pool,
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
"PersonaPlex Code2Wav replays Mimi decode from CUDA graphs on %d/%d stream(s)",
|
||||
captured,
|
||||
len(codecs),
|
||||
)
|
||||
self._set_mimi_codecs(codecs)
|
||||
self._mimi_device = torch.device(str(device))
|
||||
reported_sr = getattr(codecs[0].model.config, "sampling_rate", None)
|
||||
|
||||
@@ -29,9 +29,12 @@ codec composes with elastic slot recycling in batched duplex serving.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_temporal import (
|
||||
_apply_rope,
|
||||
@@ -42,13 +45,17 @@ DEFAULT_HF_REPO = "kyutai/mimi"
|
||||
FRAME_SIZE = 1920
|
||||
CODEBOOKS = 8
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_mimi_cudagraph import MimiFrameGraph
|
||||
|
||||
|
||||
def _normalize_active(active: torch.Tensor | None, all_active: torch.Tensor) -> torch.Tensor:
|
||||
# Stage 0 and Code2Wav currently use one B=1 codec per session and omit
|
||||
# `active`, so preserve that API by treating None as all rows active. When
|
||||
# they switch to shared B>1 codecs, their batch builders must pass bool[B]:
|
||||
# Stage 0 shares one codec with a row per session slot and passes bool[B]:
|
||||
# True advances that row's streaming state; False keeps an absent or padded
|
||||
# row's offsets and convolution carries unchanged.
|
||||
# row's offsets and convolution carries unchanged. Code2Wav leases one B=1
|
||||
# codec per request and omits `active`, which means all rows are active.
|
||||
if active is None:
|
||||
return all_active
|
||||
if active.shape != all_active.shape:
|
||||
@@ -107,6 +114,10 @@ class _StreamConv1d:
|
||||
``pad_mode`` controls the stream-start left padding: SEANet convs use zeros
|
||||
(``constant``); the down/upsample resamplers use ``replicate`` (the first
|
||||
real sample), marked per row so elastic slot recycling re-primes correctly.
|
||||
|
||||
The carry and the fresh-row flags are allocated once in ``reset`` and only
|
||||
ever updated in place, so a CUDA graph recorded over ``__call__`` keeps
|
||||
reading and writing the live state across frames, row resets and sessions.
|
||||
"""
|
||||
|
||||
def __init__(self, conv: nn.Conv1d, pad_mode: str = "constant") -> None:
|
||||
@@ -122,29 +133,36 @@ class _StreamConv1d:
|
||||
self.prev = torch.zeros(batch_size, self.conv.in_channels, pad, device=device, dtype=dtype)
|
||||
self._fresh = torch.ones(batch_size, dtype=torch.bool, device=device)
|
||||
|
||||
def reset_all(self) -> None:
|
||||
self.prev.zero_()
|
||||
self._fresh.fill_(True)
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
self.prev[b].zero_()
|
||||
self._fresh[b] = True
|
||||
|
||||
def __call__(self, x: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
|
||||
active_view = active.view(-1, 1, 1)
|
||||
if self.pad_mode == "replicate":
|
||||
pad = self.prev.shape[-1]
|
||||
edge = x[..., 0:1].expand(-1, -1, pad)
|
||||
fresh = (self._fresh & active).view(-1, 1, 1)
|
||||
self.prev = torch.where(fresh, edge.to(self.prev.dtype), self.prev)
|
||||
self._fresh[active] = False
|
||||
self.prev.copy_(torch.where(fresh, edge.to(self.prev.dtype), self.prev))
|
||||
self._fresh.logical_and_(~active)
|
||||
x = torch.cat([self.prev, x], dim=-1)
|
||||
t = x.shape[-1]
|
||||
num_frames = max(0, (t - self.kernel) // self.stride + 1)
|
||||
prev = x[..., num_frames * self.stride :]
|
||||
self.prev = torch.where(active.view(-1, 1, 1), prev, self.prev)
|
||||
self.prev.copy_(torch.where(active_view, x[..., num_frames * self.stride :], self.prev))
|
||||
if num_frames == 0:
|
||||
return x.new_zeros(x.shape[0], self.conv.out_channels, 0)
|
||||
return self.conv(x[..., : (num_frames - 1) * self.stride + self.kernel])
|
||||
|
||||
|
||||
class _StreamConvTr1d:
|
||||
"""Moshi ``RawStreamingConvTranspose1d``: overlap-add tail carry per call."""
|
||||
"""Moshi ``RawStreamingConvTranspose1d``: overlap-add tail carry per call.
|
||||
|
||||
Like ``_StreamConv1d``, the tail carry is updated in place only.
|
||||
"""
|
||||
|
||||
def __init__(self, conv: nn.ConvTranspose1d) -> None:
|
||||
self.conv = conv
|
||||
@@ -159,6 +177,10 @@ class _StreamConvTr1d:
|
||||
)
|
||||
self._fresh = torch.ones(batch_size, dtype=torch.bool, device=device)
|
||||
|
||||
def reset_all(self) -> None:
|
||||
self.partial.zero_()
|
||||
self._fresh.fill_(True)
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
self.partial[b].zero_()
|
||||
self._fresh[b] = True
|
||||
@@ -168,17 +190,17 @@ class _StreamConvTr1d:
|
||||
length = out.shape[-1]
|
||||
tail = self.kernel - self.stride
|
||||
pt = self.partial.shape[-1]
|
||||
merge = self.partial.clone()
|
||||
merge = self.partial
|
||||
if self.conv.bias is not None:
|
||||
# The carried tail already includes the bias; the fresh output adds
|
||||
# it again, so subtract one copy -- except on a row's very first
|
||||
# frame, where the carry is zeros by construction.
|
||||
merge = merge - self.conv.bias[:, None]
|
||||
merge[self._fresh & active] = 0.0
|
||||
self._fresh[active] = False
|
||||
first = (self._fresh & active).view(-1, 1, 1)
|
||||
merge = torch.where(first, torch.zeros_like(merge), merge)
|
||||
self._fresh.logical_and_(~active)
|
||||
out[..., :pt] += merge
|
||||
partial = out[..., length - tail :].clone()
|
||||
self.partial = torch.where(active.view(-1, 1, 1), partial, self.partial)
|
||||
self.partial.copy_(torch.where(active.view(-1, 1, 1), out[..., length - tail :], self.partial))
|
||||
return out[..., : length - tail]
|
||||
|
||||
|
||||
@@ -358,13 +380,16 @@ class PersonaPlexMimiCodec(nn.Module):
|
||||
n_dec = self.decoder_transformer.load_weights(sd, "decoder_transformer")
|
||||
assert n_enc == n_dec == 80, (n_enc, n_dec)
|
||||
del sd
|
||||
self._init_streaming_stages()
|
||||
|
||||
def _init_streaming_stages(self) -> None:
|
||||
m = self.model
|
||||
self._enc_stages = _walk_seanet(m.encoder.layers)
|
||||
self._downsample = _StreamConv1d(m.downsample.conv, pad_mode="replicate")
|
||||
self._upsample = _StreamConvTr1d(m.upsample.conv)
|
||||
self._dec_stages = _walk_seanet(m.decoder.layers)
|
||||
self._batch_size: int | None = None
|
||||
self._cuda_graphs: dict[str, MimiFrameGraph] = {}
|
||||
self._all_active: torch.Tensor
|
||||
|
||||
# -- streaming state ------------------------------------------------------
|
||||
@@ -380,7 +405,22 @@ class PersonaPlexMimiCodec(nn.Module):
|
||||
yield self._upsample
|
||||
|
||||
def streaming_init(self, batch_size: int) -> None:
|
||||
# Re-initializing at the same batch size resets in place: the state
|
||||
# tensors keep their storage, so captured frame graphs stay valid.
|
||||
if getattr(self, "_batch_size", None) == batch_size:
|
||||
self.reset_streaming()
|
||||
return
|
||||
if getattr(self, "_cuda_graphs", None):
|
||||
# The state is reallocated below, so graphs recorded at the old
|
||||
# batch size would read stale tensors; drop them and say so.
|
||||
logger.warning(
|
||||
"PersonaPlex Mimi stream resized from %s to %d rows; dropping its CUDA graphs (%s)",
|
||||
self._batch_size,
|
||||
batch_size,
|
||||
"/".join(sorted(self._cuda_graphs)),
|
||||
)
|
||||
self._batch_size = batch_size
|
||||
self._cuda_graphs = {}
|
||||
self._all_active = torch.ones(batch_size, dtype=torch.bool, device=self.device)
|
||||
for s in self._conv_states():
|
||||
s.reset(batch_size, self.device, self.dtype)
|
||||
@@ -390,8 +430,7 @@ class PersonaPlexMimiCodec(nn.Module):
|
||||
def reset_streaming(self) -> None:
|
||||
assert self._batch_size is not None
|
||||
for state in self._conv_states():
|
||||
for b in range(self._batch_size):
|
||||
state.reset_slot(b)
|
||||
state.reset_all()
|
||||
self.encoder_transformer.reset_streaming()
|
||||
self.decoder_transformer.reset_streaming()
|
||||
|
||||
@@ -401,6 +440,42 @@ class PersonaPlexMimiCodec(nn.Module):
|
||||
self.encoder_transformer.reset_slot(b)
|
||||
self.decoder_transformer.reset_slot(b)
|
||||
|
||||
def capture_cuda_graphs(
|
||||
self,
|
||||
*,
|
||||
encode: bool = True,
|
||||
decode_frame_counts: tuple[int, ...] = (1,),
|
||||
warmup_iters: int = 2,
|
||||
pool: tuple[int, int] | None = None,
|
||||
) -> list[str]:
|
||||
"""Record the per-frame codec steps at the streaming batch size.
|
||||
|
||||
``encode`` records ``encode_frame``; each ``F`` in ``decode_frame_counts``
|
||||
records ``decode_frames`` over ``F`` frames (``decode_frame`` is ``F=1``).
|
||||
Every row is recorded active; the ``active`` mask is a graph input, so
|
||||
one graph serves any subset of live rows. Warmup frames run through the
|
||||
real streaming state, which is then reset in place, so the codec is
|
||||
fresh afterwards. ``pool`` is a CUDA graph memory pool to share with
|
||||
other codecs replayed on the same stream; by default the codec gets a
|
||||
private one. Returns the captured graph names; on a non-CUDA device the
|
||||
codec stays eager, and so it does after a recoverable capture failure,
|
||||
which is logged with its traceback. Warmup, device execution and reset
|
||||
failures propagate instead of leaving an apparently usable codec.
|
||||
"""
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_mimi_cudagraph import (
|
||||
capture_mimi_frame_graphs,
|
||||
)
|
||||
|
||||
assert self._batch_size is not None, "call streaming_init first"
|
||||
self._cuda_graphs = capture_mimi_frame_graphs(
|
||||
self,
|
||||
encode=encode,
|
||||
decode_frame_counts=decode_frame_counts,
|
||||
warmup_iters=warmup_iters,
|
||||
pool=pool,
|
||||
)
|
||||
return sorted(self._cuda_graphs)
|
||||
|
||||
# -- per-frame codec -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
@@ -413,32 +488,68 @@ class PersonaPlexMimiCodec(nn.Module):
|
||||
x = stage(x, active) if kind in {"conv", "convtr"} else stage(x)
|
||||
return x
|
||||
|
||||
def _frame_graph(self, name: str) -> MimiFrameGraph | None:
|
||||
graph = self._cuda_graphs.get(name)
|
||||
if graph is None or torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
return graph
|
||||
|
||||
@torch.no_grad()
|
||||
def encode_frame(self, pcm: torch.Tensor, active: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""``[B, frame_size]`` float PCM -> ``[B, 8]`` codes."""
|
||||
x = pcm.to(self.device, self.dtype).view(-1, 1, FRAME_SIZE)
|
||||
active = _normalize_active(active, self._all_active)
|
||||
graph = self._frame_graph("encode")
|
||||
if graph is not None:
|
||||
return graph.replay(pcm.reshape(-1, FRAME_SIZE), active)
|
||||
return self._encode_frame_eager(pcm, active)
|
||||
|
||||
def _encode_frame_eager(self, pcm: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
|
||||
x = pcm.to(self.device, self.dtype).view(-1, 1, FRAME_SIZE)
|
||||
x = self._run_stages(x, self._enc_stages, active)
|
||||
x = self.encoder_transformer.step(x.transpose(1, 2), active).transpose(1, 2)
|
||||
x = self._downsample(x, active)
|
||||
codes = self.model.quantizer.encode(x, num_quantizers=CODEBOOKS) # [Q, B, T]
|
||||
return codes[:CODEBOOKS, :, 0].transpose(0, 1).contiguous()
|
||||
|
||||
def _quantizer_decode(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
"""``MimiSplitResidualVectorQuantizer.decode`` without its host-side scalar.
|
||||
|
||||
The Hugging Face RVQ decode seeds its sum with ``torch.tensor(0.0)``, a
|
||||
host-to-device copy that cannot be recorded in a CUDA graph. This
|
||||
seeds it with an on-device zero instead; the arithmetic is unchanged.
|
||||
"""
|
||||
quantizer = self.model.quantizer
|
||||
split = quantizer.num_semantic_quantizers
|
||||
|
||||
def rvq_decode(rvq, rvq_codes: torch.Tensor) -> torch.Tensor:
|
||||
out = torch.zeros((), device=rvq_codes.device)
|
||||
for layer, indices in zip(rvq.layers, rvq_codes.transpose(0, 1)):
|
||||
out = out + layer.decode(indices)
|
||||
if rvq.output_proj is not None:
|
||||
out = rvq.output_proj(out)
|
||||
return out
|
||||
|
||||
out = rvq_decode(quantizer.semantic_residual_vector_quantizer, codes[:, :split])
|
||||
if codes.shape[1] > split:
|
||||
out += rvq_decode(quantizer.acoustic_residual_vector_quantizer, codes[:, split:])
|
||||
return out
|
||||
|
||||
@torch.no_grad()
|
||||
def decode_frame(self, codes: torch.Tensor, active: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""``[B, 8]`` codes -> ``[B, frame_size]`` float PCM."""
|
||||
emb = self.model.quantizer.decode(codes.to(self.device).view(-1, CODEBOOKS, 1))
|
||||
active = _normalize_active(active, self._all_active)
|
||||
emb = self._upsample(emb, active)
|
||||
emb = self.decoder_transformer.step(emb.transpose(1, 2), active).transpose(1, 2)
|
||||
x = self._run_stages(emb, self._dec_stages, active)
|
||||
return x[:, 0, :]
|
||||
return self.decode_frames(codes.reshape(-1, CODEBOOKS, 1), active)
|
||||
|
||||
@torch.no_grad()
|
||||
def decode_frames(self, codes: torch.Tensor, active: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""``[B, 8, F]`` codes -> ``[B, F * frame_size]`` float PCM."""
|
||||
emb = self.model.quantizer.decode(codes.to(self.device))
|
||||
active = _normalize_active(active, self._all_active)
|
||||
graph = self._frame_graph(f"decode_f{codes.shape[-1]}")
|
||||
if graph is not None:
|
||||
return graph.replay(codes, active)
|
||||
return self._decode_frames_eager(codes, active)
|
||||
|
||||
def _decode_frames_eager(self, codes: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
|
||||
emb = self._quantizer_decode(codes.to(self.device))
|
||||
emb = self._upsample(emb, active)
|
||||
emb = self.decoder_transformer.step(emb.transpose(1, 2), active).transpose(1, 2)
|
||||
x = self._run_stages(emb, self._dec_stages, active)
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Replay the PersonaPlex streaming Mimi frame steps from CUDA graphs.
|
||||
|
||||
One 80 ms Mimi encode or decode frame is a few hundred small kernels (SEANet
|
||||
convs, an 8-layer streaming transformer and the RVQ), so eager execution is
|
||||
launch bound. Every streaming state tensor of ``PersonaPlexMimiCodec`` (conv
|
||||
carries, fresh-row flags, ring KV caches and offsets) is allocated once by
|
||||
``streaming_init`` and only updated in place, and row resets are in place too,
|
||||
so a graph recorded at the streaming batch size stays valid across frames,
|
||||
session slot recycling and full stream resets.
|
||||
|
||||
The ``active`` row mask is a graph input: one graph at the shared batch size
|
||||
serves any subset of live rows, and inactive rows keep their state exactly as
|
||||
in eager execution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_mimi import PersonaPlexMimiCodec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["MimiFrameGraph", "capture_mimi_frame_graphs"]
|
||||
|
||||
|
||||
def _is_recoverable_capture_error(error: RuntimeError) -> bool:
|
||||
"""Allow only known capture limitations, never an unexplained CUDA error."""
|
||||
pending: list[BaseException] = [error]
|
||||
seen: set[int] = set()
|
||||
known_failure = False
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if id(current) in seen:
|
||||
continue
|
||||
seen.add(id(current))
|
||||
message = str(current).split("\n", 1)[0]
|
||||
if isinstance(current, torch.cuda.OutOfMemoryError) or (
|
||||
isinstance(current, RuntimeError)
|
||||
and (
|
||||
message.startswith("CUDA out of memory.")
|
||||
or message
|
||||
in {
|
||||
"CUDA error: out of memory",
|
||||
"CUDA error: operation not permitted when stream is capturing",
|
||||
}
|
||||
)
|
||||
):
|
||||
known_failure = True
|
||||
elif not (
|
||||
isinstance(current, RuntimeError)
|
||||
and message == "CUDA error: operation failed due to a previous error during capture"
|
||||
):
|
||||
return False
|
||||
# capture_end can replace a body exception with capture-invalidated.
|
||||
# The original failure must also be recoverable, even if suppressed.
|
||||
pending.extend(exc for exc in (current.__cause__, current.__context__) if exc is not None)
|
||||
return known_failure
|
||||
|
||||
|
||||
def _capture_error_details(error: RuntimeError) -> str:
|
||||
"""Preserve diagnostics without retaining capture frames or graph pools."""
|
||||
details = "".join(traceback.format_exception(error))
|
||||
pending: list[BaseException] = [error]
|
||||
seen: set[int] = set()
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if id(current) in seen:
|
||||
continue
|
||||
seen.add(id(current))
|
||||
if current.__traceback__ is not None:
|
||||
# ExitStack frames can retain their own traceback in exception tuples.
|
||||
traceback.clear_frames(current.__traceback__)
|
||||
current.__traceback__ = None
|
||||
pending.extend(exc for exc in (current.__cause__, current.__context__) if exc is not None)
|
||||
return details
|
||||
|
||||
|
||||
class MimiFrameGraph:
|
||||
"""A recorded ``(frame, active) -> output`` codec step with static buffers."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
graph: torch.cuda.CUDAGraph,
|
||||
static_input: torch.Tensor,
|
||||
static_active: torch.Tensor,
|
||||
static_output: torch.Tensor,
|
||||
eager: Callable[[torch.Tensor, torch.Tensor], torch.Tensor],
|
||||
) -> None:
|
||||
self.graph = graph
|
||||
self.static_input = static_input
|
||||
self.static_active = static_active
|
||||
self.static_output = static_output
|
||||
self.eager = eager
|
||||
self.replays = 0
|
||||
self._warned = False
|
||||
|
||||
def replay(self, frame: torch.Tensor, active: torch.Tensor) -> torch.Tensor:
|
||||
if frame.shape != self.static_input.shape or active.shape != self.static_active.shape:
|
||||
if not self._warned:
|
||||
self._warned = True
|
||||
logger.warning(
|
||||
"PersonaPlex Mimi graph input shape %s does not match capture shape %s; running eagerly",
|
||||
tuple(frame.shape),
|
||||
tuple(self.static_input.shape),
|
||||
)
|
||||
return self.eager(frame, active)
|
||||
self.static_input.copy_(frame)
|
||||
self.static_active.copy_(active)
|
||||
self.graph.replay()
|
||||
self.replays += 1
|
||||
# The static output is overwritten by the next replay.
|
||||
return self.static_output.clone()
|
||||
|
||||
|
||||
def capture_mimi_frame_graphs(
|
||||
codec: PersonaPlexMimiCodec,
|
||||
*,
|
||||
encode: bool,
|
||||
decode_frame_counts: tuple[int, ...],
|
||||
warmup_iters: int = 2,
|
||||
pool: tuple[int, int] | None = None,
|
||||
) -> dict[str, MimiFrameGraph]:
|
||||
"""Capture the codec's encode and ``F``-frame decode steps.
|
||||
|
||||
Returns an empty dict off CUDA or after a recoverable capture failure.
|
||||
Warmup, device execution and stream-reset failures always propagate.
|
||||
"""
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_mimi import CODEBOOKS, FRAME_SIZE
|
||||
|
||||
device = torch.device(codec.device)
|
||||
if device.type != "cuda" or not torch.cuda.is_available():
|
||||
logger.info("PersonaPlex Mimi CUDA graphs skipped on device %s", device)
|
||||
return {}
|
||||
batch_size = codec._batch_size
|
||||
specs: list[tuple[str, Callable, torch.Tensor]] = []
|
||||
if encode:
|
||||
specs.append(
|
||||
(
|
||||
"encode",
|
||||
codec._encode_frame_eager,
|
||||
torch.zeros(batch_size, FRAME_SIZE, device=device, dtype=codec.dtype),
|
||||
)
|
||||
)
|
||||
for frames in sorted(set(decode_frame_counts)):
|
||||
specs.append(
|
||||
(
|
||||
f"decode_f{frames}",
|
||||
codec._decode_frames_eager,
|
||||
torch.zeros(batch_size, CODEBOOKS, frames, device=device, dtype=torch.long),
|
||||
)
|
||||
)
|
||||
# A private pool by default: these graphs are captured outside the model
|
||||
# runner's own capture sequence, so they must not reuse its pool blocks.
|
||||
if pool is None:
|
||||
pool = torch.cuda.graph_pool_handle()
|
||||
graphs: dict[str, MimiFrameGraph] = {}
|
||||
warmed_specs: list[tuple[str, Callable, torch.Tensor, torch.Tensor]] = []
|
||||
capture_error_details: str | None = None
|
||||
try:
|
||||
with torch.no_grad():
|
||||
# Validate every eager step before capture can select the fallback.
|
||||
for name, eager, static_input in specs:
|
||||
static_active = torch.ones(batch_size, dtype=torch.bool, device=device)
|
||||
for _ in range(max(warmup_iters, 1)):
|
||||
eager(static_input, static_active)
|
||||
torch.accelerator.synchronize(device)
|
||||
warmed_specs.append((name, eager, static_input, static_active))
|
||||
for name, eager, static_input, static_active in warmed_specs:
|
||||
# capture_end may raise before torch.cuda.graph restores its
|
||||
# stream. Always restore our caller's stream on that path.
|
||||
with torch.cuda.stream(torch.cuda.current_stream(device)):
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
# __enter__ synchronizes and clears the CUDA cache. A
|
||||
# failure there is a setup failure, not a capture fallback.
|
||||
capture = torch.cuda.graph(graph, pool=pool, capture_error_mode="thread_local")
|
||||
capture.__enter__()
|
||||
try:
|
||||
# Other worker threads may issue CUDA calls while this records.
|
||||
with ExitStack() as stack:
|
||||
stack.push(capture)
|
||||
static_output = eager(static_input, static_active)
|
||||
except RuntimeError as error:
|
||||
if not _is_recoverable_capture_error(error):
|
||||
raise
|
||||
capture_error_details = _capture_error_details(error)
|
||||
graphs.clear()
|
||||
# Release partial graphs and graph-owned output tensors
|
||||
# before reset, including the last loop iteration.
|
||||
graph = static_output = capture = None
|
||||
break
|
||||
# A device execution failure is not an optional capture failure.
|
||||
torch.accelerator.synchronize(device)
|
||||
graphs[name] = MimiFrameGraph(graph, static_input, static_active, static_output, eager)
|
||||
finally:
|
||||
# Warmup frames advanced the real streaming state; restore a fresh
|
||||
# stream without reallocating the tensors the graphs point at.
|
||||
codec.reset_streaming()
|
||||
# Reset enqueues device work too. Do not report usable graphs or an eager
|
||||
# fallback unless that work completed successfully.
|
||||
torch.accelerator.synchronize(device)
|
||||
if capture_error_details is not None:
|
||||
logger.warning(
|
||||
"PersonaPlex Mimi CUDA graphs were requested (%s at batch size %d) but capture failed; "
|
||||
"the codec runs eagerly, which lowers realtime session capacity\n%s",
|
||||
"/".join(name for name, _, _ in specs),
|
||||
batch_size,
|
||||
capture_error_details,
|
||||
)
|
||||
if graphs:
|
||||
logger.debug("Captured PersonaPlex Mimi %s graph(s) at batch size %d", "/".join(graphs), batch_size)
|
||||
return graphs
|
||||
@@ -30,7 +30,7 @@ encode of the input WAV (built in ``preprocess``); live duplex is Phase 2.
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -54,6 +54,11 @@ from vllm_omni.model_executor.models.personaplex.personaplex_embeddings import (
|
||||
PersonaPlexInputEmbeddings,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.model_executor.models.personaplex.duplex.stage0 import (
|
||||
PersonaPlexStage0DuplexRuntime,
|
||||
)
|
||||
|
||||
__all__ = ["PersonaPlexTalkerForConditionalGeneration"]
|
||||
|
||||
|
||||
@@ -437,6 +442,7 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
|
||||
model_path=model_path,
|
||||
device=device,
|
||||
max_sessions=int(getattr(self.vllm_config.model_config, "duplex_max_sessions", 1)),
|
||||
codec_cuda_graphs=bool(getattr(self.config, "mimi_cuda_graphs", False)),
|
||||
)
|
||||
self._personaplex_duplex_stage0_runtime = runtime
|
||||
return runtime
|
||||
@@ -485,7 +491,9 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
|
||||
if isinstance(duplex, dict) and duplex.get("data_plane") is True:
|
||||
appends.append(duplex)
|
||||
if appends:
|
||||
self._duplex_stage0_runtime().encode_appends(appends)
|
||||
runtime = self._duplex_stage0_runtime()
|
||||
runtime.encode_appends(appends)
|
||||
runtime.prepare_live_appends(appends)
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None)
|
||||
@@ -551,9 +559,48 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
|
||||
)
|
||||
text_token = input_ids.reshape(bsz).to(torch.long)
|
||||
hidden = hidden_states.reshape(bsz, 1, -1).to(self._dtype)
|
||||
audio_tokens: list[torch.Tensor] = []
|
||||
audio_provided: list[torch.Tensor] = []
|
||||
for info in req_infos:
|
||||
runtime = self._duplex_stage0_runtime()
|
||||
audio_tokens, audio_provided = self._depformer_teacher_forcing(runtime, req_ids, req_infos, hidden.device)
|
||||
codes = self.depformer(
|
||||
text_token,
|
||||
hidden,
|
||||
audio_tokens=audio_tokens,
|
||||
audio_provided=audio_provided,
|
||||
num_steps=self.num_active_codebooks,
|
||||
).to(torch.long)
|
||||
# The next append reads the agent frame with the teacher-forced
|
||||
# codebooks applied; merge it once for the whole batch.
|
||||
effective = torch.where(audio_provided[:, :8], audio_tokens[:, :8], codes[:, :8])
|
||||
for row, request_id in enumerate(req_ids):
|
||||
runtime.record_sample(
|
||||
request_id=request_id,
|
||||
text_token=text_token[row],
|
||||
effective_codes=effective[row],
|
||||
)
|
||||
return codes
|
||||
|
||||
@staticmethod
|
||||
def _depformer_teacher_forcing(
|
||||
runtime: PersonaPlexStage0DuplexRuntime,
|
||||
req_ids: list[str],
|
||||
req_infos: list[dict[str, Any]],
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Stack each request's depformer teacher-forcing tokens and mask on ``device``.
|
||||
|
||||
Live appends batched in ``preprocess_batch`` already hold device copies.
|
||||
The rest (first appends, aborted epochs) carry host tensors in their
|
||||
request information and are moved in one copy.
|
||||
"""
|
||||
tokens_rows: list[torch.Tensor] = []
|
||||
provided_rows: list[torch.Tensor] = []
|
||||
host_rows: list[int] = []
|
||||
for row, (request_id, info) in enumerate(zip(req_ids, req_infos)):
|
||||
prepared = runtime.prepared_depformer_state(request_id)
|
||||
if prepared is not None:
|
||||
tokens_rows.append(prepared[0])
|
||||
provided_rows.append(prepared[1])
|
||||
continue
|
||||
tokens = info.get("pplex_depformer_audio_tokens")
|
||||
provided = info.get("pplex_depformer_audio_provided")
|
||||
if not isinstance(tokens, torch.Tensor) or not isinstance(provided, torch.Tensor):
|
||||
@@ -565,29 +612,18 @@ class PersonaPlexTalkerForConditionalGeneration(nn.Module):
|
||||
"PersonaPlex depformer teacher-forcing token/mask shapes differ: "
|
||||
f"{tuple(tokens.shape)} != {tuple(provided.shape)}"
|
||||
)
|
||||
audio_tokens.append(tokens)
|
||||
audio_provided.append(provided)
|
||||
codes = self.depformer(
|
||||
text_token,
|
||||
hidden,
|
||||
audio_tokens=torch.stack(audio_tokens).to(
|
||||
device=hidden.device,
|
||||
dtype=torch.long,
|
||||
),
|
||||
audio_provided=torch.stack(audio_provided).to(
|
||||
device=hidden.device,
|
||||
dtype=torch.bool,
|
||||
),
|
||||
num_steps=self.num_active_codebooks,
|
||||
).to(torch.long)
|
||||
runtime = self._duplex_stage0_runtime()
|
||||
for row, request_id in enumerate(req_ids):
|
||||
runtime.record_sample(
|
||||
request_id=request_id,
|
||||
text_token=text_token[row],
|
||||
agent_codes=codes[row],
|
||||
)
|
||||
return codes
|
||||
host_rows.append(row)
|
||||
tokens_rows.append(tokens)
|
||||
provided_rows.append(provided)
|
||||
if host_rows:
|
||||
host_tokens = torch.stack([tokens_rows[row] for row in host_rows]).to(device=device, dtype=torch.long)
|
||||
host_provided = torch.stack([provided_rows[row] for row in host_rows]).to(device=device, dtype=torch.bool)
|
||||
if len(host_rows) == len(req_ids):
|
||||
return host_tokens, host_provided
|
||||
for index, row in enumerate(host_rows):
|
||||
tokens_rows[row] = host_tokens[index]
|
||||
provided_rows[row] = host_provided[index]
|
||||
return torch.stack(tokens_rows), torch.stack(provided_rows)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
|
||||
Reference in new issue
Block a user