[Feature] PersonaPlex (Moshi-based full-duplex S2S): native vLLM port + duplex serving (#4771)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: Sy0307 <24841857+Sy0307@users.noreply.github.com>
Co-authored-by: Sy0307 <24841857+Sy0307@users.noreply.github.com>
This commit is contained in:
Yueqian LinandSy0307 authored and GitHub committed 2026-08-04 20:10:33 -07:00
1 parent f24a6165ff
commit e788ef6e76
59 files changed
+10389 -12

No files matched your search

@@ -0,0 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vllm-omni integration for PersonaPlex (a Moshi finetune, full-duplex S2S).
Offline/batch runs go through a two-stage audio->audio pipeline (talker ->
code2wav); real-time conversation is served over the duplex WebSocket path.
"""
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
PersonaPlexConfig,
PersonaPlexDepformerConfig,
PersonaPlexMimiConfig,
)
__all__ = [
"PersonaPlexConfig",
"PersonaPlexDepformerConfig",
"PersonaPlexMimiConfig",
]
@@ -0,0 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for the PersonaPlex Helium temporal transformer."""
from __future__ import annotations
from typing import Any
from transformers import PretrainedConfig
__all__ = ["HeliumConfig"]
class HeliumConfig(PretrainedConfig):
"""Minimal HF config for the Moshi temporal LM backbone.
The defaults are measured from the PersonaPlex checkpoint rather than inferred
from Moshi's higher-level kwargs.
"""
model_type = "helium"
keys_to_ignore_at_inference = ("past_key_values",)
def __init__(
self,
hidden_size: int = 4096,
num_hidden_layers: int = 32,
num_attention_heads: int = 32,
head_dim: int = 128,
num_key_value_heads: int = 32,
intermediate_size: int = 11264,
max_position_embeddings: int = 3000,
rope_theta: float = 10000.0,
rms_norm_eps: float = 1e-8,
vocab_size: int = 32000,
sliding_window: int = 3000,
hidden_act: str = "silu",
attention_bias: bool = False,
mlp_bias: bool = False,
tie_word_embeddings: bool = False,
use_cache: bool = True,
**kwargs: Any,
) -> None:
super().__init__(
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
self.hidden_size = hidden_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.head_dim = head_dim
self.num_key_value_heads = num_key_value_heads
self.intermediate_size = intermediate_size
self.max_position_embeddings = max_position_embeddings
self.rope_theta = rope_theta
self.rms_norm_eps = rms_norm_eps
self.vocab_size = vocab_size
self.sliding_window = sliding_window
self.hidden_act = hidden_act
self.attention_bias = attention_bias
self.mlp_bias = mlp_bias
self.use_cache = use_cache
@@ -0,0 +1,278 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for PersonaPlex (a Moshi finetune; 2-stage audio->audio pipeline).
PersonaPlex is a staged AR speech model composed of:
* a *temporal transformer* (a Llama-variant Helium backbone) that predicts, per
frame, the next text token and the codebook-0 audio token;
* a *depformer* (a small per-step transformer) that, conditioned on the temporal
hidden state, predicts the remaining audio codebooks;
* the Mimi neural audio codec, which turns the audio codebooks into 24 kHz PCM.
The temporal-transformer hyperparameters live in
:class:`~vllm_omni.model_executor.models.personaplex.configuration_helium.HeliumConfig`
(already used by the talker the lead is building). This module reuses it as the
``temporal_config`` sub-config so the config tree carries a single source of
truth, and adds two further sub-configs (``depformer_config`` and
``mimi_config``).
All defaults are *measured* from the PersonaPlex checkpoint, whose own
``config.json`` is empty; do not treat the Moshi higher-level kwargs as
authoritative.
"""
from __future__ import annotations
from typing import Any
from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging
from vllm_omni.model_executor.models.personaplex.configuration_helium import (
HeliumConfig,
)
logger = logging.get_logger(__name__)
__all__ = [
"PersonaPlexConfig",
"PersonaPlexDepformerConfig",
"PersonaPlexMimiConfig",
"HeliumConfig",
]
class PersonaPlexDepformerConfig(PretrainedConfig):
r"""Configuration for the PersonaPlex depformer (per-step code predictor).
The depformer is a small autoregressive transformer that runs ``dep_q``
inner steps per temporal frame, conditioned on the temporal hidden state,
to predict the audio codebooks. Only the first ``num_active_codebooks``
codebooks (``cb 0..7``) are decoded to PCM by Mimi; the remaining codebooks
up to ``dep_q`` are predicted but not vocoded.
Args:
hidden_size (`int`, *optional*, defaults to 1024):
Dimension of the depformer hidden representations.
num_hidden_layers (`int`, *optional*, defaults to 6):
Number of depformer transformer layers.
num_attention_heads (`int`, *optional*, defaults to 16):
Number of attention heads per depformer layer.
head_dim (`int`, *optional*, defaults to 64):
Per-head attention dimension (``hidden_size // num_attention_heads``).
max_position_embeddings (`int`, *optional*, defaults to 8):
The depformer context length (``ctx``): one position per inner step.
dep_q (`int`, *optional*, defaults to 16):
Number of audio codebooks the depformer predicts per frame.
num_active_codebooks (`int`, *optional*, defaults to 8):
Number of leading codebooks actually decoded to PCM by Mimi.
card (`int`, *optional*, defaults to 2048):
Per-codebook cardinality (audio vocab size).
rope_theta (`float`, *optional*, defaults to 10000.0):
The base period of the RoPE embeddings.
rms_norm_eps (`float`, *optional*, defaults to 1e-8):
The epsilon used by the fp32 RMS normalization layers.
hidden_act (`str`, *optional*, defaults to `"silu"`):
SwiGLU gate activation.
attention_bias (`bool`, *optional*, defaults to `False`):
Whether attention projections carry a bias.
mlp_bias (`bool`, *optional*, defaults to `False`):
Whether the MLP projections carry a bias.
"""
model_type = "personaplex_depformer"
keys_to_ignore_at_inference = ("past_key_values",)
def __init__(
self,
hidden_size: int = 1024,
num_hidden_layers: int = 6,
num_attention_heads: int = 16,
head_dim: int = 64,
num_key_value_heads: int = 16,
intermediate_size: int = 2816,
max_position_embeddings: int = 8,
dep_q: int = 16,
num_active_codebooks: int = 8,
card: int = 2048,
rope_theta: float = 10000.0,
rms_norm_eps: float = 1e-8,
hidden_act: str = "silu",
attention_bias: bool = False,
mlp_bias: bool = False,
tie_word_embeddings: bool = False,
use_cache: bool = True,
**kwargs: Any,
) -> None:
super().__init__(
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
self.hidden_size = hidden_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.head_dim = head_dim
self.num_key_value_heads = num_key_value_heads
self.intermediate_size = intermediate_size
self.max_position_embeddings = max_position_embeddings
self.dep_q = dep_q
self.num_active_codebooks = num_active_codebooks
self.card = card
self.rope_theta = rope_theta
self.rms_norm_eps = rms_norm_eps
self.hidden_act = hidden_act
self.attention_bias = attention_bias
self.mlp_bias = mlp_bias
self.use_cache = use_cache
class PersonaPlexMimiConfig(PretrainedConfig):
r"""Configuration for the Mimi neural audio codec used by PersonaPlex.
The Mimi weights and module live in the external ``moshi`` package; this
config only carries the scalars the vllm-omni serving layer needs to size
buffers and compute audio durations. The actual decoder is instantiated by
:class:`PersonaPlexCode2Wav` via ``moshi.models.loaders.get_mimi``.
Args:
sample_rate (`int`, *optional*, defaults to 24000):
Output PCM sample rate in Hz.
frame_rate (`float`, *optional*, defaults to 12.5):
Mimi codec frame rate in Hz (one frame every 80 ms).
samples_per_frame (`int`, *optional*, defaults to 1920):
PCM samples produced per codec frame (``sample_rate / frame_rate``).
num_codebooks (`int`, *optional*, defaults to 8):
Number of active audio codebooks decoded to PCM (``cb 0..7``).
card (`int`, *optional*, defaults to 2048):
Per-codebook cardinality.
num_channels (`int`, *optional*, defaults to 1):
Number of output audio channels (mono).
mimi_name (`str`, *optional*):
Filename of the Mimi weight checkpoint inside the model repo. When
``None``, :class:`PersonaPlexCode2Wav` falls back to
``moshi.models.loaders.MIMI_NAME``.
"""
model_type = "personaplex_mimi"
def __init__(
self,
sample_rate: int = 24000,
frame_rate: float = 12.5,
samples_per_frame: int = 1920,
num_codebooks: int = 8,
card: int = 2048,
num_channels: int = 1,
mimi_name: str | None = None,
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self.sample_rate = sample_rate
self.frame_rate = frame_rate
self.samples_per_frame = samples_per_frame
self.num_codebooks = num_codebooks
self.card = card
self.num_channels = num_channels
self.mimi_name = mimi_name
class PersonaPlexConfig(PretrainedConfig):
r"""Top-level configuration for ``PersonaPlexTalkerForConditionalGeneration``.
Mirrors the Qwen3-TTS config layout: a top-level config holding sub-configs
for each component. The temporal transformer config is the *text config*
that vLLM consumes (it exposes ``hidden_size`` / ``num_attention_heads`` and
drives the talker's vLLM Llama backbone).
Args:
temporal_config (`dict` or `HeliumConfig`, *optional*):
The temporal-transformer (Helium) backbone config.
depformer_config (`dict` or `PersonaPlexDepformerConfig`, *optional*):
The depformer (per-step code predictor) config.
mimi_config (`dict` or `PersonaPlexMimiConfig`, *optional*):
The Mimi codec config.
text_vocab_size (`int`, *optional*, defaults to 32000):
Text / ``lm_head`` vocabulary size.
text_embedding_rows (`int`, *optional*, defaults to 32001):
Number of rows in the text embedding table (one extra padding row).
audio_vocab_size (`int`, *optional*, defaults to 2048):
Per-codebook audio cardinality (``card``).
num_audio_codebooks (`int`, *optional*, defaults to 16):
Total number of audio codebooks (``n_q``).
mimi_name (`str`, *optional*):
Convenience mirror of ``mimi_config.mimi_name``; if set, it
overrides the value carried inside ``mimi_config``.
"""
model_type = "personaplex"
sub_configs = {
"temporal_config": HeliumConfig,
"depformer_config": PersonaPlexDepformerConfig,
"mimi_config": PersonaPlexMimiConfig,
}
def __init__(
self,
temporal_config: dict | HeliumConfig | None = None,
depformer_config: dict | PersonaPlexDepformerConfig | None = None,
mimi_config: dict | PersonaPlexMimiConfig | None = None,
text_vocab_size: int = 32000,
text_embedding_rows: int = 32001,
audio_vocab_size: int = 2048,
num_audio_codebooks: int = 16,
mimi_name: str | None = None,
**kwargs: Any,
) -> None:
if temporal_config is None:
temporal_config = {}
logger.info("temporal_config is None. Initializing Helium backbone with default values")
if depformer_config is None:
depformer_config = {}
logger.info("depformer_config is None. Initializing depformer with default values")
if mimi_config is None:
mimi_config = {}
logger.info("mimi_config is None. Initializing Mimi codec config with default values")
self.temporal_config = self._coerce(temporal_config, HeliumConfig)
self.depformer_config = self._coerce(depformer_config, PersonaPlexDepformerConfig)
self.mimi_config = self._coerce(mimi_config, PersonaPlexMimiConfig)
super().__init__(**kwargs)
# The PersonaPlex checkpoint's config.json is empty (no ``architectures``);
# supply the primary (stage-0 talker) arch so vLLM's ModelConfig resolves a
# generate-capable model. The code2wav stage arch comes from the pipeline.
if not getattr(self, "architectures", None):
self.architectures = ["PersonaPlexTalkerForConditionalGeneration"]
self.text_vocab_size = text_vocab_size
self.text_embedding_rows = text_embedding_rows
self.audio_vocab_size = audio_vocab_size
self.num_audio_codebooks = num_audio_codebooks
# ``mimi_name`` is allowed both at the top level (convenient for
# hf_overrides / a flat config.json) and inside ``mimi_config``. Keep
# them in sync, preferring an explicit top-level value.
if mimi_name is not None:
self.mimi_config.mimi_name = mimi_name
self.mimi_name = self.mimi_config.mimi_name
@staticmethod
def _coerce(value: Any, config_cls: type[PretrainedConfig]) -> PretrainedConfig:
if isinstance(value, config_cls):
return value
if isinstance(value, dict):
return config_cls(**value)
raise TypeError(f"Expected dict or {config_cls.__name__}, got {type(value).__name__}")
@property
def sample_rate(self) -> int:
"""Output PCM sample rate in Hz (delegates to the Mimi config)."""
return int(self.mimi_config.sample_rate)
def get_text_config(self, **kwargs: Any) -> PretrainedConfig:
# vLLM expects the text config to expose hidden_size / num_attention_heads.
# For PersonaPlex the temporal-transformer (Helium) config is the text
# model config that drives the talker's vLLM Llama backbone.
return self.temporal_config
@@ -0,0 +1,548 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-native PersonaPlex Helium temporal transformer.
The module maps the Moshi temporal LM backbone onto vLLM's Llama-style decoder
components. It intentionally does not include Moshi's text/audio input
embeddings or depformer; PersonaPlex feeds temporal `inputs_embeds` directly.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from itertools import islice
import torch
from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
from vllm.model_executor.models.utils import (
PPMissingLayer,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from vllm_omni.model_executor.models.personaplex.configuration_helium import (
HeliumConfig,
)
__all__ = ["HeliumForCausalLM", "HeliumModel"]
class HeliumRMSNorm(RMSNorm):
"""Moshi-compatible RMSNorm.
Moshi stores the parameter as `alpha` with shape `[1, 1, hidden]` and
computes variance and the alpha multiply in fp32 before casting back to the
input dtype. vLLM's parameter is named `weight`; the loader squeezes alpha.
"""
def _forward_f32(self, x: torch.Tensor) -> torch.Tensor:
output_dtype = x.dtype
x_f32 = x.float()
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
x_f32 = x_f32 * torch.rsqrt(variance + self.variance_epsilon)
if self.has_weight:
x_f32 = x_f32 * self.weight.float()
return x_f32.to(output_dtype)
def forward(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if residual is None:
return self._forward_f32(x)
residual = residual + x
return self._forward_f32(residual), residual
class HeliumMLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if hidden_act != "silu":
raise ValueError(f"Unsupported activation: {hidden_act}. Only silu is supported.")
self.gate_up_proj = MergedColumnParallelLinear(
input_size=hidden_size,
output_sizes=[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
self.act_fn = SiluAndMul()
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.gate_up_proj(x)
x = self.act_fn(x)
x, _ = self.down_proj(x)
return x
def _get_helium_rope(config: HeliumConfig):
rope_kwargs = {
"head_size": config.head_dim,
"max_position": config.max_position_embeddings,
"is_neox_style": False,
"rope_parameters": {"base": config.rope_theta},
}
try:
return get_rope(**rope_kwargs)
except TypeError:
# vLLM 0.23 uses `rope_parameters`; this fallback is for older local
# source trees whose get_rope still takes rotary_dim/base/rope_scaling.
return get_rope(
head_size=config.head_dim,
rotary_dim=config.head_dim,
max_position=config.max_position_embeddings,
base=config.rope_theta,
is_neox_style=False,
)
class HeliumAttention(nn.Module):
def __init__(
self,
config: HeliumConfig,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
super().__init__()
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
assert self.total_num_kv_heads % tp_size == 0
else:
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.head_dim = config.head_dim
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.qkv_proj = QKVParallelLinear(
hidden_size=hidden_size,
head_size=self.head_dim,
total_num_heads=self.total_num_heads,
total_num_kv_heads=self.total_num_kv_heads,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
input_size=self.total_num_heads * self.head_dim,
output_size=hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = _get_helium_rope(config)
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=config.sliding_window,
prefix=f"{prefix}.attn",
attn_type=attn_type,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
class HeliumDecoderLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
config: HeliumConfig | None = None,
) -> None:
super().__init__()
config = config or vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.hidden_size = config.hidden_size
self.self_attn = HeliumAttention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=config.num_key_value_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
attn_type=AttentionType.DECODER,
)
self.mlp = HeliumMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = HeliumRMSNorm(
config.hidden_size,
eps=config.rms_norm_eps,
)
self.post_attention_layernorm = HeliumRMSNorm(
config.hidden_size,
eps=config.rms_norm_eps,
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
)
hidden_states, residual = self.post_attention_layernorm(
hidden_states,
residual,
)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
def _flatten_inputs(
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, tuple[int, int] | None]:
if hidden_states.dim() != 3:
return hidden_states, positions.reshape(-1), None
batch, seq_len, hidden = hidden_states.shape
flat_hidden = hidden_states.reshape(batch * seq_len, hidden)
if positions.dim() == 1:
if positions.numel() == seq_len:
flat_positions = positions.unsqueeze(0).expand(batch, seq_len).reshape(-1)
elif positions.numel() == batch * seq_len:
flat_positions = positions.reshape(-1)
else:
raise ValueError(
"1D positions must have length S or B*S when inputs_embeds is "
f"3D, got {positions.numel()} for B={batch}, S={seq_len}."
)
else:
flat_positions = positions.reshape(-1)
return flat_hidden, flat_positions, (batch, seq_len)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": 0,
"intermediate_tensors": 0,
"inputs_embeds": 0,
}
)
class HeliumModel(nn.Module):
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
layer_type: type[nn.Module] = HeliumDecoderLayer,
config: HeliumConfig | None = None,
) -> None:
super().__init__()
# ``config`` lets a composite model (e.g. the PersonaPlex talker) build the
# Helium backbone from its temporal sub-config while the engine's top-level
# hf_config is the composite PersonaPlexConfig. Defaults to the engine's.
config = config or vllm_config.model_config.hf_config
self.config = config
self.quant_config = vllm_config.quant_config
self.vocab_size = config.vocab_size
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: layer_type(vllm_config=vllm_config, prefix=prefix, config=config),
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
self.norm = HeliumRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"],
config.hidden_size,
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
raise ValueError(
"Helium temporal forward requires inputs_embeds; Moshi token "
"embeddings are intentionally not part of this vLLM module."
)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
unflatten_shape: tuple[int, int] | None = None
if get_pp_group().is_first_rank:
if inputs_embeds is None:
raise ValueError("HeliumModel.forward requires inputs_embeds.")
hidden_states, positions, unflatten_shape = _flatten_inputs(
inputs_embeds,
positions,
)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
positions = positions.reshape(-1)
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states, residual = layer(
positions,
hidden_states,
residual,
)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states, "residual": residual})
hidden_states, _ = self.norm(hidden_states, residual)
if unflatten_shape is not None:
batch, seq_len = unflatten_shape
hidden_states = hidden_states.reshape(batch, seq_len, -1)
return hidden_states
class HeliumForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
embedding_modules = {
"lm_head": "output_embeddings",
}
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
layer_type: type[nn.Module] = HeliumDecoderLayer,
) -> None:
super().__init__()
config = vllm_config.model_config.hf_config
self.config = config
self.quant_config = vllm_config.quant_config
self.model = HeliumModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
layer_type=layer_type,
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
return self.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
if hidden_states.dim() == 3:
batch, seq_len, hidden = hidden_states.shape
flat_hidden = hidden_states.reshape(batch * seq_len, hidden)
logits = self.logits_processor(self.lm_head, flat_hidden)
if logits is None:
return None
return logits.reshape(batch, seq_len, -1)
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(
self,
weights: Iterable[tuple[str, torch.Tensor]] | Mapping[str, torch.Tensor],
) -> set[str]:
if isinstance(weights, Mapping):
weights = weights.items()
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
mapped = self._load_moshi_weight(name, loaded_weight, params_dict)
loaded_params.update(mapped)
return loaded_params
def _load_moshi_weight(
self,
name: str,
loaded_weight: torch.Tensor,
params_dict: dict[str, nn.Parameter],
) -> set[str]:
if name == "out_norm.alpha":
return self._load_direct(
"model.norm.weight",
loaded_weight.squeeze(),
params_dict,
)
if name == "text_linear.weight":
return self._load_direct("lm_head.weight", loaded_weight, params_dict)
prefix = "transformer.layers."
if not name.startswith(prefix):
return set()
rest = name.removeprefix(prefix)
layer_index, _, suffix = rest.partition(".")
if not layer_index.isdigit() or not suffix:
return set()
base = f"model.layers.{layer_index}"
if suffix == "self_attn.in_proj_weight":
q_weight, k_weight, v_weight = loaded_weight.chunk(3, dim=0)
param_name = f"{base}.self_attn.qkv_proj.weight"
self._load_shard(param_name, q_weight, "q", params_dict)
self._load_shard(param_name, k_weight, "k", params_dict)
self._load_shard(param_name, v_weight, "v", params_dict)
return {param_name}
if suffix == "self_attn.out_proj.weight":
return self._load_direct(
f"{base}.self_attn.o_proj.weight",
loaded_weight,
params_dict,
)
if suffix == "gating.linear_in.weight":
gate_weight, up_weight = loaded_weight.chunk(2, dim=0)
param_name = f"{base}.mlp.gate_up_proj.weight"
self._load_shard(param_name, gate_weight, 0, params_dict)
self._load_shard(param_name, up_weight, 1, params_dict)
return {param_name}
if suffix == "gating.linear_out.weight":
return self._load_direct(
f"{base}.mlp.down_proj.weight",
loaded_weight,
params_dict,
)
if suffix == "norm1.alpha":
return self._load_direct(
f"{base}.input_layernorm.weight",
loaded_weight.squeeze(),
params_dict,
)
if suffix == "norm2.alpha":
return self._load_direct(
f"{base}.post_attention_layernorm.weight",
loaded_weight.squeeze(),
params_dict,
)
return set()
def _load_direct(
self,
name: str,
loaded_weight: torch.Tensor,
params_dict: dict[str, nn.Parameter],
) -> set[str]:
if name not in params_dict or is_pp_missing_parameter(name, self):
return set()
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
return {name}
def _load_shard(
self,
name: str,
loaded_weight: torch.Tensor,
shard_id: str | int,
params_dict: dict[str, nn.Parameter],
) -> None:
if name not in params_dict or is_pp_missing_parameter(name, self):
return
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
@@ -0,0 +1,433 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stage-1 Code2Wav model for PersonaPlex (Moshi finetune).
This is the ``LLM_GENERATION`` codec stage of the 2-stage PersonaPlex pipeline.
It consumes the per-frame audio codebooks produced by the talker (stage 0) and
the depformer (built by the lead), and turns them into 24 kHz PCM by calling the
external Mimi neural codec from the ``moshi`` package.
Layout contract (mirrors ``Qwen3TTSCode2Wav``):
* Per request, ``input_ids`` holds a flat codebook-major codec sequence
``[k * F]`` where ``k`` is the number of *active* codebooks Mimi decodes
(``num_codebooks``, i.e. the ``cb 0..7`` slice) and ``F`` is the number of
codec frames. The talker emits a 17-row token stack per frame; the input
processor (built by the lead) keeps only rows ``1:9`` (the 8 PCM-bearing audio
codebooks) and flattens them codebook-major before this stage.
* ``forward(...)`` returns an :class:`OmniOutput` whose ``multimodal_outputs``
carries ``{"model_outputs": [wav_per_request], "sr": [sr_per_request]}`` — the
exact shape ``Qwen3TTSCode2Wav`` returns, so the downstream audio packer is
unchanged.
The Mimi decoder is the transformers ``MimiModel`` (kyutai/mimi weights), not
in vLLM's safetensors loader, so they are loaded eagerly in ``load_weights``
part of the vLLM weights iterator (the codec owns its own checkpoint; see the
experimental fullduplex package for the same pattern).
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
_MIMI_DECODE_BATCH_FRAMES = 5
def _codec_ids_from_payload_or_input(
input_ids: torch.Tensor,
runtime_info: Mapping[str, Any] | None,
) -> torch.Tensor:
"""Prefer connector-delivered codec ids over scheduler placeholders."""
if isinstance(runtime_info, Mapping):
codes = runtime_info.get("codes")
if isinstance(codes, Mapping):
audio = codes.get("audio")
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
return audio.reshape(-1).to(device=input_ids.device, dtype=torch.long)
if isinstance(audio, (list, tuple)) and audio:
return torch.as_tensor(audio, device=input_ids.device, dtype=torch.long).reshape(-1)
return input_ids.reshape(-1).to(dtype=torch.long)
class PersonaPlexCode2Wav(nn.Module):
"""Stage-1 Code2Wav model for PersonaPlex (GenerationModelRunner).
Wraps the external Mimi decoder. The vLLM generation runner only calls a
handful of methods on this module: ``embed_input_ids`` (dummy embeddings),
``compute_logits`` (none -- this stage never samples), ``forward`` (the
actual codec->PCM decode), ``make_omni_output`` (output normalization), and
``load_weights`` (eager Mimi construction).
"""
input_modalities = "audio"
requires_request_ids = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
self.config = vllm_config.model_config.hf_config
# Runner-facing capability flags, matching Qwen3TTSCode2Wav so the
# GenerationModelRunner drives this stage the same way.
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
mimi_cfg = getattr(self.config, "mimi_config", None)
# Number of *active* codebooks Mimi decodes to PCM (cb 0..7).
self._num_codebooks = int(getattr(mimi_cfg, "num_codebooks", 8))
self._output_sample_rate = int(getattr(mimi_cfg, "sample_rate", 24000))
self._samples_per_frame = int(getattr(mimi_cfg, "samples_per_frame", 1920))
self._mimi_name = getattr(mimi_cfg, "mimi_name", None) or getattr(self.config, "mimi_name", None)
self._max_codec_sessions = int(getattr(vllm_config.model_config, "duplex_max_sessions", 1))
# The Mimi module is constructed in load_weights() (it owns its own
# weight format) and assigned here so vLLM's memory profiler can see it.
self.mimi: nn.Module | None = None
self._additional_mimi = nn.ModuleList()
self._mimi_device: torch.device | None = None
self._request_codes: dict[str, torch.Tensor] = {}
self._request_codec_slots: dict[str, int] = {}
# ------------------------------------------------------------------
# Runner-facing no-op / placeholder hooks (mirror Qwen3TTSCode2Wav).
# ------------------------------------------------------------------
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
# This stage ignores token embeddings; keep a stable dummy embedding so
# the vLLM runner has a valid hidden-state placeholder.
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(
self,
hidden_states: torch.Tensor | OmniOutput,
sampling_metadata: Any = None,
) -> None:
# Code2Wav never samples; the runner short-circuits when logits are None.
return None
def get_dummy_runtime_additional_information(self, num_reqs: int) -> list[dict[str, object]]:
return [{"meta": {"personaplex_dummy_profile": True}} for _ in range(num_reqs)]
def _split_request_ids(
self,
ids: torch.Tensor,
seq_token_counts: list[int] | None = None,
) -> list[torch.Tensor]:
"""Split concatenated ``input_ids`` into per-request segments.
Uses ``seq_token_counts`` (injected by the runner via model_kwargs) when
available, falling back to forward-context ``ubatch_slices`` when
micro-batching is active. Returns ``[ids]`` for single-request batches.
Mirrors ``Qwen3TTSCode2Wav._split_request_ids``.
"""
if seq_token_counts is not None and len(seq_token_counts) > 1:
boundaries = [0]
for count in seq_token_counts:
boundaries.append(boundaries[-1] + count)
n = ids.numel()
return [ids[boundaries[i] : min(boundaries[i + 1], n)] for i in range(len(seq_token_counts))]
if is_forward_context_available():
slices = get_forward_context().ubatch_slices
if slices is not None and len(slices) > 1 and not any(hasattr(s, "token_slice") for s in slices):
boundaries = [0]
for s in slices:
boundaries.append(boundaries[-1] + s)
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
return [ids]
# ------------------------------------------------------------------
# Decode.
# ------------------------------------------------------------------
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
"""Decode flat codebook-major codec ids into PCM via Mimi.
``input_ids`` per request is ``[k * F]`` (codebook-major), where ``k``
is ``self._num_codebooks``. The connector may send either a new delta
chunk or the cumulative prefix of a resumable request. Request-local
code history identifies the new suffix, and the streaming Mimi decoder
consumes each new frame exactly once.
"""
sr_val = int(self._output_sample_rate)
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
empty = torch.zeros((0,), dtype=torch.float32)
if input_ids is None or input_ids.numel() == 0:
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
)
ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
num_req = len(request_ids_list)
state_ids = self._resolve_request_ids(
num_req,
kwargs.get("request_ids"),
runtime_additional_information,
)
if self.mimi is None:
raise RuntimeError("PersonaPlexCode2Wav.forward called before Mimi was loaded in load_weights().")
k = int(self._num_codebooks)
device = self._mimi_device or ids.device
audios: list[torch.Tensor] = [empty] * num_req
srs = [sr_tensor] * num_req
for i, req_ids in enumerate(request_ids_list):
runtime_info = (
runtime_additional_information[i]
if runtime_additional_information is not None and i < len(runtime_additional_information)
else None
)
meta = runtime_info.get("meta") if isinstance(runtime_info, Mapping) else None
if isinstance(meta, Mapping) and meta.get("personaplex_dummy_profile") is True:
continue
flat = _codec_ids_from_payload_or_input(req_ids, runtime_info)
n = int(flat.numel())
if n == 0 or n % k != 0:
if n > 0:
logger.warning(
"PersonaPlex Code2Wav input_ids length %d not divisible by "
"num_codebooks %d; skipping malformed request.",
n,
k,
)
continue
frames = n // k
codes_kf = flat.reshape(k, frames)
state_id = state_ids[i]
delta_kf = self._new_code_suffix(state_id, codes_kf)
if delta_kf.shape[1] == 0:
continue
wav = self._decode_streaming_frames(state_id, delta_kf.to(device=device))
if wav.numel() > 0:
audios[i] = wav.to(dtype=torch.float32).reshape(-1)
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": srs},
)
@staticmethod
def _resolve_request_ids(
count: int,
request_ids: object,
runtime_additional_information: list[dict[str, Any]] | None,
) -> list[str | None]:
if isinstance(request_ids, list | tuple):
if len(request_ids) != count:
raise ValueError(
"PersonaPlex Code2Wav request id count does not match inputs: "
f"request_ids={len(request_ids)}, inputs={count}"
)
return [str(request_id) for request_id in request_ids]
infos = runtime_additional_information or []
resolved: list[str | None] = []
for index in range(count):
info = infos[index] if index < len(infos) else None
request_id = None
if isinstance(info, Mapping):
request_id = info.get("request_id")
if request_id is None:
meta = info.get("meta")
if isinstance(meta, Mapping):
request_id = meta.get("request_id")
resolved.append(str(request_id) if request_id is not None else None)
return resolved
def _new_code_suffix(self, request_id: str | None, codes_kf: torch.Tensor) -> torch.Tensor:
if request_id is None:
return codes_kf
incoming = codes_kf.detach().to(device="cpu", dtype=torch.long)
previous = self._request_codes.get(request_id)
if (
previous is not None
and incoming.shape[1] >= previous.shape[1]
and torch.equal(incoming[:, : previous.shape[1]], previous)
):
delta = incoming[:, previous.shape[1] :]
self._request_codes[request_id] = incoming
return delta
self._request_codes[request_id] = incoming if previous is None else torch.cat([previous, incoming], dim=1)
return incoming
def _decode_streaming_frames(self, request_id: str | None, codes_kf: torch.Tensor) -> torch.Tensor:
codec, ephemeral = self._codec_for_request(request_id)
if not hasattr(codec, "decode_frame"):
raise RuntimeError("PersonaPlex Code2Wav requires a streaming Mimi decoder")
decode_frames = getattr(codec, "decode_frames", None)
chunks: list[torch.Tensor] = []
for start in range(0, codes_kf.shape[1], _MIMI_DECODE_BATCH_FRAMES):
frame_batch = codes_kf[:, start : start + _MIMI_DECODE_BATCH_FRAMES]
if frame_batch.shape[1] > 1 and callable(decode_frames):
chunks.append(self._flatten_wav(decode_frames(frame_batch.unsqueeze(0))))
else:
chunks.extend(
self._flatten_wav(codec.decode_frame(frame_batch[:, frame].unsqueeze(0)))
for frame in range(frame_batch.shape[1])
)
wav = torch.cat(chunks, dim=0) if chunks else codes_kf.new_empty(0, dtype=torch.float32)
if ephemeral:
codec.reset_streaming()
return wav
def _set_mimi_codecs(self, codecs: list[nn.Module]) -> None:
if not codecs:
raise ValueError("PersonaPlex Code2Wav requires at least one Mimi decoder")
self.mimi = codecs[0]
self._additional_mimi = nn.ModuleList(codecs[1:])
self._request_codec_slots.clear()
def _mimi_codecs(self) -> list[nn.Module]:
if self.mimi is None:
return []
return [self.mimi, *self._additional_mimi]
def _codec_for_request(self, request_id: str | None) -> tuple[nn.Module, bool]:
codecs = self._mimi_codecs()
if not codecs:
raise RuntimeError("PersonaPlexCode2Wav.forward called before Mimi was loaded in load_weights().")
occupied = set(self._request_codec_slots.values())
if request_id is not None:
slot = self._request_codec_slots.get(request_id)
if slot is None:
slot = next((index for index in range(len(codecs)) if index not in occupied), None)
if slot is None:
raise RuntimeError(f"PersonaPlex Code2Wav decoder capacity {len(codecs)} is exhausted")
codecs[slot].streaming_init(1)
self._request_codec_slots[request_id] = slot
return codecs[slot], False
slot = next((index for index in range(len(codecs)) if index not in occupied), None)
if slot is None:
raise RuntimeError(f"PersonaPlex Code2Wav decoder capacity {len(codecs)} is exhausted")
codecs[slot].streaming_init(1)
return codecs[slot], True
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
for request_id in finished_req_ids:
state_id = str(request_id)
self._request_codes.pop(state_id, None)
slot = self._request_codec_slots.pop(state_id, None)
codecs = self._mimi_codecs()
if slot is not None and slot < len(codecs):
codecs[slot].reset_streaming()
@staticmethod
def _flatten_wav(wav: torch.Tensor) -> torch.Tensor:
"""Normalize Mimi's decode output to a 1D PCM tensor.
Mimi may return ``[B, samples]`` or ``[B, C, samples]`` depending on the
moshi version; PersonaPlex is mono (``C == 1``), so collapse to 1D.
"""
if wav.dim() == 3:
# [B, C, T] -> take batch 0, channel 0.
return wav[0, 0]
if wav.dim() == 2:
# [B, T] -> batch 0.
return wav[0]
return wav.reshape(-1)
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
return OmniOutput(*model_outputs)
if not (isinstance(model_outputs, tuple) and len(model_outputs) == 2):
raise TypeError(
"PersonaPlexCode2Wav expected OmniOutput, OmniOutput tuple, "
f"or (audio_tensor, sr) outputs, got {type(model_outputs)}"
)
audio_tensor, sr = model_outputs
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"model_outputs": audio_tensor,
"sr": sr,
},
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Construct and load the external Mimi decoder.
The primary vLLM weights iterator carries no Code2Wav parameters (Mimi
owns its own checkpoint format), so it is drained and the Mimi module is
built from the moshi package's loader.
"""
# Drain the primary iterator so callers don't hang on an unconsumed
# generator; none of these weights belong to this stage.
for _ in weights:
pass
device = self.vllm_config.device_config.device
from vllm_omni.model_executor.models.personaplex.personaplex_mimi import (
PersonaPlexMimiCodec,
)
checkpoint = Path(self.model_path) / (self._mimi_name or "tokenizer-e351c8d8-checkpoint125.safetensors")
codecs = [
PersonaPlexMimiCodec(
checkpoint=str(checkpoint) if checkpoint.is_file() else None,
device=str(device),
).eval()
for _ in range(self._max_codec_sessions)
]
self._set_mimi_codecs(codecs)
self._mimi_device = torch.device(str(device))
reported_sr = getattr(codecs[0].model.config, "sampling_rate", None)
if reported_sr is not None:
self._output_sample_rate = int(reported_sr)
bytes_per_mib = 1024**2
per_slot_weight_mib = (
sum(parameter.numel() * parameter.element_size() for parameter in codecs[0].parameters()) / bytes_per_mib
)
total_weight_mib = (
sum(parameter.numel() * parameter.element_size() for codec in codecs for parameter in codec.parameters())
/ bytes_per_mib
)
logger.info(
"PersonaPlex Code2Wav loaded %d Mimi stream(s): "
"per_slot_weight_mib=%.2f total_weight_mib=%.2f "
"num_codebooks=%d sample_rate=%d samples_per_frame=%d",
len(codecs),
per_slot_weight_mib,
total_weight_mib,
self._num_codebooks,
self._output_sample_rate,
self._samples_per_frame,
)
return {name for name, _ in self.named_parameters() if name.startswith(("mimi.", "_additional_mimi."))}
@@ -0,0 +1,366 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PersonaPlex depformer: the per-step audio code predictor.
The depformer is the small autoregressive head that, conditioned on one temporal
hidden state, predicts the ``dep_q`` audio codebooks of a single Mimi frame. It
is a faithful eager port of Moshi's depformer (``moshi.models.lm`` +
``moshi.modules.transformer``), kept deliberately outside vLLM's paged-attention
engine: its attention KV is reset every frame and only spans the ``dep_q`` inner
steps, so paging buys nothing here. This mirrors how the existing TTS code
predictors live alongside, not inside, the AR talker.
Faithful-port details (all measured from ``nvidia/personaplex-7b-v1``):
* **Per-step weights** (``weights_per_step``): each inner step ``t`` uses its own
attention in/out projection and its own gating MLP, selected by index (Moshi's
``multi_linear``). There are ``dep_q`` weight sets per layer.
* **No positional embedding** (``depformer_pos_emb="none"``): no RoPE, no sin.
* **Causal attention over all inner steps**: Moshi forces the depformer's
``context`` to None, so each step attends to every prior inner step (KV
capacity == ``weights_per_step``); the KV cache is rebuilt every temporal frame.
* **fp32 RMSNorm** (``rms_norm_f32``, eps 1e-8) with the weight stored as
``alpha`` of shape ``[1, 1, dim]``.
* **SiLU gating** MLP (Moshi ``ActivationGating``): ``linear_in`` projects to
``2 * hidden``; the halves are combined as ``silu(a) * b`` then ``linear_out``.
* **ScaledEmbedding** with ``zero_idx = -1`` (the special token maps to the zero
vector); audio table is ``card + 1`` rows, text table ``text_card + 1``.
The checkpoint ships ``dep_q = 8`` weight sets; the production loader (Moshi's
``get_moshi_lm``) overrides ``dep_q = 16`` and fills sets 8..15 by copying 0..7.
:meth:`PersonaPlexDepformer.load_weights` replicates that expansion so the module
loads directly from the raw ``model.safetensors`` as well as from an
already-expanded Moshi instance.
"""
from __future__ import annotations
from collections.abc import Callable, Iterable, Mapping
import torch
import torch.nn.functional as F
from torch import nn
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
PersonaPlexDepformerConfig,
)
__all__ = ["PersonaPlexDepformer"]
# Moshi's special "no token / zero embedding" sentinel (``LMModel.zero_token_id``).
_ZERO_IDX = -1
# A builder takes the source state dict and returns the tensor for one target
# param, or None when the source lacks it.
_Builder = Callable[[dict], "torch.Tensor | None"]
def _rms_norm_f32(x: torch.Tensor, alpha: torch.Tensor, eps: float) -> torch.Tensor:
"""Moshi ``_rms_norm`` with the fp32 accumulation path.
Variance and the alpha multiply run in float32 before casting back, matching
``moshi.modules.transformer._rms_norm`` exactly (``dtype=torch.float``).
"""
x_dtype = x.dtype
x_f32 = x.float()
var = eps + x_f32.pow(2).mean(dim=-1, keepdim=True)
y = x_f32 * (alpha.float() * torch.rsqrt(var))
return y.to(x_dtype)
class _ScaledEmbedding(nn.Embedding):
"""``nn.Embedding`` matching Moshi's ``ScaledEmbedding`` (no norm, zero_idx).
Inputs equal to ``zero_idx`` produce the zero vector; all other inputs are
clamped to ``>= 0`` before the lookup. ``norm_emb`` is False for PersonaPlex,
so no embedding norm is applied.
"""
def __init__(self, num_embeddings: int, embedding_dim: int, zero_idx: int = _ZERO_IDX) -> None:
super().__init__(num_embeddings, embedding_dim)
self.zero_idx = zero_idx
def forward(self, idx: torch.Tensor) -> torch.Tensor: # type: ignore[override]
is_zero = idx == self.zero_idx
y = super().forward(idx.clamp(min=0))
zero = torch.zeros(1, dtype=y.dtype, device=y.device)
return torch.where(is_zero[..., None], zero, y)
class _DepformerLayer(nn.Module):
"""One depformer transformer layer with per-step attention + gating.
Attention and gating weights carry a leading ``dep_q`` axis; the active inner
step selects the slice (Moshi ``multi_linear`` / ``gating[t]``). KV is supplied
per call so the caller controls the per-frame reset and the sliding window.
"""
def __init__(self, config: PersonaPlexDepformerConfig) -> None:
super().__init__()
dim = config.hidden_size
self.dim = dim
self.num_heads = config.num_attention_heads
self.head_dim = config.head_dim
self.dep_q = config.dep_q
self.eps = config.rms_norm_eps
# Per-step fused QKV and output projections, stored exactly as Moshi does:
# ``in_proj_weight`` is [dep_q * 3*dim, dim]; ``out_proj`` is [dep_q*dim, dim].
self.in_proj_weight = nn.Parameter(torch.empty(self.dep_q * 3 * dim, dim))
self.out_proj_weight = nn.Parameter(torch.empty(self.dep_q * dim, dim))
# fp32 RMSNorm weights (Moshi ``alpha``, shape [1, 1, dim]).
self.norm1_alpha = nn.Parameter(torch.ones(1, 1, dim))
self.norm2_alpha = nn.Parameter(torch.ones(1, 1, dim))
# Per-step SiLU gating MLP. ``linear_in`` -> [2*hidden, dim]; ``linear_out`` -> [dim, hidden].
hidden = config.intermediate_size
self.gating_in = nn.Parameter(torch.empty(self.dep_q, 2 * hidden, dim))
self.gating_out = nn.Parameter(torch.empty(self.dep_q, dim, hidden))
def _in_proj(self, step: int) -> torch.Tensor:
return self.in_proj_weight.view(self.dep_q, 3 * self.dim, self.dim)[step]
def _out_proj(self, step: int) -> torch.Tensor:
return self.out_proj_weight.view(self.dep_q, self.dim, self.dim)[step]
def forward(
self,
x: torch.Tensor,
step: int,
kv: dict[str, torch.Tensor | None],
) -> torch.Tensor:
"""Run inner step ``step`` for a ``[B, 1, dim]`` input, updating ``kv`` in place."""
# --- self-attention (per-step weights, causal sliding window, no RoPE) ---
residual = x
h = _rms_norm_f32(x, self.norm1_alpha, self.eps).squeeze(1) # [B, dim]
qkv = F.linear(h, self._in_proj(step)) # [B, 3*dim]
q, k, v = qkv.split(self.dim, dim=-1)
b = q.shape[0]
q = q.view(b, self.num_heads, 1, self.head_dim)
k = k.view(b, self.num_heads, 1, self.head_dim)
v = v.view(b, self.num_heads, 1, self.head_dim)
if kv["k"] is None:
k_hist, v_hist = k, v
else:
k_hist = torch.cat([kv["k"], k], dim=2)
v_hist = torch.cat([kv["v"], v], dim=2)
kv["k"], kv["v"] = k_hist, v_hist
# Moshi forces the depformer's ``context`` to None (lm.py: kwargs_dep[
# "context"] = None), so attention spans ALL prior inner steps -- the KV
# capacity is ``weights_per_step`` (== dep_q), never windowed. The query is
# the newest step and every key is causal w.r.t. it, so no mask is needed.
attn = F.scaled_dot_product_attention(q, k_hist, v_hist) # [B, H, 1, Dh]
attn = attn.reshape(b, self.dim)
out = F.linear(attn, self._out_proj(step)).unsqueeze(1) # [B, 1, dim]
x = residual + out
# --- feed-forward (per-step SiLU gating) ---
residual = x
h = _rms_norm_f32(x, self.norm2_alpha, self.eps).squeeze(1) # [B, dim]
gated = F.linear(h, self.gating_in[step]) # [B, 2*hidden]
a, b_half = gated.chunk(2, dim=-1)
update = F.linear(F.silu(a) * b_half, self.gating_out[step]).unsqueeze(1)
return residual + update
class PersonaPlexDepformer(nn.Module):
"""Moshi depformer as a standalone per-frame code predictor.
Call :meth:`forward` once per temporal frame with the frame's text token and
temporal hidden state; it runs ``dep_q`` inner AR steps (greedy by default,
matching Moshi's ``use_sampling=False``) and returns the ``dep_q`` audio codes.
"""
def __init__(
self,
config: PersonaPlexDepformerConfig,
temporal_hidden_size: int,
text_card: int = 32000,
) -> None:
super().__init__()
self.config = config
self.dep_q = config.dep_q
self.card = config.card
dim = config.hidden_size
# Per-codebook projection of the temporal hidden state (``depformer_multi_linear``).
self.depformer_in = nn.ModuleList([nn.Linear(temporal_hidden_size, dim, bias=False) for _ in range(self.dep_q)])
# Step-0 conditions on the text token; steps 1..dep_q-1 on the previous audio code.
self.depformer_text_emb = _ScaledEmbedding(text_card + 1, dim)
self.depformer_emb = nn.ModuleList([_ScaledEmbedding(self.card + 1, dim) for _ in range(self.dep_q - 1)])
self.layers = nn.ModuleList([_DepformerLayer(config) for _ in range(config.num_hidden_layers)])
# Per-codebook output heads (Moshi ``linears``).
self.linears = nn.ModuleList([nn.Linear(dim, self.card, bias=False) for _ in range(self.dep_q)])
@torch.inference_mode()
def forward(
self,
text_token: torch.Tensor,
transformer_out: torch.Tensor,
audio_tokens: torch.Tensor | None = None,
audio_provided: torch.Tensor | None = None,
return_logits: bool = False,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Predict the ``dep_q`` audio codes for one frame.
Args:
text_token: ``[B]`` sampled text token for the frame.
transformer_out: ``[B, 1, temporal_hidden]`` temporal hidden state.
audio_tokens: optional ``[B, dep_q]`` teacher-forcing codes.
audio_provided: optional ``[B, dep_q]`` bool mask; where True the
corresponding ``audio_tokens`` entry overrides the sampled code as
the next-step input (partial teacher forcing, as in Moshi).
return_logits: also return the per-step logits ``[B, dep_q, card]``.
Returns:
``[B, dep_q]`` long tensor of sampled (greedy) audio codes, plus the
per-step logits when ``return_logits`` is set.
"""
if transformer_out.dim() != 3 or transformer_out.shape[1] != 1:
raise ValueError(f"transformer_out must be [B, 1, H]; got {tuple(transformer_out.shape)}")
prev = text_token
kv = [{"k": None, "v": None} for _ in self.layers]
codes: list[torch.Tensor] = []
logits_all: list[torch.Tensor] = []
for step in range(self.dep_q):
x = self.depformer_in[step](transformer_out) # [B, 1, dim]
if step == 0:
cond = self.depformer_text_emb(prev.unsqueeze(1))
else:
cond = self.depformer_emb[step - 1](prev.unsqueeze(1))
x = x + cond
for li, layer in enumerate(self.layers):
x = layer(x, step, kv[li])
logits = self.linears[step](x).squeeze(1) # [B, card]
sampled = logits.float().argmax(dim=-1) # greedy [B]
codes.append(sampled)
if return_logits:
logits_all.append(logits.float())
if audio_provided is not None and audio_tokens is not None:
prev = torch.where(audio_provided[:, step], audio_tokens[:, step], sampled)
else:
prev = sampled
stacked = torch.stack(codes, dim=1) # [B, dep_q]
if return_logits:
return stacked, torch.stack(logits_all, dim=1) # [B, dep_q, card]
return stacked
# ------------------------------------------------------------------
# Weight loading
# ------------------------------------------------------------------
def load_weights(
self,
weights: Iterable[tuple[str, torch.Tensor]] | Mapping[str, torch.Tensor],
) -> set[str]:
"""Load Moshi depformer weights, expanding ``dep_q`` 8 -> 16 if needed.
Accepts the raw checkpoint names (``depformer.*``, ``depformer_in.N``,
``depformer_emb.N``, ``depformer_text_emb``, ``linears.N``). Per-step
modules present only for steps 0..7 are copied to 8..15, and the fused
attention projections are tiled, exactly as Moshi's ``get_moshi_lm`` does.
"""
src = dict(weights.items() if isinstance(weights, Mapping) else weights)
src = {k: v for k, v in src.items() if self._is_depformer_key(k)}
params = dict(self.named_parameters())
loaded: set[str] = set()
for tgt, build in self._weight_plan().items():
tensor = build(src)
if tensor is None:
continue
param = params[tgt]
if tensor.shape != param.shape:
raise ValueError(
f"shape mismatch for {tgt}: checkpoint {tuple(tensor.shape)} vs param {tuple(param.shape)}"
)
param.data.copy_(tensor.to(param.dtype))
loaded.add(tgt)
return loaded
@staticmethod
def _is_depformer_key(name: str) -> bool:
return name.startswith(("depformer.", "depformer_in.", "depformer_emb.", "depformer_text_emb", "linears."))
def _weight_plan(self) -> dict[str, _Builder]:
"""Map each of this module's params to a builder over the source state dict."""
dim = self.config.hidden_size
def step_or_copy(prefix: str, step: int, suffix: str = "weight") -> _Builder:
# cb 8..15 fall back to cb (step-8) when absent (Moshi copy_missing_weights).
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
key = f"{prefix}.{step}.{suffix}"
if key in src:
return src[key]
if step >= 8:
return src.get(f"{prefix}.{step - 8}.{suffix}")
return None
return build
def expand_attn(layer: int, kind: str, per_step_rows: int) -> _Builder:
# ``in_proj_weight`` [steps*per_step_rows, dim] / ``out_proj.weight`` likewise;
# tile an 8-step checkpoint block up to dep_q (Moshi concat-copy expansion).
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
key = f"depformer.layers.{layer}.self_attn.{kind}"
t = src.get(key)
if t is None:
return None
steps = t.shape[0] // per_step_rows
if steps < self.dep_q:
reps = (self.dep_q + steps - 1) // steps
t = t.repeat(reps, 1)[: self.dep_q * per_step_rows]
return t
return build
plan: dict[str, _Builder] = {}
# depformer_in / linears / depformer_emb / text_emb
for step in range(self.dep_q):
plan[f"depformer_in.{step}.weight"] = step_or_copy("depformer_in", step)
plan[f"linears.{step}.weight"] = step_or_copy("linears", step)
plan["depformer_text_emb.weight"] = lambda src: src.get("depformer_text_emb.weight")
for step in range(self.dep_q - 1):
plan[f"depformer_emb.{step}.weight"] = step_or_copy("depformer_emb", step)
# per-layer attention + norms + gating
for layer in range(self.config.num_hidden_layers):
lp = f"depformer.layers.{layer}"
plan[f"layers.{layer}.in_proj_weight"] = expand_attn(layer, "in_proj_weight", 3 * dim)
plan[f"layers.{layer}.out_proj_weight"] = expand_attn(layer, "out_proj.weight", dim)
plan[f"layers.{layer}.norm1_alpha"] = self._reshape_alpha(f"{lp}.norm1.alpha")
plan[f"layers.{layer}.norm2_alpha"] = self._reshape_alpha(f"{lp}.norm2.alpha")
plan[f"layers.{layer}.gating_in"] = self._stack_gating(lp, "linear_in")
plan[f"layers.{layer}.gating_out"] = self._stack_gating(lp, "linear_out")
return plan
@staticmethod
def _reshape_alpha(key: str) -> _Builder:
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
t = src.get(key)
return None if t is None else t.reshape(1, 1, -1)
return build
def _stack_gating(self, layer_prefix: str, which: str) -> _Builder:
"""Stack the dep_q per-step gating weights into one [dep_q, ...] tensor.
Steps 8..15 fall back to step (n-8) when the checkpoint only ships 8
(Moshi ``copy_missing_weights``).
"""
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
slabs: list[torch.Tensor] = []
for step in range(self.dep_q):
key = f"{layer_prefix}.gating.{step}.{which}.weight"
if key not in src and step >= 8:
key = f"{layer_prefix}.gating.{step - 8}.{which}.weight"
t = src.get(key)
if t is None:
return None
slabs.append(t)
return torch.stack(slabs, dim=0)
return build
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PersonaPlex input embeddings (Moshi ``embed_codes``).
The temporal transformer (:class:`HeliumForCausalLM`) consumes precomputed
``inputs_embeds`` rather than token ids. Those embeddings come from Moshi's
``embed_codes``: a per-frame ``[B, 1 + n_q, S]`` token stack (row 0 = text,
rows 1..n_q = audio codebooks) is mapped through one text embedding plus ``n_q``
audio-codebook embeddings and summed into ``[B, S, hidden]``.
This module owns exactly those embedding tables (Moshi ``text_emb`` and
``emb.0..n_q-1``), which ``modeling_helium`` deliberately omits. The talker
(Task 2) uses it to build the temporal ``inputs_embeds`` for both the system /
voice-prompt prefill and the per-frame decode step.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
import torch
from torch import nn
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
PersonaPlexConfig,
)
from vllm_omni.model_executor.models.personaplex.personaplex_depformer import (
_ScaledEmbedding,
)
__all__ = ["PersonaPlexInputEmbeddings"]
class PersonaPlexInputEmbeddings(nn.Module):
"""Moshi ``embed_codes``: summed text + audio-codebook embeddings.
Args:
config: the top-level PersonaPlex config (supplies the temporal hidden
size, the audio cardinality / codebook count, and the text table size).
"""
def __init__(self, config: PersonaPlexConfig) -> None:
super().__init__()
dim = config.temporal_config.hidden_size
self.num_audio_codebooks = config.num_audio_codebooks # n_q (16)
# Text table has one extra row for the special token (text_card + 1).
self.text_emb = _ScaledEmbedding(config.text_embedding_rows, dim)
# Each audio table is card + 1 rows; ``embed_codes`` uses all n_q codebooks.
self.audio_emb = nn.ModuleList(
[_ScaledEmbedding(config.audio_vocab_size + 1, dim) for _ in range(self.num_audio_codebooks)]
)
def forward(self, sequence: torch.Tensor) -> torch.Tensor:
"""Map a ``[B, 1 + n_q, S]`` token stack to ``[B, S, hidden]``.
Row 0 is the text stream; rows ``1..n_q`` are the audio codebooks (in
Moshi's layout the audio rows carry both the agent and user streams).
"""
if sequence.dim() != 3 or sequence.shape[1] != self.num_audio_codebooks + 1:
raise ValueError(f"sequence must be [B, {self.num_audio_codebooks + 1}, S]; got {tuple(sequence.shape)}")
emb = self.text_emb(sequence[:, 0]) # [B, S, hidden]
for cb in range(self.num_audio_codebooks):
emb = emb + self.audio_emb[cb](sequence[:, cb + 1])
return emb
def load_weights(
self,
weights: Iterable[tuple[str, torch.Tensor]] | Mapping[str, torch.Tensor],
) -> set[str]:
"""Load Moshi ``text_emb`` and ``emb.0..n_q-1`` embedding tables."""
src = dict(weights.items() if isinstance(weights, Mapping) else weights)
params = dict(self.named_parameters())
loaded: set[str] = set()
plan: dict[str, str] = {"text_emb.weight": "text_emb.weight"}
for cb in range(self.num_audio_codebooks):
plan[f"audio_emb.{cb}.weight"] = f"emb.{cb}.weight"
for tgt, key in plan.items():
if key not in src:
continue
param = params[tgt]
tensor = src[key]
if tensor.shape != param.shape:
raise ValueError(
f"shape mismatch for {tgt}: checkpoint {tuple(tensor.shape)} vs param {tuple(param.shape)}"
)
param.data.copy_(tensor.to(param.dtype))
loaded.add(tgt)
return loaded
@@ -0,0 +1,413 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Streaming Mimi codec for PersonaPlex (moshi-free).
Frame-clocked duplex needs a codec that encodes/decodes exactly one 80 ms frame
per call with state carried across the conversation. The ``moshi`` package
provided that; this module removes the dependency by combining:
- **transformers ``MimiModel``** (``kyutai/mimi`` — the same checkpoint family
PersonaPlex ships) for the quantizer and every SEANet conv weight. Its
streaming support only covers the encoder convs, so conv streaming is done
here instead, uniformly.
- **Our own streaming wrappers** mirroring the Moshi reference semantics (MIT),
verified against recorded reference outputs:
* ``Conv1d``: left-context carry of ``effective_kernel - stride`` samples,
zero-initialized at stream start (``pad_mode="constant"``).
* ``ConvTranspose1d``: overlap-add tail carry of ``kernel - stride`` output
samples, with the double-counted bias subtracted on merge.
* transformer: the encoder/decoder transformers use a 250-position sliding
context over a ring KV with absolute-offset RoPE. Hugging Face's cache path
diverges once the window engages (position 250), so the transformers are
reimplemented here on the same ring-KV design as
``personaplex_temporal.py`` and loaded directly from the PersonaPlex
checkpoint's fused layout (LayerNorm + per-layer LayerScale + GELU FFN).
All per-stream state is ``[B, ...]`` with per-row reset (``reset_slot``), so the
codec composes with elastic slot recycling in batched duplex serving.
"""
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm_omni.model_executor.models.personaplex.personaplex_temporal import (
_apply_rope,
_RingKV,
)
DEFAULT_HF_REPO = "kyutai/mimi"
FRAME_SIZE = 1920
CODEBOOKS = 8
def _map_moshi_codec_weights(
state_dict: dict[str, torch.Tensor],
) -> dict[str, torch.Tensor]:
"""Map bundled Moshi codec keys to ``transformers.MimiModel`` keys."""
mapped: dict[str, torch.Tensor] = {}
transformer_prefixes = ("encoder_transformer.", "decoder_transformer.")
for name, tensor in state_dict.items():
if name.startswith(transformer_prefixes):
continue
if name.startswith("encoder.model."):
target = name.replace("encoder.model.", "encoder.layers.", 1)
target = target.replace(".conv.conv.", ".conv.")
elif name.startswith("decoder.model."):
target = name.replace("decoder.model.", "decoder.layers.", 1)
target = target.replace(".convtr.convtr.", ".conv.")
target = target.replace(".conv.conv.", ".conv.")
elif name.startswith("downsample.conv.conv.conv."):
target = name.replace("downsample.conv.conv.conv.", "downsample.conv.", 1)
elif name.startswith("upsample.convtr.convtr.convtr."):
target = name.replace("upsample.convtr.convtr.convtr.", "upsample.conv.", 1)
elif name.startswith("quantizer.rvq_first."):
target = name.replace(
"quantizer.rvq_first.",
"quantizer.semantic_residual_vector_quantizer.",
1,
)
elif name.startswith("quantizer.rvq_rest."):
target = name.replace(
"quantizer.rvq_rest.",
"quantizer.acoustic_residual_vector_quantizer.",
1,
)
else:
raise KeyError(f"unrecognized PersonaPlex Mimi checkpoint key: {name}")
target = target.replace(".vq.layers.", ".layers.")
target = target.replace("._codebook._initialized", ".codebook.initialized")
target = target.replace("._codebook.cluster_usage", ".codebook.cluster_usage")
target = target.replace("._codebook.embedding_sum", ".codebook.embed_sum")
mapped[target] = tensor
return mapped
class _StreamConv1d:
"""Moshi ``RawStreamingConv1d``: causal left-context carry per call.
``pad_mode`` controls the stream-start left padding: SEANet convs use zeros
(``constant``); the down/upsample resamplers use ``replicate`` (the first
real sample), marked per row so elastic slot recycling re-primes correctly.
"""
def __init__(self, conv: nn.Conv1d, pad_mode: str = "constant") -> None:
self.conv = conv
self.kernel = (conv.kernel_size[0] - 1) * conv.dilation[0] + 1
self.stride = conv.stride[0]
self.pad_mode = pad_mode
self.prev: torch.Tensor | None = None
self._fresh: torch.Tensor | None = None
def reset(self, batch_size: int, device, dtype) -> None:
pad = self.kernel - self.stride
self.prev = torch.zeros(batch_size, self.conv.in_channels, pad, device=device, dtype=dtype)
self._fresh = torch.ones(batch_size, dtype=torch.bool)
def reset_slot(self, b: int) -> None:
self.prev[b].zero_()
self._fresh[b] = True
def __call__(self, x: torch.Tensor) -> torch.Tensor:
if self.pad_mode == "replicate" and bool(self._fresh.any()):
pad = self.prev.shape[-1]
edge = x[..., 0:1].expand(-1, -1, pad)
fresh = self._fresh.to(x.device).view(-1, 1, 1)
self.prev = torch.where(fresh, edge.to(self.prev.dtype), self.prev)
self._fresh[:] = False
x = torch.cat([self.prev, x], dim=-1)
t = x.shape[-1]
num_frames = max(0, (t - self.kernel) // self.stride + 1)
self.prev = x[..., num_frames * self.stride :]
if num_frames == 0:
return x.new_zeros(x.shape[0], self.conv.out_channels, 0)
return self.conv(x[..., : (num_frames - 1) * self.stride + self.kernel])
class _StreamConvTr1d:
"""Moshi ``RawStreamingConvTranspose1d``: overlap-add tail carry per call."""
def __init__(self, conv: nn.ConvTranspose1d) -> None:
self.conv = conv
self.kernel = conv.kernel_size[0]
self.stride = conv.stride[0]
self.partial: torch.Tensor | None = None
def reset(self, batch_size: int, device, dtype) -> None:
self.partial = torch.zeros(
batch_size, self.conv.out_channels, self.kernel - self.stride, device=device, dtype=dtype
)
self._fresh = torch.ones(batch_size, dtype=torch.bool)
def reset_slot(self, b: int) -> None:
self.partial[b].zero_()
self._fresh[b] = True
def __call__(self, x: torch.Tensor) -> torch.Tensor:
out = self.conv(x)
length = out.shape[-1]
tail = self.kernel - self.stride
pt = self.partial.shape[-1]
merge = self.partial.clone()
if self.conv.bias is not None:
# The carried tail already includes the bias; the fresh output adds
# it again, so subtract one copy -- except on a row's very first
# frame, where the carry is zeros by construction.
merge = merge - self.conv.bias[:, None]
merge[self._fresh] = 0.0
self._fresh[:] = False
out[..., :pt] += merge
self.partial = out[..., length - tail :].clone()
return out[..., : length - tail]
class _MimiTransformerLayer(nn.Module):
"""Moshi mimi transformer layer: LayerNorm, LayerScale, GELU FFN, no biases."""
def __init__(self, dim: int = 512, num_heads: int = 8, ffn: int = 2048) -> None:
super().__init__()
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.in_proj_weight = nn.Parameter(torch.empty(3 * dim, dim))
self.out_proj_weight = nn.Parameter(torch.empty(dim, dim))
self.linear1 = nn.Parameter(torch.empty(ffn, dim))
self.linear2 = nn.Parameter(torch.empty(dim, ffn))
self.norm1 = nn.LayerNorm(dim)
self.norm2 = nn.LayerNorm(dim)
self.scale1 = nn.Parameter(torch.empty(dim))
self.scale2 = nn.Parameter(torch.empty(dim))
def forward(self, x: torch.Tensor, kv: _RingKV, offset: torch.Tensor, context: int) -> torch.Tensor:
B, T, _ = x.shape
h = self.norm1(x)
qkv = F.linear(h, self.in_proj_weight)
qkv = qkv.view(B, T, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv[0], qkv[1], qkv[2]
q, k = _apply_rope(q, k, offset)
keys, values, pos_k = kv.complete(k, v)
pos_k = pos_k.view(pos_k.shape[0], 1, pos_k.shape[1])
pos_q = offset + torch.arange(T, device=q.device, dtype=torch.long).view(1, -1, 1)
delta = pos_q - pos_k
attn_bias = (pos_k >= 0) & (delta >= 0) & (delta < context)
attn = F.scaled_dot_product_attention(q, keys, values, attn_bias.unsqueeze(1), dropout_p=0.0)
attn = attn.transpose(1, 2).reshape(B, T, self.dim)
x = x + self.scale1 * F.linear(attn, self.out_proj_weight)
h = self.norm2(x)
h = F.linear(F.gelu(F.linear(h, self.linear1)), self.linear2)
return x + self.scale2 * h
class _MimiStreamingTransformer(nn.Module):
"""The 8-layer mimi encoder/decoder transformer as a stateful stepper."""
def __init__(self, num_layers: int = 8, dim: int = 512, num_heads: int = 8, context: int = 250) -> None:
super().__init__()
self.context = context
self.layers = nn.ModuleList([_MimiTransformerLayer(dim, num_heads) for _ in range(num_layers)])
self._kv: list[_RingKV] | None = None
self._offset: torch.Tensor | None = None
def streaming_init(self, batch_size: int) -> None:
p = next(self.parameters())
heads = self.layers[0].num_heads
hd = self.layers[0].head_dim
self._kv = [_RingKV(batch_size, heads, hd, self.context, p.device, p.dtype) for _ in self.layers]
self._offset = torch.zeros(1, device=p.device, dtype=torch.long)
def reset_streaming(self) -> None:
for kv in self._kv:
kv.reset()
self._offset.zero_()
def reset_slot(self, b: int) -> None:
for kv in self._kv:
kv.reset_slot(b)
def step(self, x: torch.Tensor) -> torch.Tensor:
"""``x`` is ``[B, T, dim]`` (T = positions this frame, typically 2)."""
for layer, kv in zip(self.layers, self._kv):
x = layer(x, kv, self._offset, self.context)
self._offset.add_(x.shape[1])
return x
def load_weights(self, state_dict: dict[str, torch.Tensor], prefix: str) -> int:
loaded = 0
with torch.no_grad():
for i, layer in enumerate(self.layers):
base = f"{prefix}.transformer.layers.{i}"
pairs = [
(layer.in_proj_weight, f"{base}.self_attn.in_proj_weight"),
(layer.out_proj_weight, f"{base}.self_attn.out_proj.weight"),
(layer.linear1, f"{base}.linear1.weight"),
(layer.linear2, f"{base}.linear2.weight"),
(layer.norm1.weight, f"{base}.norm1.weight"),
(layer.norm1.bias, f"{base}.norm1.bias"),
(layer.norm2.weight, f"{base}.norm2.weight"),
(layer.norm2.bias, f"{base}.norm2.bias"),
(layer.scale1, f"{base}.layer_scale_1.scale"),
(layer.scale2, f"{base}.layer_scale_2.scale"),
]
for param, name in pairs:
param.data.copy_(state_dict[name].reshape(param.shape).to(param.dtype))
loaded += 1
return loaded
def _walk_seanet(layers) -> list[tuple[str, object]]:
"""Wrap a Hugging Face Mimi SEANet layer list with streaming conv state."""
stages: list[tuple[str, object]] = []
for layer in layers:
kind = type(layer).__name__
if kind == "MimiConv1d":
stages.append(("conv", _StreamConv1d(layer.conv)))
elif kind == "MimiConvTranspose1d":
stages.append(("convtr", _StreamConvTr1d(layer.conv)))
elif kind == "ELU":
stages.append(("act", layer))
elif kind == "MimiResnetBlock":
block = (
layer.block[0],
_StreamConv1d(layer.block[1].conv),
layer.block[2],
_StreamConv1d(layer.block[3].conv),
)
stages.append(("res", block))
else: # pragma: no cover - unexpected layer type
raise ValueError(f"unhandled Mimi SEANet layer: {kind}")
return stages
class PersonaPlexMimiCodec(nn.Module):
"""Streaming Mimi encode/decode at one 80 ms frame per call (moshi-free)."""
def __init__(self, hf_repo: str = DEFAULT_HF_REPO, checkpoint: str | None = None, device: str = "cuda") -> None:
super().__init__()
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
from transformers import MimiConfig, MimiModel
self.device = torch.device(device)
# The PersonaPlex repo ships the reference mimi checkpoint in the moshi
# fused layout. Build the matching Transformers graph locally, then load
# every codec tensor from that bundled checkpoint. Only the two HF
# transformer stacks remain absent because the streaming implementations
# below replace them with the checkpoint's fused QKV layout.
if checkpoint is None:
checkpoint = hf_hub_download(
"nvidia/personaplex-7b-v1",
"tokenizer-e351c8d8-checkpoint125.safetensors",
)
sd = load_file(checkpoint, device=str(self.device))
self.model = MimiModel(MimiConfig())
codec_state = _map_moshi_codec_weights(sd)
incompatible = self.model.load_state_dict(codec_state, strict=False)
expected_missing = {
name
for name in self.model.state_dict()
if name.startswith(("encoder_transformer.", "decoder_transformer."))
}
if set(incompatible.missing_keys) != expected_missing or incompatible.unexpected_keys:
raise RuntimeError(
"PersonaPlex Mimi checkpoint did not exactly cover the local codec graph: "
f"missing={sorted(set(incompatible.missing_keys) - expected_missing)}, "
f"unexpected={sorted(incompatible.unexpected_keys)}"
)
# The fused streaming transformers below replace these unloaded HF
# stacks. Drop their randomly initialized weights before moving Mimi to
# the target device.
del self.model.encoder_transformer, self.model.decoder_transformer
self.model = self.model.to(self.device).eval()
self.dtype = next(self.model.parameters()).dtype
self.encoder_transformer = _MimiStreamingTransformer().to(self.device, self.dtype)
self.decoder_transformer = _MimiStreamingTransformer().to(self.device, self.dtype)
n_enc = self.encoder_transformer.load_weights(sd, "encoder_transformer")
n_dec = self.decoder_transformer.load_weights(sd, "decoder_transformer")
assert n_enc == n_dec == 80, (n_enc, n_dec)
del sd
m = self.model
self._enc_stages = _walk_seanet(m.encoder.layers)
self._downsample = _StreamConv1d(m.downsample.conv, pad_mode="replicate")
self._upsample = _StreamConvTr1d(m.upsample.conv)
self._dec_stages = _walk_seanet(m.decoder.layers)
self._batch_size: int | None = None
# -- streaming state ------------------------------------------------------
def _conv_states(self):
for _, stage in (*self._enc_stages, *self._dec_stages):
if isinstance(stage, (_StreamConv1d, _StreamConvTr1d)):
yield stage
elif isinstance(stage, tuple):
yield stage[1]
yield stage[3]
yield self._downsample
yield self._upsample
def streaming_init(self, batch_size: int) -> None:
self._batch_size = batch_size
for s in self._conv_states():
s.reset(batch_size, self.device, self.dtype)
self.encoder_transformer.streaming_init(batch_size)
self.decoder_transformer.streaming_init(batch_size)
def reset_streaming(self) -> None:
assert self._batch_size is not None
for state in self._conv_states():
for b in range(self._batch_size):
state.reset_slot(b)
self.encoder_transformer.reset_streaming()
self.decoder_transformer.reset_streaming()
def reset_slot(self, b: int) -> None:
for s in self._conv_states():
s.reset_slot(b)
self.encoder_transformer.reset_slot(b)
self.decoder_transformer.reset_slot(b)
# -- per-frame codec -------------------------------------------------------
@staticmethod
def _run_stages(x: torch.Tensor, stages) -> torch.Tensor:
for kind, stage in stages:
if kind == "res":
act0, conv1, act2, conv3 = stage
x = x + conv3(act2(conv1(act0(x))))
else:
x = stage(x)
return x
@torch.no_grad()
def encode_frame(self, pcm: torch.Tensor) -> torch.Tensor:
"""``[B, frame_size]`` float PCM -> ``[B, 8]`` codes."""
x = pcm.to(self.device, self.dtype).view(-1, 1, FRAME_SIZE)
x = self._run_stages(x, self._enc_stages)
x = self.encoder_transformer.step(x.transpose(1, 2)).transpose(1, 2)
x = self._downsample(x)
codes = self.model.quantizer.encode(x) # [Q, B, T]
return codes[:CODEBOOKS, :, 0].transpose(0, 1).contiguous()
@torch.no_grad()
def decode_frame(self, codes: torch.Tensor) -> torch.Tensor:
"""``[B, 8]`` codes -> ``[B, frame_size]`` float PCM."""
emb = self.model.quantizer.decode(codes.to(self.device).view(-1, CODEBOOKS, 1))
emb = self._upsample(emb)
emb = self.decoder_transformer.step(emb.transpose(1, 2)).transpose(1, 2)
x = self._run_stages(emb, self._dec_stages)
return x[:, 0, :]
@torch.no_grad()
def decode_frames(self, codes: torch.Tensor) -> torch.Tensor:
"""``[B, 8, F]`` codes -> ``[B, F * frame_size]`` float PCM."""
emb = self.model.quantizer.decode(codes.to(self.device))
emb = self._upsample(emb)
emb = self.decoder_transformer.step(emb.transpose(1, 2)).transpose(1, 2)
x = self._run_stages(emb, self._dec_stages)
return x[:, 0, :]
@@ -0,0 +1,639 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PersonaPlex talker: the temporal transformer as a vLLM-native omni AR stage.
This is the stage-0 (``LLM_AR``) model the OmniGPUModelRunner drives. It composes
three pieces, each verified in isolation against Moshi:
* the Helium temporal transformer (:class:`HeliumModel`) on vLLM paged attention,
consuming per-frame ``inputs_embeds`` and producing the per-frame hidden state;
* the input embeddings (:class:`PersonaPlexInputEmbeddings`, ``embed_codes``) that
build those ``inputs_embeds`` from the delayed 17-row token stack;
* the depformer (:class:`PersonaPlexDepformer`) that, conditioned on the temporal
hidden state and the sampled text token, predicts the per-frame audio codes.
Per-frame protocol (OmniGPUModelRunner, gpu_model_runner.py):
1. ``compute_logits`` produces the text logits; the engine samples the text token.
2. ``preprocess`` (per-request, with that request's ``additional_information``)
carries Moshi's acoustic-delay cache and the precomputed user-audio code stream,
builds the base ``inputs_embeds`` for the current frame, and exposes the
previous frame's temporal hidden + text-step embedding via ``mtp_inputs``.
3. ``talker_mtp`` (batched, stateless) runs the depformer to predict the agent
codes and finishes the next frame's ``inputs_embeds``; the codes are stored
under ``talker_mtp_output_key=("codes","audio")`` for the Mimi code2wav stage.
Phase 1 is turn-based: the user-audio rows (9..16) come from a precomputed Mimi
encode of the input WAV (built in ``preprocess``); live duplex is Phase 2.
"""
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
PersonaPlexConfig,
)
from vllm_omni.model_executor.models.personaplex.modeling_helium import HeliumModel
from vllm_omni.model_executor.models.personaplex.personaplex_depformer import (
PersonaPlexDepformer,
)
from vllm_omni.model_executor.models.personaplex.personaplex_embeddings import (
PersonaPlexInputEmbeddings,
)
__all__ = ["PersonaPlexTalkerForConditionalGeneration"]
class PersonaPlexTalkerForConditionalGeneration(nn.Module):
"""vLLM-native PersonaPlex talker (temporal transformer + depformer).
Plain ``nn.Module`` (not ``SupportsPP``): inheriting a vLLM Protocol puts
``Protocol`` in the MRO and breaks vLLM's runtime ``isinstance`` check for
``VllmModelForTextGeneration``, which would misclassify the talker as a
non-generate model. Qwen3-TTS's talker is plain ``nn.Module`` for the same
reason; PersonaPlex does not need pipeline parallelism.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.vllm_config = vllm_config
config: PersonaPlexConfig = vllm_config.model_config.hf_config # type: ignore[assignment]
self.config = config
self.temporal_config = config.temporal_config
hidden = self.temporal_config.hidden_size
self._dtype = getattr(vllm_config.model_config, "dtype", torch.bfloat16)
# Temporal backbone on vLLM paged attention (consumes inputs_embeds).
self.model = HeliumModel(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
config=self.temporal_config,
)
# Text head (Moshi text_linear -> lm_head), vocab = text_vocab_size (32000).
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.text_vocab_size,
hidden,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.text_vocab_size)
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
# Verified custom components: embed_codes + depformer.
self.input_embeddings = PersonaPlexInputEmbeddings(config)
self.depformer = PersonaPlexDepformer(
config.depformer_config,
temporal_hidden_size=hidden,
text_card=config.text_vocab_size,
)
# Omni AR runner contract.
self.have_multimodal_outputs = True
self.has_preprocess = True
self.has_postprocess = True # capture each frame's hidden for the next depformer step
self.requires_full_prefix_cached_hidden_states = False
# Keep the per-frame "last" hidden on GPU (avoids a CPU round-trip each step).
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {("hidden_states", "last")}
self.mtp_hidden_size = hidden
self.talker_mtp_output_key = ("codes", "audio")
# dep_q audio codebooks per frame; only cb 0..num_active are vocoded.
self.dep_q = config.depformer_config.dep_q
self.num_active_codebooks = config.depformer_config.num_active_codebooks
# ------------------------------------------------------------------
# Core forward / logits
# ------------------------------------------------------------------
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
"""Placeholder embedding for vLLM's ``VllmModel`` protocol.
The talker is driven by precomputed ``inputs_embeds`` (built in
``preprocess`` via ``embed_codes``); ``input_ids`` are only in-vocab
bookkeeping placeholders. Return a zero embedding of the temporal hidden
size — the runner replaces it with the real per-frame ``inputs_embeds``.
"""
return torch.zeros(
(input_ids.shape[0], self.mtp_hidden_size),
device=input_ids.device,
dtype=self._dtype,
)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**_: Any,
) -> torch.Tensor | IntermediateTensors:
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
def compute_logits(
self,
hidden_states: torch.Tensor | OmniOutput,
sampling_metadata: Any = None,
) -> torch.Tensor | None:
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if hidden_states is None:
return None
if hidden_states.dim() == 3:
b, s, h = hidden_states.shape
logits = self.logits_processor(self.lm_head, hidden_states.reshape(b * s, h))
return None if logits is None else logits.reshape(b, s, -1)
return self.logits_processor(self.lm_head, hidden_states)
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
hidden = model_outputs
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information") or []
audio_codes_list: list[torch.Tensor] = []
for info in info_dicts:
if not isinstance(info, dict):
continue
ac = info.get("codes", {}).get("audio")
if isinstance(ac, torch.Tensor):
audio_codes_list.append(ac)
if not audio_codes_list:
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
audio_codes = torch.cat(audio_codes_list, dim=0)
hidden = hidden[: int(audio_codes.shape[0])]
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={"codes": {"audio": audio_codes}})
# ------------------------------------------------------------------
# Per-frame omni AR protocol: preprocess (stateful) + talker_mtp (batched)
# ------------------------------------------------------------------
def _initial_frame_embed(self, device: torch.device) -> torch.Tensor:
"""``embed_codes`` of Moshi's initial token frame (text+audio SOS)."""
text_init = self.config.text_vocab_size # text_initial_token_id
audio_init = self.config.audio_vocab_size # card == initial audio token
n_q = self.config.num_audio_codebooks
stack = torch.empty((1, 1 + n_q, 1), dtype=torch.long, device=device)
stack[:, 0] = text_init
stack[:, 1:] = audio_init
return self.input_embeddings(stack).reshape(1, -1) # [1, hidden]
def _build_frame_embed(
self,
text_token: torch.Tensor,
last_agent: torch.Tensor | None,
prev_agent: torch.Tensor | None,
device: torch.device,
user_d0: torch.Tensor | None = None,
user_d1: torch.Tensor | None = None,
) -> torch.Tensor:
"""Full ``inputs_embeds`` for a decode frame (Moshi cache read at offset-1).
Acoustic-delay decomposition (delays ``[0, 0,1x7, 0,1x7]``): after a
depformer step, Moshi writes every unforced agent codebook into the
current cache target. The next temporal input therefore reads the whole
previous effective agent frame. User writes are delayed by codebook, so
user cb0 and cb1..7 still come from ``user_d0`` and ``user_d1``.
Unfilled positions default to the initial token (card).
"""
n_q = self.config.num_audio_codebooks # 16 (8 agent + 8 user)
n_user = n_q // 2 # 8
audio_init = self.config.audio_vocab_size # card == initial audio token (Moshi cache default)
# No-history / not-yet-generated delayed positions = the initial token (card),
# NOT 0 (verified against Moshi's per-frame input stack).
stack = torch.full((1, 1 + n_q, 1), audio_init, dtype=torch.long, device=device)
stack[:, 0] = text_token.reshape(1)
# The full effective agent frame was stored at the previous cache target
# after applying any teacher forcing. The next cache read sees all eight
# codebooks from that frame.
if last_agent is not None and last_agent.numel() >= n_user:
stack[0, 1 : 1 + n_user, 0] = last_agent.reshape(-1)[:n_user].to(device)
# user cb0 (row 9) delay-0; user cb1..7 (rows 10..16) delay-1.
user_base = 1 + n_user # row index 9
if user_d0 is not None and user_d0.numel() >= 1:
stack[0, user_base, 0] = user_d0.reshape(-1)[0].to(device)
if user_d1 is not None and user_d1.numel() >= n_user:
stack[0, user_base + 1 : user_base + n_user, 0] = user_d1.reshape(-1)[1:n_user].to(device)
return self.input_embeddings(stack).reshape(1, -1) # [1, hidden]
def _build_prefill_embed(
self,
prefill_text: torch.Tensor,
offset: int,
span: int,
device: torch.device,
silence: torch.Tensor | None = None,
user_sine: torch.Tensor | None = None,
) -> torch.Tensor:
"""Per-frame embeds for the persona system-prompt prefill.
Each frame forces the persona/silence text token (row 0). The agent rows
use Mimi-encoded silence, while the user rows optionally use the encoded
sine frame from PersonaPlex's native text-prompt prefill. Without an
explicit user frame, both sides fall back to silence.
"""
n_q = self.config.num_audio_codebooks
n_user = n_q // 2
zero_text = 3 # Moshi LMGen.zero_text_code (silence/pad text)
sil = (
silence.reshape(-1).to(device) if isinstance(silence, torch.Tensor) and silence.numel() >= n_user else None
)
user = (
user_sine.reshape(-1).to(device)
if isinstance(user_sine, torch.Tensor) and user_sine.numel() >= n_user
else sil
)
total = int(prefill_text.numel())
rows = []
for i in range(span):
pos = offset + i
text_tok = int(prefill_text[pos].item()) if pos < total else zero_text
stack = torch.zeros((1, 1 + n_q, 1), dtype=torch.long, device=device)
stack[:, 0] = text_tok
if sil is not None:
stack[0, 1 : 1 + n_user, 0] = sil[:n_user] # agent rows = encoded silence
if user is not None:
stack[0, 1 + n_user : 1 + 2 * n_user, 0] = user[:n_user]
rows.append(self.input_embeddings(stack).reshape(1, -1))
return torch.cat(rows, dim=0) # [span, hidden]
@staticmethod
def _user_frame(info: dict[str, Any], frame_idx: int) -> torch.Tensor | None:
"""Fetch the Mimi-encoded user codes for ``frame_idx`` (or None / out of range)."""
uc = info.get("pplex_user_codes")
if uc is None:
return None
if not isinstance(uc, torch.Tensor):
uc = torch.as_tensor(uc, dtype=torch.long)
if uc.ndim != 2 or frame_idx < 0 or frame_idx >= uc.shape[0]:
return None
return uc[frame_idx]
@staticmethod
def _last_agent_codes(info: dict[str, Any]) -> torch.Tensor | None:
codes = info.get("codes")
if isinstance(codes, dict):
ac = codes.get("audio")
if isinstance(ac, torch.Tensor) and ac.numel() > 0:
return (ac[-1] if ac.ndim == 2 else ac).reshape(-1)
return None
def preprocess(
self,
input_ids: torch.Tensor,
input_embeds: torch.Tensor | None,
**info_dict: Any,
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
"""Build the current frame's ``inputs_embeds`` and the mtp inputs.
Prefill emits the initial-token frame embedding; each decode step builds the
delayed base frame (everything but agent cb0, which ``talker_mtp`` adds).
"""
additional = info_dict.get("additional_information")
if isinstance(additional, dict):
merged = {k: v for k, v in info_dict.items() if k != "additional_information"}
for k, v in additional.items():
merged.setdefault(k, v)
info_dict = merged
meta = info_dict.get("meta", {}) if isinstance(info_dict.get("meta"), dict) else {}
device = input_ids.device
span = int(input_ids.shape[0])
is_prefill_raw = info_dict.get("_omni_is_prefill")
if isinstance(is_prefill_raw, bool):
is_prefill = is_prefill_raw
else:
is_prefill = span > 1
duplex = info_dict.get("duplex")
if isinstance(duplex, dict) and duplex.get("data_plane") is True and is_prefill:
prompt_len_raw = info_dict.get("duplex_prompt_len", span)
try:
prompt_len = int(prompt_len_raw)
except (TypeError, ValueError):
prompt_len = span
prepared = self._duplex_stage0_runtime().prepare_append(
duplex,
prompt_len=prompt_len,
request_id=(str(info_dict["request_id"]) if isinstance(info_dict.get("request_id"), str) else None),
)
offset_raw = info_dict.get("duplex_token_offset", 0)
try:
offset = max(0, int(offset_raw))
except (TypeError, ValueError):
offset = 0
local_offset = offset - prepared.prompt_offset
if local_offset < 0:
raise ValueError(
"PersonaPlex scheduled span precedes the current append: "
f"offset={offset}, append_offset={prepared.prompt_offset}, "
f"span={span}, prompt={prompt_len}"
)
req_embeds = prepared.inputs_embeds[local_offset : local_offset + span].to(
device=device,
dtype=self._dtype,
)
req_input_ids = prepared.input_ids[local_offset : local_offset + span].to(device=device)
if req_embeds.shape[0] != span or req_input_ids.shape[0] != span:
raise ValueError(
"PersonaPlex duplex prompt slice is shorter than the scheduled span: "
f"offset={offset}, span={span}, prompt={prompt_len}"
)
return req_input_ids, req_embeds, prepared.info_update
zero_hidden = torch.zeros((1, self.mtp_hidden_size), device=device, dtype=self._dtype)
prefill_text = info_dict.get("pplex_prefill_text")
prefill_len = int(prefill_text.numel()) if isinstance(prefill_text, torch.Tensor) else 0
if is_prefill:
offset = max(0, int(info_dict.get("_omni_num_computed_tokens", 0) or 0))
if isinstance(prefill_text, torch.Tensor) and prefill_text.numel() > 0:
# Persona system-prompt prefill (step_system_prompts analog).
silence = info_dict.get("pplex_silence_codes")
emb = self._build_prefill_embed(prefill_text, offset, span, device, silence)
else:
emb = self._initial_frame_embed(device)
if span > 1:
emb = emb.expand(span, -1).contiguous()
ids_out = input_ids.clone()
info_update = {
"meta": {
"pplex_frame": int(meta.get("pplex_frame", 0)) + span,
"pplex_prefill_len": prefill_len,
},
"mtp_inputs": (zero_hidden, zero_hidden),
}
return ids_out, emb, info_update
# Decode: input_ids == the text token sampled last step (delay-0 text).
text_token = input_ids.reshape(-1)[:1]
# Agent acoustic delay: cb0 (delay 0) = gen[t-1] = the last stored codes;
# cb1..7 (delay 1) = gen[t-2] = the codes carried from the previous frame.
last_agent = self._last_agent_codes(info_dict)
prev_agent = info_dict.get("pplex_prev_agent")
if prev_agent is not None and not isinstance(prev_agent, torch.Tensor):
prev_agent = torch.as_tensor(prev_agent, dtype=torch.long)
# Real user stream (Phase-1 turn-based): user cb0 delay-0, cb1..7 delay-1.
# Index into the user stream relative to the first decode frame (after the
# persona prefill), so the user audio aligns with the agent's response.
prefill_len = int(meta.get("pplex_prefill_len", prefill_len) or 0)
# decode_frame = pplex_frame - prefill_len (the prefill frame already advanced
# pplex_frame, so this is d+1 at decode step d). Verified vs Moshi's per-frame
# stack: user cb0 (delay 0) = enc[decode_frame-1] = enc[d]; cb1..7 (delay 1) =
# enc[decode_frame-2]. (The agent stream lags one more: agent cb0 = gen[d-1].)
decode_frame = max(0, int(meta.get("pplex_frame", 0)) - prefill_len)
user_d0 = self._user_frame(info_dict, decode_frame - 1)
user_d1 = self._user_frame(info_dict, decode_frame - 2)
base = self._build_frame_embed(text_token, last_agent, prev_agent, device, user_d0, user_d1)
text_step = self.input_embeddings.text_emb(text_token.reshape(1, 1)).reshape(1, -1)
# The previous frame's temporal hidden (written by postprocess into
# hidden_states["last"]) conditions this frame's depformer. Falls back to
# zeros only on the very first decode step (no prior hidden yet).
hs = info_dict.get("hidden_states", {}) if isinstance(info_dict.get("hidden_states"), dict) else {}
last_hidden = hs.get("last")
if isinstance(last_hidden, torch.Tensor) and last_hidden.numel() > 0:
last_hidden = last_hidden.reshape(1, -1).to(device=device, dtype=self._dtype)
else:
last_hidden = zero_hidden
info_update = {
"meta": {"pplex_frame": int(meta.get("pplex_frame", 0)) + 1},
"mtp_inputs": (last_hidden, text_step),
}
# Carry this frame's gen[t-1] forward; next frame reads it as gen[t-2] (cb1..7).
if last_agent is not None:
info_update["pplex_prev_agent"] = last_agent.detach().to(torch.long).cpu()
return input_ids, base, info_update
def _duplex_stage0_runtime(self):
runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None)
if runtime is not None:
return runtime
from vllm_omni.experimental.fullduplex.personaplex.stage0 import (
PersonaPlexStage0DuplexRuntime,
)
model_path = str(getattr(self.vllm_config.model_config, "model", ""))
device = str(next(self.parameters()).device)
runtime = PersonaPlexStage0DuplexRuntime(
self,
model_path=model_path,
device=device,
max_sessions=int(getattr(self.vllm_config.model_config, "duplex_max_sessions", 1)),
)
self._personaplex_duplex_stage0_runtime = runtime
return runtime
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None)
if runtime is None:
return
for request_id in finished_req_ids:
runtime.close_request(request_id)
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
"""Capture this frame's last hidden for the next step's depformer (mtp)."""
if hidden_states is None or hidden_states.numel() == 0:
return {}
return {"hidden_states": {"last": hidden_states[-1, :].detach()}}
def talker_mtp(
self,
input_ids: torch.Tensor,
input_embeds: torch.Tensor,
last_talker_hidden: torch.Tensor,
text_step: torch.Tensor,
**kwargs: Any,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run the depformer for the frame and finish its ``inputs_embeds``.
Returns ``(inputs_embeds, audio_codes[B, dep_q])``. ``audio_codes`` are
stored under ``("codes","audio")``; ``inputs_embeds`` (base from preprocess
+ the fresh agent cb0 embedding, the delay-0 acoustic token) feeds the next
temporal forward.
"""
bsz = int(input_ids.shape[0])
dtype = self._dtype
text_token = input_ids.reshape(bsz).to(torch.long)
hidden = last_talker_hidden.reshape(bsz, 1, -1).to(dtype)
codes = self.depformer(text_token, hidden) # [B, dep_q] == gen[t]
# The frame's inputs_embeds is built fully in preprocess (Moshi cache read at
# offset-1, with agent cb0 = gen[t-1]); gen[t] feeds the NEXT frame, so pass
# the embed through unchanged and just emit this frame's codes.
inputs_embeds = input_embeds.reshape(bsz, -1).to(dtype)
return inputs_embeds, codes.to(torch.long)
def post_sample_talker_mtp(
self,
*,
input_ids: torch.Tensor,
hidden_states: torch.Tensor,
req_ids: list[str],
req_infos: list[dict[str, Any]],
) -> torch.Tensor:
"""Generate depformer codes for a one-token resumable duplex segment.
The normal runner invokes ``talker_mtp`` at the start of the next decode
step. PersonaPlex's unified duplex request instead appends one audio
frame and stops after one sampled text token, so there is no next decode
step. Run the same depformer dependency immediately from the current
sampled text token and temporal hidden state.
"""
bsz = int(input_ids.shape[0])
if len(req_infos) != bsz:
raise ValueError(
f"PersonaPlex depformer request information does not match batch: {len(req_infos)} != {bsz}"
)
text_token = input_ids.reshape(bsz).to(torch.long)
hidden = hidden_states.reshape(bsz, 1, -1).to(self._dtype)
audio_tokens: list[torch.Tensor] = []
audio_provided: list[torch.Tensor] = []
for info in req_infos:
tokens = info.get("pplex_depformer_audio_tokens")
provided = info.get("pplex_depformer_audio_provided")
if not isinstance(tokens, torch.Tensor) or not isinstance(provided, torch.Tensor):
raise ValueError("PersonaPlex duplex depformer teacher-forcing state is missing")
tokens = tokens.reshape(-1)
provided = provided.reshape(-1)
if tokens.shape != provided.shape:
raise ValueError(
"PersonaPlex depformer teacher-forcing token/mask shapes differ: "
f"{tuple(tokens.shape)} != {tuple(provided.shape)}"
)
audio_tokens.append(tokens)
audio_provided.append(provided)
codes = self.depformer(
text_token,
hidden,
audio_tokens=torch.stack(audio_tokens).to(
device=hidden.device,
dtype=torch.long,
),
audio_provided=torch.stack(audio_provided).to(
device=hidden.device,
dtype=torch.bool,
),
).to(torch.long)
runtime = self._duplex_stage0_runtime()
for row, request_id in enumerate(req_ids):
runtime.record_sample(
request_id=request_id,
text_token=text_token[row],
agent_codes=codes[row],
)
return codes
# ------------------------------------------------------------------
# Weight loading
# ------------------------------------------------------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Route the Moshi checkpoint into the three components.
* ``transformer.*`` / ``out_norm.alpha`` -> the Helium temporal backbone
(same q/k-split + gate/up-split + alpha-squeeze map as HeliumForCausalLM).
* ``text_linear.weight`` -> ``lm_head``.
* ``emb.*`` / ``text_emb.weight`` -> input embeddings.
* ``depformer*`` / ``linears.*`` -> depformer.
"""
weights = list(weights)
params = dict(self.named_parameters(remove_duplicate=False))
loaded: set[str] = set()
emb_w: list[tuple[str, torch.Tensor]] = []
dep_w: list[tuple[str, torch.Tensor]] = []
for name, w in weights:
if name.startswith(("emb.", "text_emb.")):
emb_w.append((name, w))
elif name.startswith(("depformer.", "depformer_in.", "depformer_emb.", "depformer_text_emb", "linears.")):
dep_w.append((name, w))
elif name == "text_linear.weight":
self._load_direct("lm_head.weight", w, params, loaded)
else:
loaded |= self._load_temporal(name, w, params)
# Delegate to the verified component loaders, prefixed into this module.
for sub, sub_w in (("input_embeddings", emb_w), ("depformer", dep_w)):
module = getattr(self, sub)
for tgt in module.load_weights(sub_w):
loaded.add(f"{sub}.{tgt}")
return loaded
def _load_temporal(
self,
name: str,
loaded_weight: torch.Tensor,
params: dict[str, nn.Parameter],
) -> set[str]:
out: set[str] = set()
if name == "out_norm.alpha":
self._load_direct("model.norm.weight", loaded_weight.squeeze(), params, out)
return out
prefix = "transformer.layers."
if not name.startswith(prefix):
return out
rest = name.removeprefix(prefix)
layer_index, _, suffix = rest.partition(".")
if not layer_index.isdigit() or not suffix:
return out
base = f"model.layers.{layer_index}"
if suffix == "self_attn.in_proj_weight":
q, k, v = loaded_weight.chunk(3, dim=0)
pname = f"{base}.self_attn.qkv_proj.weight"
self._load_shard(pname, q, "q", params)
self._load_shard(pname, k, "k", params)
self._load_shard(pname, v, "v", params)
out.add(pname)
elif suffix == "self_attn.out_proj.weight":
self._load_direct(f"{base}.self_attn.o_proj.weight", loaded_weight, params, out)
elif suffix == "gating.linear_in.weight":
gate, up = loaded_weight.chunk(2, dim=0)
pname = f"{base}.mlp.gate_up_proj.weight"
self._load_shard(pname, gate, 0, params)
self._load_shard(pname, up, 1, params)
out.add(pname)
elif suffix == "gating.linear_out.weight":
self._load_direct(f"{base}.mlp.down_proj.weight", loaded_weight, params, out)
elif suffix == "norm1.alpha":
self._load_direct(f"{base}.input_layernorm.weight", loaded_weight.squeeze(), params, out)
elif suffix == "norm2.alpha":
self._load_direct(f"{base}.post_attention_layernorm.weight", loaded_weight.squeeze(), params, out)
return out
@staticmethod
def _load_direct(
name: str,
loaded_weight: torch.Tensor,
params: dict[str, nn.Parameter],
loaded: set[str],
) -> None:
if name not in params:
return
param = params[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded.add(name)
@staticmethod
def _load_shard(
name: str,
loaded_weight: torch.Tensor,
shard_id: str | int,
params: dict[str, nn.Parameter],
) -> None:
if name not in params:
return
param = params[name]
param.weight_loader(param, loaded_weight, shard_id)
@@ -0,0 +1,257 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Streaming Helium temporal transformer for PersonaPlex (moshi-free, plain torch).
Frame-clocked duplex needs a stateful per-frame forward: one 80 ms step consumes
one summed frame embedding and returns the temporal hidden + text logits, with a
sliding-window KV carried across the conversation. vLLM's engine path serves the
offline/staged flavor of this model (``modeling_helium.HeliumForCausalLM``); this
module is the REAL-TIME flavor: no engine context, a fixed-capacity ring KV, and
static shapes so the whole step is CUDA-graphable.
Semantics mirror the Moshi streaming transformer exactly (MIT; the reference for
the ``nvidia/personaplex-7b-v1`` checkpoint), op for op, so greedy decoding is
bit-comparable against recorded reference outputs:
- fp32 RMSNorm with ``alpha`` weights (norm1/norm2/out_norm)
- fused in-proj attention, interleaved (non-neox) RoPE applied at the ABSOLUTE
offset in fp32, relative-position causal mask over a ring KV with
``context``-window truncation
- SiLU gating MLP with fused ``linear_in`` (gate/up) and ``linear_out``
- ``transformer_out`` is the POST-``out_norm`` hidden; ``text_logits`` is
``text_linear`` of that hidden
Weights load directly from the moshi checkpoint's fused layout
(``transformer.layers.{i}.self_attn.in_proj_weight`` etc.), keeping the exact
computation order of the reference.
Per-slot elastic recycle (concurrent serving) uses the same per-row
``start_offset`` masking verified for the moshi-hosted path: writes stay uniform
because every slot ticks in lockstep; ``reset_slot`` only moves that row's
valid-window start. The LM-specific "+1 sacrifice tick" bump is the CALLER's
job (see the fullduplex PersonaPlex stage0), matching the split introduced when
the +1 was found to be wrong for codec streams.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm_omni.model_executor.models.personaplex.personaplex_depformer import _rms_norm_f32
def _apply_rope(q: torch.Tensor, k: torch.Tensor, offset: torch.Tensor, max_period: float = 10_000.0):
"""Interleaved RoPE at absolute ``offset``, fp32 rotation (moshi ``apply_rope``)."""
B, H, T, D = q.shape
ds = torch.arange(D // 2, device=q.device, dtype=torch.float32)
freqs = torch.exp(ds * (-math.log(max_period) * 2 / D))
ts = offset.float() + torch.arange(T, device=q.device, dtype=torch.float32)
ts = ts.view(1, -1, 1)
dims = q.shape[:-1]
q = q.view(*dims, D // 2, 2)
k = k.view(*dims, D // 2, 2)
qr, qi = q[..., 0].float(), q[..., 1].float()
kr, ki = k[..., 0].float(), k[..., 1].float()
rotr = torch.cos(freqs * ts)
roti = torch.sin(freqs * ts)
qor = qr * rotr - qi * roti
qoi = qr * roti + qi * rotr
kor = kr * rotr - ki * roti
koi = kr * roti + ki * rotr
dtype = q.dtype
qo = torch.stack([qor.to(dtype), qoi.to(dtype)], dim=-1)
ko = torch.stack([kor.to(dtype), koi.to(dtype)], dim=-1)
return qo.view(*dims, D), ko.view(*dims, D)
class _RingKV:
"""Fixed-capacity KV ring with per-row valid-window start (elastic recycle).
Uniform writes (all rows tick together) + a per-row mask via absolute
positions; with ``start_offset == 0`` everywhere this is the plain streaming
cache. All shapes static -> CUDA-graph safe.
"""
def __init__(self, batch_size: int, num_heads: int, dim_per_head: int, capacity: int, device, dtype):
self.capacity = capacity
self.cache = torch.zeros((2, batch_size, num_heads, capacity, dim_per_head), device=device, dtype=dtype)
self.end_offset = torch.zeros(1, device=device, dtype=torch.long)
self.start_offset = torch.zeros(batch_size, device=device, dtype=torch.long)
def reset(self) -> None:
self.end_offset.zero_()
self.start_offset.zero_()
def reset_slot(self, b: int) -> None:
# Mask everything written so far for row b; the row's next write is its
# first visible entry. (LM sacrifice-tick +1 is applied by the caller.)
self.start_offset[b] = self.end_offset.clone()
def bump_slot_start(self, b: int) -> None:
self.start_offset[b] += 1
def complete(self, k: torch.Tensor, v: torch.Tensor):
B, H, T, D = k.shape
indexes = torch.arange(T, device=self.end_offset.device, dtype=self.end_offset.dtype) + self.end_offset
indexes = indexes % self.capacity
self.cache[0].index_copy_(2, indexes, k)
self.cache[1].index_copy_(2, indexes, v)
self.end_offset.add_(T)
idx = torch.arange(self.capacity, device=self.end_offset.device, dtype=torch.long)
invalid = idx >= self.end_offset
end_index = self.end_offset % self.capacity
delta = idx - end_index
# `delta <= 0` (not `< 0`) is moshi's exact convention (transformer.py
# RingKVCache.complete). It labels the just-past-newest slot as the future
# write position, so once the ring has wrapped the single oldest in-window
# cell is excluded and the effective window is capacity-1. This is inherited
# verbatim from the reference and only shows after the window fills (Helium
# ~3000 frames / 240 s); it costs one frame out of thousands. Do NOT change
# this to `< 0`: it would diverge from moshi and break greedy bit-parity.
positions = torch.where(delta <= 0, self.end_offset + delta, self.end_offset + delta - self.capacity)
positions = torch.where(invalid, torch.full_like(positions, -1), positions)
positions = positions.view(1, -1) # [1, capacity]
below = positions < self.start_offset.view(-1, 1) # [B, capacity]
positions = torch.where(below, torch.full_like(positions, -1), positions)
return self.cache[0], self.cache[1], positions
class _TemporalLayer(nn.Module):
"""One Helium decoder layer in the checkpoint's fused layout."""
def __init__(self, dim: int, num_heads: int, hidden: int) -> None:
super().__init__()
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.in_proj_weight = nn.Parameter(torch.empty(3 * dim, dim))
self.out_proj_weight = nn.Parameter(torch.empty(dim, dim))
self.gating_in = nn.Parameter(torch.empty(2 * hidden, dim))
self.gating_out = nn.Parameter(torch.empty(dim, hidden))
self.norm1_alpha = nn.Parameter(torch.ones(1, 1, dim))
self.norm2_alpha = nn.Parameter(torch.ones(1, 1, dim))
def forward(self, x: torch.Tensor, kv: _RingKV, offset: torch.Tensor, context: int) -> torch.Tensor:
B, T, _ = x.shape
h = _rms_norm_f32(x, self.norm1_alpha, 1e-8)
qkv = F.linear(h, self.in_proj_weight)
# Layout mirrors moshi's einops "b t (p h d) -> p b h t d" exactly, so the
# downstream kernels see identical strides (bit-level replay agreement).
qkv = qkv.view(B, T, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
q, k, v = qkv[0], qkv[1], qkv[2]
q, k = _apply_rope(q, k, offset)
keys, values, pos_k = kv.complete(k, v)
pos_k = pos_k.view(pos_k.shape[0], 1, pos_k.shape[1]) # [B, 1, cap]
pos_q = offset + torch.arange(T, device=q.device, dtype=torch.long).view(1, -1, 1)
delta = pos_q - pos_k
attn_bias = (pos_k >= 0) & (delta >= 0) & (delta < context)
attn_bias = attn_bias.unsqueeze(1) # [B, 1, T, cap]
attn = F.scaled_dot_product_attention(q, keys, values, attn_bias, dropout_p=0.0)
attn = attn.transpose(1, 2).reshape(B, T, self.dim)
x = x + F.linear(attn, self.out_proj_weight)
h = _rms_norm_f32(x, self.norm2_alpha, 1e-8)
a, b = F.linear(h, self.gating_in).chunk(2, dim=-1)
return x + F.linear(F.silu(a) * b, self.gating_out)
class PersonaPlexTemporalStreaming(nn.Module):
"""The 7B Helium temporal backbone as a stateful per-frame stepper.
``step(frame_embedding [B, 1, dim]) -> (transformer_out [B, 1, dim],
text_logits [B, 1, 1, text_card])``, matching moshi's
``LMModel.forward_embeddings`` contract so the existing native depformer and
input embeddings compose unchanged.
"""
def __init__(
self,
dim: int = 4096,
num_layers: int = 32,
num_heads: int = 32,
hidden: int = 11264,
context: int = 3000,
text_card: int = 32000,
max_period: float = 10_000.0,
) -> None:
super().__init__()
self.dim = dim
self.context = context
self.max_period = max_period
self.layers = nn.ModuleList([_TemporalLayer(dim, num_heads, hidden) for _ in range(num_layers)])
self.out_norm_alpha = nn.Parameter(torch.ones(1, 1, dim))
self.text_linear = nn.Parameter(torch.empty(text_card, dim))
self._kv: list[_RingKV] | None = None
self._offset: torch.Tensor | None = None
# -- streaming state ----------------------------------------------------
def streaming_init(self, batch_size: int) -> None:
p = next(self.parameters())
heads = self.layers[0].num_heads
head_dim = self.layers[0].head_dim
# Capacity = context window (the mask truncates at `context` anyway).
self._kv = [_RingKV(batch_size, heads, head_dim, self.context, p.device, p.dtype) for _ in self.layers]
self._offset = torch.zeros(1, device=p.device, dtype=torch.long)
def reset_streaming(self) -> None:
assert self._kv is not None, "call streaming_init first"
for kv in self._kv:
kv.reset()
self._offset.zero_()
def reset_slot(self, b: int) -> None:
assert self._kv is not None
for kv in self._kv:
kv.reset_slot(b)
def bump_slot_start(self, b: int) -> None:
"""LM sacrifice-tick semantics: mask the one spurious post-recycle write."""
assert self._kv is not None
for kv in self._kv:
kv.bump_slot_start(b)
# -- per-frame step -------------------------------------------------------
@torch.no_grad()
def step(self, frame_embedding: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
assert self._kv is not None, "call streaming_init first"
x = frame_embedding
for layer, kv in zip(self.layers, self._kv):
x = layer(x, kv, self._offset, self.context)
self._offset.add_(x.shape[1])
out = _rms_norm_f32(x, self.out_norm_alpha, 1e-8)
text_logits = F.linear(out, self.text_linear)
return out, text_logits[:, None]
# -- weights --------------------------------------------------------------
def load_weights(self, state_dict: dict[str, torch.Tensor]) -> int:
"""Load from the moshi checkpoint layout (fused, exact tensor reuse)."""
loaded = 0
with torch.no_grad():
for i, layer in enumerate(self.layers):
base = f"transformer.layers.{i}"
pairs = [
(layer.in_proj_weight, f"{base}.self_attn.in_proj_weight"),
(layer.out_proj_weight, f"{base}.self_attn.out_proj.weight"),
(layer.gating_in, f"{base}.gating.linear_in.weight"),
(layer.gating_out, f"{base}.gating.linear_out.weight"),
(layer.norm1_alpha, f"{base}.norm1.alpha"),
(layer.norm2_alpha, f"{base}.norm2.alpha"),
]
for param, name in pairs:
src = state_dict[name]
param.data.copy_(src.reshape(param.shape).to(param.dtype))
loaded += 1
self.out_norm_alpha.data.copy_(state_dict["out_norm.alpha"].reshape(self.out_norm_alpha.shape))
self.text_linear.data.copy_(state_dict["text_linear.weight"].to(self.text_linear.dtype))
loaded += 2
return loaded
@@ -0,0 +1,75 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PersonaPlex pipeline: Talker (AR decode -> Mimi codebooks) -> Code2Wav (codebooks -> 24 kHz PCM).
PersonaPlex is a Moshi finetune (full-duplex speech-to-speech). For offline/batch
runs it is served as a 2-stage vllm-omni audio->audio pipeline, reusing the
Qwen3-TTS staged topology:
* Stage 0 (``personaplex``) is the AR talker: the Helium temporal transformer
plus the depformer (both built by the lead). It emits the per-frame audio
codebooks as latents.
* Stage 1 (``code2wav``) wraps the external Mimi codec and decodes the active
codebooks (``cb 0..7``) into PCM.
The inter-stage input processors named below live in
``vllm_omni.model_executor.stage_input_processors.personaplex`` and are part of
the talker<->code2wav seam; they are built by the lead alongside the talker.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
# Talker<->Code2Wav input processors (built by lead; coupled to talker output).
_PROC = "vllm_omni.model_executor.stage_input_processors.personaplex"
PERSONAPLEX_PIPELINE = PipelineConfig(
model_type="personaplex",
# Pipeline-level default; the code2wav stage overrides per-stage below.
model_arch="PersonaPlexTalkerForConditionalGeneration",
default_deploy_config_name="personaplex.yaml",
duplex_runtime_extension=(
"vllm_omni.experimental.fullduplex.personaplex.runtime_extension.PersonaPlexDuplexRuntimeExtension"
),
duplex_serving_adapter=(
"vllm_omni.experimental.fullduplex.personaplex.serving_adapter.PersonaPlexServingRuntimeAdapter"
),
duplex_control_enabled=True,
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="personaplex",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
# built by lead: talker emits a 17-row Mimi token stack per frame;
# these processors keep the PCM-bearing rows (cb 0..7) and flatten
# them codebook-major for the Code2Wav stage.
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
sampling_constraints={
"detokenize": False,
},
),
StagePipelineConfig(
stage_id=1,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="PersonaPlexCode2Wav",
# In sync (non-async-chunk) mode the only pre-stage input proc is a
# length-only placeholder; the bulk codec payload ships via the
# worker connector from stage 0's full-payload producer.
# built by lead.
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
sampling_constraints={"detokenize": True},
),
),
)
@@ -164,6 +164,18 @@ _OMNI_MODELS = {
"qwen3_tts_code2wav",
"Qwen3TTSCode2Wav",
),
## personaplex (Moshi finetune, full-duplex S2S; 2-stage audio->audio offline pipeline)
# built by lead: personaplex_talker module + PersonaPlexTalkerForConditionalGeneration.
"PersonaPlexTalkerForConditionalGeneration": (
"personaplex",
"personaplex_talker",
"PersonaPlexTalkerForConditionalGeneration",
),
"PersonaPlexCode2Wav": (
"personaplex",
"personaplex_code2wav",
"PersonaPlexCode2Wav",
),
## higgs-audio v2
"HiggsAudioV2ForConditionalGeneration": (
"higgs_audio_v2",