[Feature] PersonaPlex (Moshi-based full-duplex S2S): native vLLM port + duplex serving (#4771)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: Sy0307 <24841857+Sy0307@users.noreply.github.com> Co-authored-by: Sy0307 <24841857+Sy0307@users.noreply.github.com>
This commit is contained in:
1 parent
f24a6165ff
commit
e788ef6e76
59 files changed
+10389
-12
No files matched your search
@@ -0,0 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vllm-omni integration for PersonaPlex (a Moshi finetune, full-duplex S2S).
|
||||
|
||||
Offline/batch runs go through a two-stage audio->audio pipeline (talker ->
|
||||
code2wav); real-time conversation is served over the duplex WebSocket path.
|
||||
"""
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
|
||||
PersonaPlexConfig,
|
||||
PersonaPlexDepformerConfig,
|
||||
PersonaPlexMimiConfig,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"PersonaPlexConfig",
|
||||
"PersonaPlexDepformerConfig",
|
||||
"PersonaPlexMimiConfig",
|
||||
]
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for the PersonaPlex Helium temporal transformer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
__all__ = ["HeliumConfig"]
|
||||
|
||||
|
||||
class HeliumConfig(PretrainedConfig):
|
||||
"""Minimal HF config for the Moshi temporal LM backbone.
|
||||
|
||||
The defaults are measured from the PersonaPlex checkpoint rather than inferred
|
||||
from Moshi's higher-level kwargs.
|
||||
"""
|
||||
|
||||
model_type = "helium"
|
||||
keys_to_ignore_at_inference = ("past_key_values",)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 4096,
|
||||
num_hidden_layers: int = 32,
|
||||
num_attention_heads: int = 32,
|
||||
head_dim: int = 128,
|
||||
num_key_value_heads: int = 32,
|
||||
intermediate_size: int = 11264,
|
||||
max_position_embeddings: int = 3000,
|
||||
rope_theta: float = 10000.0,
|
||||
rms_norm_eps: float = 1e-8,
|
||||
vocab_size: int = 32000,
|
||||
sliding_window: int = 3000,
|
||||
hidden_act: str = "silu",
|
||||
attention_bias: bool = False,
|
||||
mlp_bias: bool = False,
|
||||
tie_word_embeddings: bool = False,
|
||||
use_cache: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.head_dim = head_dim
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.rope_theta = rope_theta
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.vocab_size = vocab_size
|
||||
self.sliding_window = sliding_window
|
||||
self.hidden_act = hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_cache = use_cache
|
||||
@@ -0,0 +1,278 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Configuration for PersonaPlex (a Moshi finetune; 2-stage audio->audio pipeline).
|
||||
|
||||
PersonaPlex is a staged AR speech model composed of:
|
||||
|
||||
* a *temporal transformer* (a Llama-variant Helium backbone) that predicts, per
|
||||
frame, the next text token and the codebook-0 audio token;
|
||||
* a *depformer* (a small per-step transformer) that, conditioned on the temporal
|
||||
hidden state, predicts the remaining audio codebooks;
|
||||
* the Mimi neural audio codec, which turns the audio codebooks into 24 kHz PCM.
|
||||
|
||||
The temporal-transformer hyperparameters live in
|
||||
:class:`~vllm_omni.model_executor.models.personaplex.configuration_helium.HeliumConfig`
|
||||
(already used by the talker the lead is building). This module reuses it as the
|
||||
``temporal_config`` sub-config so the config tree carries a single source of
|
||||
truth, and adds two further sub-configs (``depformer_config`` and
|
||||
``mimi_config``).
|
||||
|
||||
All defaults are *measured* from the PersonaPlex checkpoint, whose own
|
||||
``config.json`` is empty; do not treat the Moshi higher-level kwargs as
|
||||
authoritative.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_helium import (
|
||||
HeliumConfig,
|
||||
)
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"PersonaPlexConfig",
|
||||
"PersonaPlexDepformerConfig",
|
||||
"PersonaPlexMimiConfig",
|
||||
"HeliumConfig",
|
||||
]
|
||||
|
||||
|
||||
class PersonaPlexDepformerConfig(PretrainedConfig):
|
||||
r"""Configuration for the PersonaPlex depformer (per-step code predictor).
|
||||
|
||||
The depformer is a small autoregressive transformer that runs ``dep_q``
|
||||
inner steps per temporal frame, conditioned on the temporal hidden state,
|
||||
to predict the audio codebooks. Only the first ``num_active_codebooks``
|
||||
codebooks (``cb 0..7``) are decoded to PCM by Mimi; the remaining codebooks
|
||||
up to ``dep_q`` are predicted but not vocoded.
|
||||
|
||||
Args:
|
||||
hidden_size (`int`, *optional*, defaults to 1024):
|
||||
Dimension of the depformer hidden representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 6):
|
||||
Number of depformer transformer layers.
|
||||
num_attention_heads (`int`, *optional*, defaults to 16):
|
||||
Number of attention heads per depformer layer.
|
||||
head_dim (`int`, *optional*, defaults to 64):
|
||||
Per-head attention dimension (``hidden_size // num_attention_heads``).
|
||||
max_position_embeddings (`int`, *optional*, defaults to 8):
|
||||
The depformer context length (``ctx``): one position per inner step.
|
||||
dep_q (`int`, *optional*, defaults to 16):
|
||||
Number of audio codebooks the depformer predicts per frame.
|
||||
num_active_codebooks (`int`, *optional*, defaults to 8):
|
||||
Number of leading codebooks actually decoded to PCM by Mimi.
|
||||
card (`int`, *optional*, defaults to 2048):
|
||||
Per-codebook cardinality (audio vocab size).
|
||||
rope_theta (`float`, *optional*, defaults to 10000.0):
|
||||
The base period of the RoPE embeddings.
|
||||
rms_norm_eps (`float`, *optional*, defaults to 1e-8):
|
||||
The epsilon used by the fp32 RMS normalization layers.
|
||||
hidden_act (`str`, *optional*, defaults to `"silu"`):
|
||||
SwiGLU gate activation.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether attention projections carry a bias.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether the MLP projections carry a bias.
|
||||
"""
|
||||
|
||||
model_type = "personaplex_depformer"
|
||||
keys_to_ignore_at_inference = ("past_key_values",)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 1024,
|
||||
num_hidden_layers: int = 6,
|
||||
num_attention_heads: int = 16,
|
||||
head_dim: int = 64,
|
||||
num_key_value_heads: int = 16,
|
||||
intermediate_size: int = 2816,
|
||||
max_position_embeddings: int = 8,
|
||||
dep_q: int = 16,
|
||||
num_active_codebooks: int = 8,
|
||||
card: int = 2048,
|
||||
rope_theta: float = 10000.0,
|
||||
rms_norm_eps: float = 1e-8,
|
||||
hidden_act: str = "silu",
|
||||
attention_bias: bool = False,
|
||||
mlp_bias: bool = False,
|
||||
tie_word_embeddings: bool = False,
|
||||
use_cache: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.head_dim = head_dim
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.dep_q = dep_q
|
||||
self.num_active_codebooks = num_active_codebooks
|
||||
self.card = card
|
||||
self.rope_theta = rope_theta
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.hidden_act = hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_cache = use_cache
|
||||
|
||||
|
||||
class PersonaPlexMimiConfig(PretrainedConfig):
|
||||
r"""Configuration for the Mimi neural audio codec used by PersonaPlex.
|
||||
|
||||
The Mimi weights and module live in the external ``moshi`` package; this
|
||||
config only carries the scalars the vllm-omni serving layer needs to size
|
||||
buffers and compute audio durations. The actual decoder is instantiated by
|
||||
:class:`PersonaPlexCode2Wav` via ``moshi.models.loaders.get_mimi``.
|
||||
|
||||
Args:
|
||||
sample_rate (`int`, *optional*, defaults to 24000):
|
||||
Output PCM sample rate in Hz.
|
||||
frame_rate (`float`, *optional*, defaults to 12.5):
|
||||
Mimi codec frame rate in Hz (one frame every 80 ms).
|
||||
samples_per_frame (`int`, *optional*, defaults to 1920):
|
||||
PCM samples produced per codec frame (``sample_rate / frame_rate``).
|
||||
num_codebooks (`int`, *optional*, defaults to 8):
|
||||
Number of active audio codebooks decoded to PCM (``cb 0..7``).
|
||||
card (`int`, *optional*, defaults to 2048):
|
||||
Per-codebook cardinality.
|
||||
num_channels (`int`, *optional*, defaults to 1):
|
||||
Number of output audio channels (mono).
|
||||
mimi_name (`str`, *optional*):
|
||||
Filename of the Mimi weight checkpoint inside the model repo. When
|
||||
``None``, :class:`PersonaPlexCode2Wav` falls back to
|
||||
``moshi.models.loaders.MIMI_NAME``.
|
||||
"""
|
||||
|
||||
model_type = "personaplex_mimi"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 24000,
|
||||
frame_rate: float = 12.5,
|
||||
samples_per_frame: int = 1920,
|
||||
num_codebooks: int = 8,
|
||||
card: int = 2048,
|
||||
num_channels: int = 1,
|
||||
mimi_name: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.sample_rate = sample_rate
|
||||
self.frame_rate = frame_rate
|
||||
self.samples_per_frame = samples_per_frame
|
||||
self.num_codebooks = num_codebooks
|
||||
self.card = card
|
||||
self.num_channels = num_channels
|
||||
self.mimi_name = mimi_name
|
||||
|
||||
|
||||
class PersonaPlexConfig(PretrainedConfig):
|
||||
r"""Top-level configuration for ``PersonaPlexTalkerForConditionalGeneration``.
|
||||
|
||||
Mirrors the Qwen3-TTS config layout: a top-level config holding sub-configs
|
||||
for each component. The temporal transformer config is the *text config*
|
||||
that vLLM consumes (it exposes ``hidden_size`` / ``num_attention_heads`` and
|
||||
drives the talker's vLLM Llama backbone).
|
||||
|
||||
Args:
|
||||
temporal_config (`dict` or `HeliumConfig`, *optional*):
|
||||
The temporal-transformer (Helium) backbone config.
|
||||
depformer_config (`dict` or `PersonaPlexDepformerConfig`, *optional*):
|
||||
The depformer (per-step code predictor) config.
|
||||
mimi_config (`dict` or `PersonaPlexMimiConfig`, *optional*):
|
||||
The Mimi codec config.
|
||||
text_vocab_size (`int`, *optional*, defaults to 32000):
|
||||
Text / ``lm_head`` vocabulary size.
|
||||
text_embedding_rows (`int`, *optional*, defaults to 32001):
|
||||
Number of rows in the text embedding table (one extra padding row).
|
||||
audio_vocab_size (`int`, *optional*, defaults to 2048):
|
||||
Per-codebook audio cardinality (``card``).
|
||||
num_audio_codebooks (`int`, *optional*, defaults to 16):
|
||||
Total number of audio codebooks (``n_q``).
|
||||
mimi_name (`str`, *optional*):
|
||||
Convenience mirror of ``mimi_config.mimi_name``; if set, it
|
||||
overrides the value carried inside ``mimi_config``.
|
||||
"""
|
||||
|
||||
model_type = "personaplex"
|
||||
sub_configs = {
|
||||
"temporal_config": HeliumConfig,
|
||||
"depformer_config": PersonaPlexDepformerConfig,
|
||||
"mimi_config": PersonaPlexMimiConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
temporal_config: dict | HeliumConfig | None = None,
|
||||
depformer_config: dict | PersonaPlexDepformerConfig | None = None,
|
||||
mimi_config: dict | PersonaPlexMimiConfig | None = None,
|
||||
text_vocab_size: int = 32000,
|
||||
text_embedding_rows: int = 32001,
|
||||
audio_vocab_size: int = 2048,
|
||||
num_audio_codebooks: int = 16,
|
||||
mimi_name: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if temporal_config is None:
|
||||
temporal_config = {}
|
||||
logger.info("temporal_config is None. Initializing Helium backbone with default values")
|
||||
if depformer_config is None:
|
||||
depformer_config = {}
|
||||
logger.info("depformer_config is None. Initializing depformer with default values")
|
||||
if mimi_config is None:
|
||||
mimi_config = {}
|
||||
logger.info("mimi_config is None. Initializing Mimi codec config with default values")
|
||||
|
||||
self.temporal_config = self._coerce(temporal_config, HeliumConfig)
|
||||
self.depformer_config = self._coerce(depformer_config, PersonaPlexDepformerConfig)
|
||||
self.mimi_config = self._coerce(mimi_config, PersonaPlexMimiConfig)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
# The PersonaPlex checkpoint's config.json is empty (no ``architectures``);
|
||||
# supply the primary (stage-0 talker) arch so vLLM's ModelConfig resolves a
|
||||
# generate-capable model. The code2wav stage arch comes from the pipeline.
|
||||
if not getattr(self, "architectures", None):
|
||||
self.architectures = ["PersonaPlexTalkerForConditionalGeneration"]
|
||||
|
||||
self.text_vocab_size = text_vocab_size
|
||||
self.text_embedding_rows = text_embedding_rows
|
||||
self.audio_vocab_size = audio_vocab_size
|
||||
self.num_audio_codebooks = num_audio_codebooks
|
||||
|
||||
# ``mimi_name`` is allowed both at the top level (convenient for
|
||||
# hf_overrides / a flat config.json) and inside ``mimi_config``. Keep
|
||||
# them in sync, preferring an explicit top-level value.
|
||||
if mimi_name is not None:
|
||||
self.mimi_config.mimi_name = mimi_name
|
||||
self.mimi_name = self.mimi_config.mimi_name
|
||||
|
||||
@staticmethod
|
||||
def _coerce(value: Any, config_cls: type[PretrainedConfig]) -> PretrainedConfig:
|
||||
if isinstance(value, config_cls):
|
||||
return value
|
||||
if isinstance(value, dict):
|
||||
return config_cls(**value)
|
||||
raise TypeError(f"Expected dict or {config_cls.__name__}, got {type(value).__name__}")
|
||||
|
||||
@property
|
||||
def sample_rate(self) -> int:
|
||||
"""Output PCM sample rate in Hz (delegates to the Mimi config)."""
|
||||
return int(self.mimi_config.sample_rate)
|
||||
|
||||
def get_text_config(self, **kwargs: Any) -> PretrainedConfig:
|
||||
# vLLM expects the text config to expose hidden_size / num_attention_heads.
|
||||
# For PersonaPlex the temporal-transformer (Helium) config is the text
|
||||
# model config that drives the talker's vLLM Llama backbone.
|
||||
return self.temporal_config
|
||||
@@ -0,0 +1,548 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM-native PersonaPlex Helium temporal transformer.
|
||||
|
||||
The module maps the Moshi temporal LM backbone onto vLLM's Llama-style decoder
|
||||
components. It intentionally does not include Moshi's text/audio input
|
||||
embeddings or depformer; PersonaPlex feeds temporal `inputs_embeds` directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
PPMissingLayer,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_helium import (
|
||||
HeliumConfig,
|
||||
)
|
||||
|
||||
__all__ = ["HeliumForCausalLM", "HeliumModel"]
|
||||
|
||||
|
||||
class HeliumRMSNorm(RMSNorm):
|
||||
"""Moshi-compatible RMSNorm.
|
||||
|
||||
Moshi stores the parameter as `alpha` with shape `[1, 1, hidden]` and
|
||||
computes variance and the alpha multiply in fp32 before casting back to the
|
||||
input dtype. vLLM's parameter is named `weight`; the loader squeezes alpha.
|
||||
"""
|
||||
|
||||
def _forward_f32(self, x: torch.Tensor) -> torch.Tensor:
|
||||
output_dtype = x.dtype
|
||||
x_f32 = x.float()
|
||||
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
|
||||
x_f32 = x_f32 * torch.rsqrt(variance + self.variance_epsilon)
|
||||
if self.has_weight:
|
||||
x_f32 = x_f32 * self.weight.float()
|
||||
return x_f32.to(output_dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
return self._forward_f32(x)
|
||||
residual = residual + x
|
||||
return self._forward_f32(residual), residual
|
||||
|
||||
|
||||
class HeliumMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(f"Unsupported activation: {hidden_act}. Only silu is supported.")
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
output_sizes=[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
input_size=intermediate_size,
|
||||
output_size=hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
def _get_helium_rope(config: HeliumConfig):
|
||||
rope_kwargs = {
|
||||
"head_size": config.head_dim,
|
||||
"max_position": config.max_position_embeddings,
|
||||
"is_neox_style": False,
|
||||
"rope_parameters": {"base": config.rope_theta},
|
||||
}
|
||||
try:
|
||||
return get_rope(**rope_kwargs)
|
||||
except TypeError:
|
||||
# vLLM 0.23 uses `rope_parameters`; this fallback is for older local
|
||||
# source trees whose get_rope still takes rotary_dim/base/rope_scaling.
|
||||
return get_rope(
|
||||
head_size=config.head_dim,
|
||||
rotary_dim=config.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
base=config.rope_theta,
|
||||
is_neox_style=False,
|
||||
)
|
||||
|
||||
|
||||
class HeliumAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: HeliumConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = config.head_dim
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.total_num_heads,
|
||||
total_num_kv_heads=self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
input_size=self.total_num_heads * self.head_dim,
|
||||
output_size=hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.rotary_emb = _get_helium_rope(config)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
per_layer_sliding_window=config.sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class HeliumDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
config: HeliumConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config = config or vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.self_attn = HeliumAttention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
self.mlp = HeliumMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = HeliumRMSNorm(
|
||||
config.hidden_size,
|
||||
eps=config.rms_norm_eps,
|
||||
)
|
||||
self.post_attention_layernorm = HeliumRMSNorm(
|
||||
config.hidden_size,
|
||||
eps=config.rms_norm_eps,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
|
||||
hidden_states, residual = self.post_attention_layernorm(
|
||||
hidden_states,
|
||||
residual,
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _flatten_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, tuple[int, int] | None]:
|
||||
if hidden_states.dim() != 3:
|
||||
return hidden_states, positions.reshape(-1), None
|
||||
|
||||
batch, seq_len, hidden = hidden_states.shape
|
||||
flat_hidden = hidden_states.reshape(batch * seq_len, hidden)
|
||||
if positions.dim() == 1:
|
||||
if positions.numel() == seq_len:
|
||||
flat_positions = positions.unsqueeze(0).expand(batch, seq_len).reshape(-1)
|
||||
elif positions.numel() == batch * seq_len:
|
||||
flat_positions = positions.reshape(-1)
|
||||
else:
|
||||
raise ValueError(
|
||||
"1D positions must have length S or B*S when inputs_embeds is "
|
||||
f"3D, got {positions.numel()} for B={batch}, S={seq_len}."
|
||||
)
|
||||
else:
|
||||
flat_positions = positions.reshape(-1)
|
||||
return flat_hidden, flat_positions, (batch, seq_len)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": 0,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
}
|
||||
)
|
||||
class HeliumModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
layer_type: type[nn.Module] = HeliumDecoderLayer,
|
||||
config: HeliumConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# ``config`` lets a composite model (e.g. the PersonaPlex talker) build the
|
||||
# Helium backbone from its temporal sub-config while the engine's top-level
|
||||
# hf_config is the composite PersonaPlexConfig. Defaults to the engine's.
|
||||
config = config or vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: layer_type(vllm_config=vllm_config, prefix=prefix, config=config),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = HeliumRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"],
|
||||
config.hidden_size,
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
raise ValueError(
|
||||
"Helium temporal forward requires inputs_embeds; Moshi token "
|
||||
"embeddings are intentionally not part of this vLLM module."
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
unflatten_shape: tuple[int, int] | None = None
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is None:
|
||||
raise ValueError("HeliumModel.forward requires inputs_embeds.")
|
||||
hidden_states, positions, unflatten_shape = _flatten_inputs(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
positions = positions.reshape(-1)
|
||||
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
residual,
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states, "residual": residual})
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
if unflatten_shape is not None:
|
||||
batch, seq_len = unflatten_shape
|
||||
hidden_states = hidden_states.reshape(batch, seq_len, -1)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class HeliumForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
embedding_modules = {
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
layer_type: type[nn.Module] = HeliumDecoderLayer,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.config = config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.model = HeliumModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
layer_type=layer_type,
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
return self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
if hidden_states.dim() == 3:
|
||||
batch, seq_len, hidden = hidden_states.shape
|
||||
flat_hidden = hidden_states.reshape(batch * seq_len, hidden)
|
||||
logits = self.logits_processor(self.lm_head, flat_hidden)
|
||||
if logits is None:
|
||||
return None
|
||||
return logits.reshape(batch, seq_len, -1)
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]] | Mapping[str, torch.Tensor],
|
||||
) -> set[str]:
|
||||
if isinstance(weights, Mapping):
|
||||
weights = weights.items()
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
mapped = self._load_moshi_weight(name, loaded_weight, params_dict)
|
||||
loaded_params.update(mapped)
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _load_moshi_weight(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
params_dict: dict[str, nn.Parameter],
|
||||
) -> set[str]:
|
||||
if name == "out_norm.alpha":
|
||||
return self._load_direct(
|
||||
"model.norm.weight",
|
||||
loaded_weight.squeeze(),
|
||||
params_dict,
|
||||
)
|
||||
if name == "text_linear.weight":
|
||||
return self._load_direct("lm_head.weight", loaded_weight, params_dict)
|
||||
|
||||
prefix = "transformer.layers."
|
||||
if not name.startswith(prefix):
|
||||
return set()
|
||||
|
||||
rest = name.removeprefix(prefix)
|
||||
layer_index, _, suffix = rest.partition(".")
|
||||
if not layer_index.isdigit() or not suffix:
|
||||
return set()
|
||||
|
||||
base = f"model.layers.{layer_index}"
|
||||
if suffix == "self_attn.in_proj_weight":
|
||||
q_weight, k_weight, v_weight = loaded_weight.chunk(3, dim=0)
|
||||
param_name = f"{base}.self_attn.qkv_proj.weight"
|
||||
self._load_shard(param_name, q_weight, "q", params_dict)
|
||||
self._load_shard(param_name, k_weight, "k", params_dict)
|
||||
self._load_shard(param_name, v_weight, "v", params_dict)
|
||||
return {param_name}
|
||||
if suffix == "self_attn.out_proj.weight":
|
||||
return self._load_direct(
|
||||
f"{base}.self_attn.o_proj.weight",
|
||||
loaded_weight,
|
||||
params_dict,
|
||||
)
|
||||
if suffix == "gating.linear_in.weight":
|
||||
gate_weight, up_weight = loaded_weight.chunk(2, dim=0)
|
||||
param_name = f"{base}.mlp.gate_up_proj.weight"
|
||||
self._load_shard(param_name, gate_weight, 0, params_dict)
|
||||
self._load_shard(param_name, up_weight, 1, params_dict)
|
||||
return {param_name}
|
||||
if suffix == "gating.linear_out.weight":
|
||||
return self._load_direct(
|
||||
f"{base}.mlp.down_proj.weight",
|
||||
loaded_weight,
|
||||
params_dict,
|
||||
)
|
||||
if suffix == "norm1.alpha":
|
||||
return self._load_direct(
|
||||
f"{base}.input_layernorm.weight",
|
||||
loaded_weight.squeeze(),
|
||||
params_dict,
|
||||
)
|
||||
if suffix == "norm2.alpha":
|
||||
return self._load_direct(
|
||||
f"{base}.post_attention_layernorm.weight",
|
||||
loaded_weight.squeeze(),
|
||||
params_dict,
|
||||
)
|
||||
return set()
|
||||
|
||||
def _load_direct(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
params_dict: dict[str, nn.Parameter],
|
||||
) -> set[str]:
|
||||
if name not in params_dict or is_pp_missing_parameter(name, self):
|
||||
return set()
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
return {name}
|
||||
|
||||
def _load_shard(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str | int,
|
||||
params_dict: dict[str, nn.Parameter],
|
||||
) -> None:
|
||||
if name not in params_dict or is_pp_missing_parameter(name, self):
|
||||
return
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
@@ -0,0 +1,433 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stage-1 Code2Wav model for PersonaPlex (Moshi finetune).
|
||||
|
||||
This is the ``LLM_GENERATION`` codec stage of the 2-stage PersonaPlex pipeline.
|
||||
It consumes the per-frame audio codebooks produced by the talker (stage 0) and
|
||||
the depformer (built by the lead), and turns them into 24 kHz PCM by calling the
|
||||
external Mimi neural codec from the ``moshi`` package.
|
||||
|
||||
Layout contract (mirrors ``Qwen3TTSCode2Wav``):
|
||||
|
||||
* Per request, ``input_ids`` holds a flat codebook-major codec sequence
|
||||
``[k * F]`` where ``k`` is the number of *active* codebooks Mimi decodes
|
||||
(``num_codebooks``, i.e. the ``cb 0..7`` slice) and ``F`` is the number of
|
||||
codec frames. The talker emits a 17-row token stack per frame; the input
|
||||
processor (built by the lead) keeps only rows ``1:9`` (the 8 PCM-bearing audio
|
||||
codebooks) and flattens them codebook-major before this stage.
|
||||
* ``forward(...)`` returns an :class:`OmniOutput` whose ``multimodal_outputs``
|
||||
carries ``{"model_outputs": [wav_per_request], "sr": [sr_per_request]}`` — the
|
||||
exact shape ``Qwen3TTSCode2Wav`` returns, so the downstream audio packer is
|
||||
unchanged.
|
||||
|
||||
The Mimi decoder is the transformers ``MimiModel`` (kyutai/mimi weights), not
|
||||
in vLLM's safetensors loader, so they are loaded eagerly in ``load_weights``
|
||||
part of the vLLM weights iterator (the codec owns its own checkpoint; see the
|
||||
experimental fullduplex package for the same pattern).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MIMI_DECODE_BATCH_FRAMES = 5
|
||||
|
||||
|
||||
def _codec_ids_from_payload_or_input(
|
||||
input_ids: torch.Tensor,
|
||||
runtime_info: Mapping[str, Any] | None,
|
||||
) -> torch.Tensor:
|
||||
"""Prefer connector-delivered codec ids over scheduler placeholders."""
|
||||
if isinstance(runtime_info, Mapping):
|
||||
codes = runtime_info.get("codes")
|
||||
if isinstance(codes, Mapping):
|
||||
audio = codes.get("audio")
|
||||
if isinstance(audio, torch.Tensor) and audio.numel() > 0:
|
||||
return audio.reshape(-1).to(device=input_ids.device, dtype=torch.long)
|
||||
if isinstance(audio, (list, tuple)) and audio:
|
||||
return torch.as_tensor(audio, device=input_ids.device, dtype=torch.long).reshape(-1)
|
||||
return input_ids.reshape(-1).to(dtype=torch.long)
|
||||
|
||||
|
||||
class PersonaPlexCode2Wav(nn.Module):
|
||||
"""Stage-1 Code2Wav model for PersonaPlex (GenerationModelRunner).
|
||||
|
||||
Wraps the external Mimi decoder. The vLLM generation runner only calls a
|
||||
handful of methods on this module: ``embed_input_ids`` (dummy embeddings),
|
||||
``compute_logits`` (none -- this stage never samples), ``forward`` (the
|
||||
actual codec->PCM decode), ``make_omni_output`` (output normalization), and
|
||||
``load_weights`` (eager Mimi construction).
|
||||
"""
|
||||
|
||||
input_modalities = "audio"
|
||||
requires_request_ids = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
|
||||
# Runner-facing capability flags, matching Qwen3TTSCode2Wav so the
|
||||
# GenerationModelRunner drives this stage the same way.
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
mimi_cfg = getattr(self.config, "mimi_config", None)
|
||||
# Number of *active* codebooks Mimi decodes to PCM (cb 0..7).
|
||||
self._num_codebooks = int(getattr(mimi_cfg, "num_codebooks", 8))
|
||||
self._output_sample_rate = int(getattr(mimi_cfg, "sample_rate", 24000))
|
||||
self._samples_per_frame = int(getattr(mimi_cfg, "samples_per_frame", 1920))
|
||||
self._mimi_name = getattr(mimi_cfg, "mimi_name", None) or getattr(self.config, "mimi_name", None)
|
||||
self._max_codec_sessions = int(getattr(vllm_config.model_config, "duplex_max_sessions", 1))
|
||||
|
||||
# The Mimi module is constructed in load_weights() (it owns its own
|
||||
# weight format) and assigned here so vLLM's memory profiler can see it.
|
||||
self.mimi: nn.Module | None = None
|
||||
self._additional_mimi = nn.ModuleList()
|
||||
self._mimi_device: torch.device | None = None
|
||||
self._request_codes: dict[str, torch.Tensor] = {}
|
||||
self._request_codec_slots: dict[str, int] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Runner-facing no-op / placeholder hooks (mirror Qwen3TTSCode2Wav).
|
||||
# ------------------------------------------------------------------
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
# This stage ignores token embeddings; keep a stable dummy embedding so
|
||||
# the vLLM runner has a valid hidden-state placeholder.
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor | OmniOutput,
|
||||
sampling_metadata: Any = None,
|
||||
) -> None:
|
||||
# Code2Wav never samples; the runner short-circuits when logits are None.
|
||||
return None
|
||||
|
||||
def get_dummy_runtime_additional_information(self, num_reqs: int) -> list[dict[str, object]]:
|
||||
return [{"meta": {"personaplex_dummy_profile": True}} for _ in range(num_reqs)]
|
||||
|
||||
def _split_request_ids(
|
||||
self,
|
||||
ids: torch.Tensor,
|
||||
seq_token_counts: list[int] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Split concatenated ``input_ids`` into per-request segments.
|
||||
|
||||
Uses ``seq_token_counts`` (injected by the runner via model_kwargs) when
|
||||
available, falling back to forward-context ``ubatch_slices`` when
|
||||
micro-batching is active. Returns ``[ids]`` for single-request batches.
|
||||
Mirrors ``Qwen3TTSCode2Wav._split_request_ids``.
|
||||
"""
|
||||
if seq_token_counts is not None and len(seq_token_counts) > 1:
|
||||
boundaries = [0]
|
||||
for count in seq_token_counts:
|
||||
boundaries.append(boundaries[-1] + count)
|
||||
n = ids.numel()
|
||||
return [ids[boundaries[i] : min(boundaries[i + 1], n)] for i in range(len(seq_token_counts))]
|
||||
if is_forward_context_available():
|
||||
slices = get_forward_context().ubatch_slices
|
||||
if slices is not None and len(slices) > 1 and not any(hasattr(s, "token_slice") for s in slices):
|
||||
boundaries = [0]
|
||||
for s in slices:
|
||||
boundaries.append(boundaries[-1] + s)
|
||||
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
|
||||
return [ids]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Decode.
|
||||
# ------------------------------------------------------------------
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
"""Decode flat codebook-major codec ids into PCM via Mimi.
|
||||
|
||||
``input_ids`` per request is ``[k * F]`` (codebook-major), where ``k``
|
||||
is ``self._num_codebooks``. The connector may send either a new delta
|
||||
chunk or the cumulative prefix of a resumable request. Request-local
|
||||
code history identifies the new suffix, and the streaming Mimi decoder
|
||||
consumes each new frame exactly once.
|
||||
"""
|
||||
sr_val = int(self._output_sample_rate)
|
||||
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
|
||||
empty = torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
if input_ids is None or input_ids.numel() == 0:
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
|
||||
)
|
||||
|
||||
ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
|
||||
num_req = len(request_ids_list)
|
||||
state_ids = self._resolve_request_ids(
|
||||
num_req,
|
||||
kwargs.get("request_ids"),
|
||||
runtime_additional_information,
|
||||
)
|
||||
|
||||
if self.mimi is None:
|
||||
raise RuntimeError("PersonaPlexCode2Wav.forward called before Mimi was loaded in load_weights().")
|
||||
|
||||
k = int(self._num_codebooks)
|
||||
device = self._mimi_device or ids.device
|
||||
audios: list[torch.Tensor] = [empty] * num_req
|
||||
srs = [sr_tensor] * num_req
|
||||
|
||||
for i, req_ids in enumerate(request_ids_list):
|
||||
runtime_info = (
|
||||
runtime_additional_information[i]
|
||||
if runtime_additional_information is not None and i < len(runtime_additional_information)
|
||||
else None
|
||||
)
|
||||
meta = runtime_info.get("meta") if isinstance(runtime_info, Mapping) else None
|
||||
if isinstance(meta, Mapping) and meta.get("personaplex_dummy_profile") is True:
|
||||
continue
|
||||
flat = _codec_ids_from_payload_or_input(req_ids, runtime_info)
|
||||
n = int(flat.numel())
|
||||
if n == 0 or n % k != 0:
|
||||
if n > 0:
|
||||
logger.warning(
|
||||
"PersonaPlex Code2Wav input_ids length %d not divisible by "
|
||||
"num_codebooks %d; skipping malformed request.",
|
||||
n,
|
||||
k,
|
||||
)
|
||||
continue
|
||||
frames = n // k
|
||||
codes_kf = flat.reshape(k, frames)
|
||||
state_id = state_ids[i]
|
||||
delta_kf = self._new_code_suffix(state_id, codes_kf)
|
||||
if delta_kf.shape[1] == 0:
|
||||
continue
|
||||
wav = self._decode_streaming_frames(state_id, delta_kf.to(device=device))
|
||||
if wav.numel() > 0:
|
||||
audios[i] = wav.to(dtype=torch.float32).reshape(-1)
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": audios, "sr": srs},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_request_ids(
|
||||
count: int,
|
||||
request_ids: object,
|
||||
runtime_additional_information: list[dict[str, Any]] | None,
|
||||
) -> list[str | None]:
|
||||
if isinstance(request_ids, list | tuple):
|
||||
if len(request_ids) != count:
|
||||
raise ValueError(
|
||||
"PersonaPlex Code2Wav request id count does not match inputs: "
|
||||
f"request_ids={len(request_ids)}, inputs={count}"
|
||||
)
|
||||
return [str(request_id) for request_id in request_ids]
|
||||
|
||||
infos = runtime_additional_information or []
|
||||
resolved: list[str | None] = []
|
||||
for index in range(count):
|
||||
info = infos[index] if index < len(infos) else None
|
||||
request_id = None
|
||||
if isinstance(info, Mapping):
|
||||
request_id = info.get("request_id")
|
||||
if request_id is None:
|
||||
meta = info.get("meta")
|
||||
if isinstance(meta, Mapping):
|
||||
request_id = meta.get("request_id")
|
||||
resolved.append(str(request_id) if request_id is not None else None)
|
||||
return resolved
|
||||
|
||||
def _new_code_suffix(self, request_id: str | None, codes_kf: torch.Tensor) -> torch.Tensor:
|
||||
if request_id is None:
|
||||
return codes_kf
|
||||
|
||||
incoming = codes_kf.detach().to(device="cpu", dtype=torch.long)
|
||||
previous = self._request_codes.get(request_id)
|
||||
if (
|
||||
previous is not None
|
||||
and incoming.shape[1] >= previous.shape[1]
|
||||
and torch.equal(incoming[:, : previous.shape[1]], previous)
|
||||
):
|
||||
delta = incoming[:, previous.shape[1] :]
|
||||
self._request_codes[request_id] = incoming
|
||||
return delta
|
||||
|
||||
self._request_codes[request_id] = incoming if previous is None else torch.cat([previous, incoming], dim=1)
|
||||
return incoming
|
||||
|
||||
def _decode_streaming_frames(self, request_id: str | None, codes_kf: torch.Tensor) -> torch.Tensor:
|
||||
codec, ephemeral = self._codec_for_request(request_id)
|
||||
if not hasattr(codec, "decode_frame"):
|
||||
raise RuntimeError("PersonaPlex Code2Wav requires a streaming Mimi decoder")
|
||||
|
||||
decode_frames = getattr(codec, "decode_frames", None)
|
||||
chunks: list[torch.Tensor] = []
|
||||
for start in range(0, codes_kf.shape[1], _MIMI_DECODE_BATCH_FRAMES):
|
||||
frame_batch = codes_kf[:, start : start + _MIMI_DECODE_BATCH_FRAMES]
|
||||
if frame_batch.shape[1] > 1 and callable(decode_frames):
|
||||
chunks.append(self._flatten_wav(decode_frames(frame_batch.unsqueeze(0))))
|
||||
else:
|
||||
chunks.extend(
|
||||
self._flatten_wav(codec.decode_frame(frame_batch[:, frame].unsqueeze(0)))
|
||||
for frame in range(frame_batch.shape[1])
|
||||
)
|
||||
wav = torch.cat(chunks, dim=0) if chunks else codes_kf.new_empty(0, dtype=torch.float32)
|
||||
if ephemeral:
|
||||
codec.reset_streaming()
|
||||
return wav
|
||||
|
||||
def _set_mimi_codecs(self, codecs: list[nn.Module]) -> None:
|
||||
if not codecs:
|
||||
raise ValueError("PersonaPlex Code2Wav requires at least one Mimi decoder")
|
||||
self.mimi = codecs[0]
|
||||
self._additional_mimi = nn.ModuleList(codecs[1:])
|
||||
self._request_codec_slots.clear()
|
||||
|
||||
def _mimi_codecs(self) -> list[nn.Module]:
|
||||
if self.mimi is None:
|
||||
return []
|
||||
return [self.mimi, *self._additional_mimi]
|
||||
|
||||
def _codec_for_request(self, request_id: str | None) -> tuple[nn.Module, bool]:
|
||||
codecs = self._mimi_codecs()
|
||||
if not codecs:
|
||||
raise RuntimeError("PersonaPlexCode2Wav.forward called before Mimi was loaded in load_weights().")
|
||||
occupied = set(self._request_codec_slots.values())
|
||||
if request_id is not None:
|
||||
slot = self._request_codec_slots.get(request_id)
|
||||
if slot is None:
|
||||
slot = next((index for index in range(len(codecs)) if index not in occupied), None)
|
||||
if slot is None:
|
||||
raise RuntimeError(f"PersonaPlex Code2Wav decoder capacity {len(codecs)} is exhausted")
|
||||
codecs[slot].streaming_init(1)
|
||||
self._request_codec_slots[request_id] = slot
|
||||
return codecs[slot], False
|
||||
|
||||
slot = next((index for index in range(len(codecs)) if index not in occupied), None)
|
||||
if slot is None:
|
||||
raise RuntimeError(f"PersonaPlex Code2Wav decoder capacity {len(codecs)} is exhausted")
|
||||
codecs[slot].streaming_init(1)
|
||||
return codecs[slot], True
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
for request_id in finished_req_ids:
|
||||
state_id = str(request_id)
|
||||
self._request_codes.pop(state_id, None)
|
||||
slot = self._request_codec_slots.pop(state_id, None)
|
||||
codecs = self._mimi_codecs()
|
||||
if slot is not None and slot < len(codecs):
|
||||
codecs[slot].reset_streaming()
|
||||
|
||||
@staticmethod
|
||||
def _flatten_wav(wav: torch.Tensor) -> torch.Tensor:
|
||||
"""Normalize Mimi's decode output to a 1D PCM tensor.
|
||||
|
||||
Mimi may return ``[B, samples]`` or ``[B, C, samples]`` depending on the
|
||||
moshi version; PersonaPlex is mono (``C == 1``), so collapse to 1D.
|
||||
"""
|
||||
if wav.dim() == 3:
|
||||
# [B, C, T] -> take batch 0, channel 0.
|
||||
return wav[0, 0]
|
||||
if wav.dim() == 2:
|
||||
# [B, T] -> batch 0.
|
||||
return wav[0]
|
||||
return wav.reshape(-1)
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput | tuple, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
|
||||
if isinstance(model_outputs, tuple) and len(model_outputs) == len(OmniOutput._fields):
|
||||
return OmniOutput(*model_outputs)
|
||||
|
||||
if not (isinstance(model_outputs, tuple) and len(model_outputs) == 2):
|
||||
raise TypeError(
|
||||
"PersonaPlexCode2Wav expected OmniOutput, OmniOutput tuple, "
|
||||
f"or (audio_tensor, sr) outputs, got {type(model_outputs)}"
|
||||
)
|
||||
|
||||
audio_tensor, sr = model_outputs
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={
|
||||
"model_outputs": audio_tensor,
|
||||
"sr": sr,
|
||||
},
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Construct and load the external Mimi decoder.
|
||||
|
||||
The primary vLLM weights iterator carries no Code2Wav parameters (Mimi
|
||||
owns its own checkpoint format), so it is drained and the Mimi module is
|
||||
built from the moshi package's loader.
|
||||
"""
|
||||
# Drain the primary iterator so callers don't hang on an unconsumed
|
||||
# generator; none of these weights belong to this stage.
|
||||
for _ in weights:
|
||||
pass
|
||||
|
||||
device = self.vllm_config.device_config.device
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_mimi import (
|
||||
PersonaPlexMimiCodec,
|
||||
)
|
||||
|
||||
checkpoint = Path(self.model_path) / (self._mimi_name or "tokenizer-e351c8d8-checkpoint125.safetensors")
|
||||
codecs = [
|
||||
PersonaPlexMimiCodec(
|
||||
checkpoint=str(checkpoint) if checkpoint.is_file() else None,
|
||||
device=str(device),
|
||||
).eval()
|
||||
for _ in range(self._max_codec_sessions)
|
||||
]
|
||||
self._set_mimi_codecs(codecs)
|
||||
self._mimi_device = torch.device(str(device))
|
||||
reported_sr = getattr(codecs[0].model.config, "sampling_rate", None)
|
||||
if reported_sr is not None:
|
||||
self._output_sample_rate = int(reported_sr)
|
||||
bytes_per_mib = 1024**2
|
||||
per_slot_weight_mib = (
|
||||
sum(parameter.numel() * parameter.element_size() for parameter in codecs[0].parameters()) / bytes_per_mib
|
||||
)
|
||||
total_weight_mib = (
|
||||
sum(parameter.numel() * parameter.element_size() for codec in codecs for parameter in codec.parameters())
|
||||
/ bytes_per_mib
|
||||
)
|
||||
logger.info(
|
||||
"PersonaPlex Code2Wav loaded %d Mimi stream(s): "
|
||||
"per_slot_weight_mib=%.2f total_weight_mib=%.2f "
|
||||
"num_codebooks=%d sample_rate=%d samples_per_frame=%d",
|
||||
len(codecs),
|
||||
per_slot_weight_mib,
|
||||
total_weight_mib,
|
||||
self._num_codebooks,
|
||||
self._output_sample_rate,
|
||||
self._samples_per_frame,
|
||||
)
|
||||
return {name for name, _ in self.named_parameters() if name.startswith(("mimi.", "_additional_mimi."))}
|
||||
@@ -0,0 +1,366 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PersonaPlex depformer: the per-step audio code predictor.
|
||||
|
||||
The depformer is the small autoregressive head that, conditioned on one temporal
|
||||
hidden state, predicts the ``dep_q`` audio codebooks of a single Mimi frame. It
|
||||
is a faithful eager port of Moshi's depformer (``moshi.models.lm`` +
|
||||
``moshi.modules.transformer``), kept deliberately outside vLLM's paged-attention
|
||||
engine: its attention KV is reset every frame and only spans the ``dep_q`` inner
|
||||
steps, so paging buys nothing here. This mirrors how the existing TTS code
|
||||
predictors live alongside, not inside, the AR talker.
|
||||
|
||||
Faithful-port details (all measured from ``nvidia/personaplex-7b-v1``):
|
||||
|
||||
* **Per-step weights** (``weights_per_step``): each inner step ``t`` uses its own
|
||||
attention in/out projection and its own gating MLP, selected by index (Moshi's
|
||||
``multi_linear``). There are ``dep_q`` weight sets per layer.
|
||||
* **No positional embedding** (``depformer_pos_emb="none"``): no RoPE, no sin.
|
||||
* **Causal attention over all inner steps**: Moshi forces the depformer's
|
||||
``context`` to None, so each step attends to every prior inner step (KV
|
||||
capacity == ``weights_per_step``); the KV cache is rebuilt every temporal frame.
|
||||
* **fp32 RMSNorm** (``rms_norm_f32``, eps 1e-8) with the weight stored as
|
||||
``alpha`` of shape ``[1, 1, dim]``.
|
||||
* **SiLU gating** MLP (Moshi ``ActivationGating``): ``linear_in`` projects to
|
||||
``2 * hidden``; the halves are combined as ``silu(a) * b`` then ``linear_out``.
|
||||
* **ScaledEmbedding** with ``zero_idx = -1`` (the special token maps to the zero
|
||||
vector); audio table is ``card + 1`` rows, text table ``text_card + 1``.
|
||||
|
||||
The checkpoint ships ``dep_q = 8`` weight sets; the production loader (Moshi's
|
||||
``get_moshi_lm``) overrides ``dep_q = 16`` and fills sets 8..15 by copying 0..7.
|
||||
:meth:`PersonaPlexDepformer.load_weights` replicates that expansion so the module
|
||||
loads directly from the raw ``model.safetensors`` as well as from an
|
||||
already-expanded Moshi instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
|
||||
PersonaPlexDepformerConfig,
|
||||
)
|
||||
|
||||
__all__ = ["PersonaPlexDepformer"]
|
||||
|
||||
# Moshi's special "no token / zero embedding" sentinel (``LMModel.zero_token_id``).
|
||||
_ZERO_IDX = -1
|
||||
|
||||
# A builder takes the source state dict and returns the tensor for one target
|
||||
# param, or None when the source lacks it.
|
||||
_Builder = Callable[[dict], "torch.Tensor | None"]
|
||||
|
||||
|
||||
def _rms_norm_f32(x: torch.Tensor, alpha: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
"""Moshi ``_rms_norm`` with the fp32 accumulation path.
|
||||
|
||||
Variance and the alpha multiply run in float32 before casting back, matching
|
||||
``moshi.modules.transformer._rms_norm`` exactly (``dtype=torch.float``).
|
||||
"""
|
||||
x_dtype = x.dtype
|
||||
x_f32 = x.float()
|
||||
var = eps + x_f32.pow(2).mean(dim=-1, keepdim=True)
|
||||
y = x_f32 * (alpha.float() * torch.rsqrt(var))
|
||||
return y.to(x_dtype)
|
||||
|
||||
|
||||
class _ScaledEmbedding(nn.Embedding):
|
||||
"""``nn.Embedding`` matching Moshi's ``ScaledEmbedding`` (no norm, zero_idx).
|
||||
|
||||
Inputs equal to ``zero_idx`` produce the zero vector; all other inputs are
|
||||
clamped to ``>= 0`` before the lookup. ``norm_emb`` is False for PersonaPlex,
|
||||
so no embedding norm is applied.
|
||||
"""
|
||||
|
||||
def __init__(self, num_embeddings: int, embedding_dim: int, zero_idx: int = _ZERO_IDX) -> None:
|
||||
super().__init__(num_embeddings, embedding_dim)
|
||||
self.zero_idx = zero_idx
|
||||
|
||||
def forward(self, idx: torch.Tensor) -> torch.Tensor: # type: ignore[override]
|
||||
is_zero = idx == self.zero_idx
|
||||
y = super().forward(idx.clamp(min=0))
|
||||
zero = torch.zeros(1, dtype=y.dtype, device=y.device)
|
||||
return torch.where(is_zero[..., None], zero, y)
|
||||
|
||||
|
||||
class _DepformerLayer(nn.Module):
|
||||
"""One depformer transformer layer with per-step attention + gating.
|
||||
|
||||
Attention and gating weights carry a leading ``dep_q`` axis; the active inner
|
||||
step selects the slice (Moshi ``multi_linear`` / ``gating[t]``). KV is supplied
|
||||
per call so the caller controls the per-frame reset and the sliding window.
|
||||
"""
|
||||
|
||||
def __init__(self, config: PersonaPlexDepformerConfig) -> None:
|
||||
super().__init__()
|
||||
dim = config.hidden_size
|
||||
self.dim = dim
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.head_dim = config.head_dim
|
||||
self.dep_q = config.dep_q
|
||||
self.eps = config.rms_norm_eps
|
||||
|
||||
# Per-step fused QKV and output projections, stored exactly as Moshi does:
|
||||
# ``in_proj_weight`` is [dep_q * 3*dim, dim]; ``out_proj`` is [dep_q*dim, dim].
|
||||
self.in_proj_weight = nn.Parameter(torch.empty(self.dep_q * 3 * dim, dim))
|
||||
self.out_proj_weight = nn.Parameter(torch.empty(self.dep_q * dim, dim))
|
||||
|
||||
# fp32 RMSNorm weights (Moshi ``alpha``, shape [1, 1, dim]).
|
||||
self.norm1_alpha = nn.Parameter(torch.ones(1, 1, dim))
|
||||
self.norm2_alpha = nn.Parameter(torch.ones(1, 1, dim))
|
||||
|
||||
# Per-step SiLU gating MLP. ``linear_in`` -> [2*hidden, dim]; ``linear_out`` -> [dim, hidden].
|
||||
hidden = config.intermediate_size
|
||||
self.gating_in = nn.Parameter(torch.empty(self.dep_q, 2 * hidden, dim))
|
||||
self.gating_out = nn.Parameter(torch.empty(self.dep_q, dim, hidden))
|
||||
|
||||
def _in_proj(self, step: int) -> torch.Tensor:
|
||||
return self.in_proj_weight.view(self.dep_q, 3 * self.dim, self.dim)[step]
|
||||
|
||||
def _out_proj(self, step: int) -> torch.Tensor:
|
||||
return self.out_proj_weight.view(self.dep_q, self.dim, self.dim)[step]
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
step: int,
|
||||
kv: dict[str, torch.Tensor | None],
|
||||
) -> torch.Tensor:
|
||||
"""Run inner step ``step`` for a ``[B, 1, dim]`` input, updating ``kv`` in place."""
|
||||
# --- self-attention (per-step weights, causal sliding window, no RoPE) ---
|
||||
residual = x
|
||||
h = _rms_norm_f32(x, self.norm1_alpha, self.eps).squeeze(1) # [B, dim]
|
||||
qkv = F.linear(h, self._in_proj(step)) # [B, 3*dim]
|
||||
q, k, v = qkv.split(self.dim, dim=-1)
|
||||
b = q.shape[0]
|
||||
q = q.view(b, self.num_heads, 1, self.head_dim)
|
||||
k = k.view(b, self.num_heads, 1, self.head_dim)
|
||||
v = v.view(b, self.num_heads, 1, self.head_dim)
|
||||
|
||||
if kv["k"] is None:
|
||||
k_hist, v_hist = k, v
|
||||
else:
|
||||
k_hist = torch.cat([kv["k"], k], dim=2)
|
||||
v_hist = torch.cat([kv["v"], v], dim=2)
|
||||
kv["k"], kv["v"] = k_hist, v_hist
|
||||
|
||||
# Moshi forces the depformer's ``context`` to None (lm.py: kwargs_dep[
|
||||
# "context"] = None), so attention spans ALL prior inner steps -- the KV
|
||||
# capacity is ``weights_per_step`` (== dep_q), never windowed. The query is
|
||||
# the newest step and every key is causal w.r.t. it, so no mask is needed.
|
||||
attn = F.scaled_dot_product_attention(q, k_hist, v_hist) # [B, H, 1, Dh]
|
||||
attn = attn.reshape(b, self.dim)
|
||||
out = F.linear(attn, self._out_proj(step)).unsqueeze(1) # [B, 1, dim]
|
||||
x = residual + out
|
||||
|
||||
# --- feed-forward (per-step SiLU gating) ---
|
||||
residual = x
|
||||
h = _rms_norm_f32(x, self.norm2_alpha, self.eps).squeeze(1) # [B, dim]
|
||||
gated = F.linear(h, self.gating_in[step]) # [B, 2*hidden]
|
||||
a, b_half = gated.chunk(2, dim=-1)
|
||||
update = F.linear(F.silu(a) * b_half, self.gating_out[step]).unsqueeze(1)
|
||||
return residual + update
|
||||
|
||||
|
||||
class PersonaPlexDepformer(nn.Module):
|
||||
"""Moshi depformer as a standalone per-frame code predictor.
|
||||
|
||||
Call :meth:`forward` once per temporal frame with the frame's text token and
|
||||
temporal hidden state; it runs ``dep_q`` inner AR steps (greedy by default,
|
||||
matching Moshi's ``use_sampling=False``) and returns the ``dep_q`` audio codes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PersonaPlexDepformerConfig,
|
||||
temporal_hidden_size: int,
|
||||
text_card: int = 32000,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.dep_q = config.dep_q
|
||||
self.card = config.card
|
||||
dim = config.hidden_size
|
||||
|
||||
# Per-codebook projection of the temporal hidden state (``depformer_multi_linear``).
|
||||
self.depformer_in = nn.ModuleList([nn.Linear(temporal_hidden_size, dim, bias=False) for _ in range(self.dep_q)])
|
||||
# Step-0 conditions on the text token; steps 1..dep_q-1 on the previous audio code.
|
||||
self.depformer_text_emb = _ScaledEmbedding(text_card + 1, dim)
|
||||
self.depformer_emb = nn.ModuleList([_ScaledEmbedding(self.card + 1, dim) for _ in range(self.dep_q - 1)])
|
||||
self.layers = nn.ModuleList([_DepformerLayer(config) for _ in range(config.num_hidden_layers)])
|
||||
# Per-codebook output heads (Moshi ``linears``).
|
||||
self.linears = nn.ModuleList([nn.Linear(dim, self.card, bias=False) for _ in range(self.dep_q)])
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
text_token: torch.Tensor,
|
||||
transformer_out: torch.Tensor,
|
||||
audio_tokens: torch.Tensor | None = None,
|
||||
audio_provided: torch.Tensor | None = None,
|
||||
return_logits: bool = False,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Predict the ``dep_q`` audio codes for one frame.
|
||||
|
||||
Args:
|
||||
text_token: ``[B]`` sampled text token for the frame.
|
||||
transformer_out: ``[B, 1, temporal_hidden]`` temporal hidden state.
|
||||
audio_tokens: optional ``[B, dep_q]`` teacher-forcing codes.
|
||||
audio_provided: optional ``[B, dep_q]`` bool mask; where True the
|
||||
corresponding ``audio_tokens`` entry overrides the sampled code as
|
||||
the next-step input (partial teacher forcing, as in Moshi).
|
||||
return_logits: also return the per-step logits ``[B, dep_q, card]``.
|
||||
|
||||
Returns:
|
||||
``[B, dep_q]`` long tensor of sampled (greedy) audio codes, plus the
|
||||
per-step logits when ``return_logits`` is set.
|
||||
"""
|
||||
if transformer_out.dim() != 3 or transformer_out.shape[1] != 1:
|
||||
raise ValueError(f"transformer_out must be [B, 1, H]; got {tuple(transformer_out.shape)}")
|
||||
prev = text_token
|
||||
kv = [{"k": None, "v": None} for _ in self.layers]
|
||||
codes: list[torch.Tensor] = []
|
||||
logits_all: list[torch.Tensor] = []
|
||||
for step in range(self.dep_q):
|
||||
x = self.depformer_in[step](transformer_out) # [B, 1, dim]
|
||||
if step == 0:
|
||||
cond = self.depformer_text_emb(prev.unsqueeze(1))
|
||||
else:
|
||||
cond = self.depformer_emb[step - 1](prev.unsqueeze(1))
|
||||
x = x + cond
|
||||
for li, layer in enumerate(self.layers):
|
||||
x = layer(x, step, kv[li])
|
||||
logits = self.linears[step](x).squeeze(1) # [B, card]
|
||||
sampled = logits.float().argmax(dim=-1) # greedy [B]
|
||||
codes.append(sampled)
|
||||
if return_logits:
|
||||
logits_all.append(logits.float())
|
||||
if audio_provided is not None and audio_tokens is not None:
|
||||
prev = torch.where(audio_provided[:, step], audio_tokens[:, step], sampled)
|
||||
else:
|
||||
prev = sampled
|
||||
stacked = torch.stack(codes, dim=1) # [B, dep_q]
|
||||
if return_logits:
|
||||
return stacked, torch.stack(logits_all, dim=1) # [B, dep_q, card]
|
||||
return stacked
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
# ------------------------------------------------------------------
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]] | Mapping[str, torch.Tensor],
|
||||
) -> set[str]:
|
||||
"""Load Moshi depformer weights, expanding ``dep_q`` 8 -> 16 if needed.
|
||||
|
||||
Accepts the raw checkpoint names (``depformer.*``, ``depformer_in.N``,
|
||||
``depformer_emb.N``, ``depformer_text_emb``, ``linears.N``). Per-step
|
||||
modules present only for steps 0..7 are copied to 8..15, and the fused
|
||||
attention projections are tiled, exactly as Moshi's ``get_moshi_lm`` does.
|
||||
"""
|
||||
src = dict(weights.items() if isinstance(weights, Mapping) else weights)
|
||||
src = {k: v for k, v in src.items() if self._is_depformer_key(k)}
|
||||
params = dict(self.named_parameters())
|
||||
loaded: set[str] = set()
|
||||
|
||||
for tgt, build in self._weight_plan().items():
|
||||
tensor = build(src)
|
||||
if tensor is None:
|
||||
continue
|
||||
param = params[tgt]
|
||||
if tensor.shape != param.shape:
|
||||
raise ValueError(
|
||||
f"shape mismatch for {tgt}: checkpoint {tuple(tensor.shape)} vs param {tuple(param.shape)}"
|
||||
)
|
||||
param.data.copy_(tensor.to(param.dtype))
|
||||
loaded.add(tgt)
|
||||
return loaded
|
||||
|
||||
@staticmethod
|
||||
def _is_depformer_key(name: str) -> bool:
|
||||
return name.startswith(("depformer.", "depformer_in.", "depformer_emb.", "depformer_text_emb", "linears."))
|
||||
|
||||
def _weight_plan(self) -> dict[str, _Builder]:
|
||||
"""Map each of this module's params to a builder over the source state dict."""
|
||||
dim = self.config.hidden_size
|
||||
|
||||
def step_or_copy(prefix: str, step: int, suffix: str = "weight") -> _Builder:
|
||||
# cb 8..15 fall back to cb (step-8) when absent (Moshi copy_missing_weights).
|
||||
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
|
||||
key = f"{prefix}.{step}.{suffix}"
|
||||
if key in src:
|
||||
return src[key]
|
||||
if step >= 8:
|
||||
return src.get(f"{prefix}.{step - 8}.{suffix}")
|
||||
return None
|
||||
|
||||
return build
|
||||
|
||||
def expand_attn(layer: int, kind: str, per_step_rows: int) -> _Builder:
|
||||
# ``in_proj_weight`` [steps*per_step_rows, dim] / ``out_proj.weight`` likewise;
|
||||
# tile an 8-step checkpoint block up to dep_q (Moshi concat-copy expansion).
|
||||
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
|
||||
key = f"depformer.layers.{layer}.self_attn.{kind}"
|
||||
t = src.get(key)
|
||||
if t is None:
|
||||
return None
|
||||
steps = t.shape[0] // per_step_rows
|
||||
if steps < self.dep_q:
|
||||
reps = (self.dep_q + steps - 1) // steps
|
||||
t = t.repeat(reps, 1)[: self.dep_q * per_step_rows]
|
||||
return t
|
||||
|
||||
return build
|
||||
|
||||
plan: dict[str, _Builder] = {}
|
||||
# depformer_in / linears / depformer_emb / text_emb
|
||||
for step in range(self.dep_q):
|
||||
plan[f"depformer_in.{step}.weight"] = step_or_copy("depformer_in", step)
|
||||
plan[f"linears.{step}.weight"] = step_or_copy("linears", step)
|
||||
plan["depformer_text_emb.weight"] = lambda src: src.get("depformer_text_emb.weight")
|
||||
for step in range(self.dep_q - 1):
|
||||
plan[f"depformer_emb.{step}.weight"] = step_or_copy("depformer_emb", step)
|
||||
|
||||
# per-layer attention + norms + gating
|
||||
for layer in range(self.config.num_hidden_layers):
|
||||
lp = f"depformer.layers.{layer}"
|
||||
plan[f"layers.{layer}.in_proj_weight"] = expand_attn(layer, "in_proj_weight", 3 * dim)
|
||||
plan[f"layers.{layer}.out_proj_weight"] = expand_attn(layer, "out_proj.weight", dim)
|
||||
plan[f"layers.{layer}.norm1_alpha"] = self._reshape_alpha(f"{lp}.norm1.alpha")
|
||||
plan[f"layers.{layer}.norm2_alpha"] = self._reshape_alpha(f"{lp}.norm2.alpha")
|
||||
plan[f"layers.{layer}.gating_in"] = self._stack_gating(lp, "linear_in")
|
||||
plan[f"layers.{layer}.gating_out"] = self._stack_gating(lp, "linear_out")
|
||||
return plan
|
||||
|
||||
@staticmethod
|
||||
def _reshape_alpha(key: str) -> _Builder:
|
||||
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
|
||||
t = src.get(key)
|
||||
return None if t is None else t.reshape(1, 1, -1)
|
||||
|
||||
return build
|
||||
|
||||
def _stack_gating(self, layer_prefix: str, which: str) -> _Builder:
|
||||
"""Stack the dep_q per-step gating weights into one [dep_q, ...] tensor.
|
||||
|
||||
Steps 8..15 fall back to step (n-8) when the checkpoint only ships 8
|
||||
(Moshi ``copy_missing_weights``).
|
||||
"""
|
||||
|
||||
def build(src: dict[str, torch.Tensor]) -> torch.Tensor | None:
|
||||
slabs: list[torch.Tensor] = []
|
||||
for step in range(self.dep_q):
|
||||
key = f"{layer_prefix}.gating.{step}.{which}.weight"
|
||||
if key not in src and step >= 8:
|
||||
key = f"{layer_prefix}.gating.{step - 8}.{which}.weight"
|
||||
t = src.get(key)
|
||||
if t is None:
|
||||
return None
|
||||
slabs.append(t)
|
||||
return torch.stack(slabs, dim=0)
|
||||
|
||||
return build
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PersonaPlex input embeddings (Moshi ``embed_codes``).
|
||||
|
||||
The temporal transformer (:class:`HeliumForCausalLM`) consumes precomputed
|
||||
``inputs_embeds`` rather than token ids. Those embeddings come from Moshi's
|
||||
``embed_codes``: a per-frame ``[B, 1 + n_q, S]`` token stack (row 0 = text,
|
||||
rows 1..n_q = audio codebooks) is mapped through one text embedding plus ``n_q``
|
||||
audio-codebook embeddings and summed into ``[B, S, hidden]``.
|
||||
|
||||
This module owns exactly those embedding tables (Moshi ``text_emb`` and
|
||||
``emb.0..n_q-1``), which ``modeling_helium`` deliberately omits. The talker
|
||||
(Task 2) uses it to build the temporal ``inputs_embeds`` for both the system /
|
||||
voice-prompt prefill and the per-frame decode step.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
|
||||
PersonaPlexConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_depformer import (
|
||||
_ScaledEmbedding,
|
||||
)
|
||||
|
||||
__all__ = ["PersonaPlexInputEmbeddings"]
|
||||
|
||||
|
||||
class PersonaPlexInputEmbeddings(nn.Module):
|
||||
"""Moshi ``embed_codes``: summed text + audio-codebook embeddings.
|
||||
|
||||
Args:
|
||||
config: the top-level PersonaPlex config (supplies the temporal hidden
|
||||
size, the audio cardinality / codebook count, and the text table size).
|
||||
"""
|
||||
|
||||
def __init__(self, config: PersonaPlexConfig) -> None:
|
||||
super().__init__()
|
||||
dim = config.temporal_config.hidden_size
|
||||
self.num_audio_codebooks = config.num_audio_codebooks # n_q (16)
|
||||
# Text table has one extra row for the special token (text_card + 1).
|
||||
self.text_emb = _ScaledEmbedding(config.text_embedding_rows, dim)
|
||||
# Each audio table is card + 1 rows; ``embed_codes`` uses all n_q codebooks.
|
||||
self.audio_emb = nn.ModuleList(
|
||||
[_ScaledEmbedding(config.audio_vocab_size + 1, dim) for _ in range(self.num_audio_codebooks)]
|
||||
)
|
||||
|
||||
def forward(self, sequence: torch.Tensor) -> torch.Tensor:
|
||||
"""Map a ``[B, 1 + n_q, S]`` token stack to ``[B, S, hidden]``.
|
||||
|
||||
Row 0 is the text stream; rows ``1..n_q`` are the audio codebooks (in
|
||||
Moshi's layout the audio rows carry both the agent and user streams).
|
||||
"""
|
||||
if sequence.dim() != 3 or sequence.shape[1] != self.num_audio_codebooks + 1:
|
||||
raise ValueError(f"sequence must be [B, {self.num_audio_codebooks + 1}, S]; got {tuple(sequence.shape)}")
|
||||
emb = self.text_emb(sequence[:, 0]) # [B, S, hidden]
|
||||
for cb in range(self.num_audio_codebooks):
|
||||
emb = emb + self.audio_emb[cb](sequence[:, cb + 1])
|
||||
return emb
|
||||
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]] | Mapping[str, torch.Tensor],
|
||||
) -> set[str]:
|
||||
"""Load Moshi ``text_emb`` and ``emb.0..n_q-1`` embedding tables."""
|
||||
src = dict(weights.items() if isinstance(weights, Mapping) else weights)
|
||||
params = dict(self.named_parameters())
|
||||
loaded: set[str] = set()
|
||||
|
||||
plan: dict[str, str] = {"text_emb.weight": "text_emb.weight"}
|
||||
for cb in range(self.num_audio_codebooks):
|
||||
plan[f"audio_emb.{cb}.weight"] = f"emb.{cb}.weight"
|
||||
|
||||
for tgt, key in plan.items():
|
||||
if key not in src:
|
||||
continue
|
||||
param = params[tgt]
|
||||
tensor = src[key]
|
||||
if tensor.shape != param.shape:
|
||||
raise ValueError(
|
||||
f"shape mismatch for {tgt}: checkpoint {tuple(tensor.shape)} vs param {tuple(param.shape)}"
|
||||
)
|
||||
param.data.copy_(tensor.to(param.dtype))
|
||||
loaded.add(tgt)
|
||||
return loaded
|
||||
@@ -0,0 +1,413 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Streaming Mimi codec for PersonaPlex (moshi-free).
|
||||
|
||||
Frame-clocked duplex needs a codec that encodes/decodes exactly one 80 ms frame
|
||||
per call with state carried across the conversation. The ``moshi`` package
|
||||
provided that; this module removes the dependency by combining:
|
||||
|
||||
- **transformers ``MimiModel``** (``kyutai/mimi`` — the same checkpoint family
|
||||
PersonaPlex ships) for the quantizer and every SEANet conv weight. Its
|
||||
streaming support only covers the encoder convs, so conv streaming is done
|
||||
here instead, uniformly.
|
||||
- **Our own streaming wrappers** mirroring the Moshi reference semantics (MIT),
|
||||
verified against recorded reference outputs:
|
||||
* ``Conv1d``: left-context carry of ``effective_kernel - stride`` samples,
|
||||
zero-initialized at stream start (``pad_mode="constant"``).
|
||||
* ``ConvTranspose1d``: overlap-add tail carry of ``kernel - stride`` output
|
||||
samples, with the double-counted bias subtracted on merge.
|
||||
* transformer: the encoder/decoder transformers use a 250-position sliding
|
||||
context over a ring KV with absolute-offset RoPE. Hugging Face's cache path
|
||||
diverges once the window engages (position 250), so the transformers are
|
||||
reimplemented here on the same ring-KV design as
|
||||
``personaplex_temporal.py`` and loaded directly from the PersonaPlex
|
||||
checkpoint's fused layout (LayerNorm + per-layer LayerScale + GELU FFN).
|
||||
|
||||
All per-stream state is ``[B, ...]`` with per-row reset (``reset_slot``), so the
|
||||
codec composes with elastic slot recycling in batched duplex serving.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_temporal import (
|
||||
_apply_rope,
|
||||
_RingKV,
|
||||
)
|
||||
|
||||
DEFAULT_HF_REPO = "kyutai/mimi"
|
||||
FRAME_SIZE = 1920
|
||||
CODEBOOKS = 8
|
||||
|
||||
|
||||
def _map_moshi_codec_weights(
|
||||
state_dict: dict[str, torch.Tensor],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Map bundled Moshi codec keys to ``transformers.MimiModel`` keys."""
|
||||
|
||||
mapped: dict[str, torch.Tensor] = {}
|
||||
transformer_prefixes = ("encoder_transformer.", "decoder_transformer.")
|
||||
for name, tensor in state_dict.items():
|
||||
if name.startswith(transformer_prefixes):
|
||||
continue
|
||||
|
||||
if name.startswith("encoder.model."):
|
||||
target = name.replace("encoder.model.", "encoder.layers.", 1)
|
||||
target = target.replace(".conv.conv.", ".conv.")
|
||||
elif name.startswith("decoder.model."):
|
||||
target = name.replace("decoder.model.", "decoder.layers.", 1)
|
||||
target = target.replace(".convtr.convtr.", ".conv.")
|
||||
target = target.replace(".conv.conv.", ".conv.")
|
||||
elif name.startswith("downsample.conv.conv.conv."):
|
||||
target = name.replace("downsample.conv.conv.conv.", "downsample.conv.", 1)
|
||||
elif name.startswith("upsample.convtr.convtr.convtr."):
|
||||
target = name.replace("upsample.convtr.convtr.convtr.", "upsample.conv.", 1)
|
||||
elif name.startswith("quantizer.rvq_first."):
|
||||
target = name.replace(
|
||||
"quantizer.rvq_first.",
|
||||
"quantizer.semantic_residual_vector_quantizer.",
|
||||
1,
|
||||
)
|
||||
elif name.startswith("quantizer.rvq_rest."):
|
||||
target = name.replace(
|
||||
"quantizer.rvq_rest.",
|
||||
"quantizer.acoustic_residual_vector_quantizer.",
|
||||
1,
|
||||
)
|
||||
else:
|
||||
raise KeyError(f"unrecognized PersonaPlex Mimi checkpoint key: {name}")
|
||||
|
||||
target = target.replace(".vq.layers.", ".layers.")
|
||||
target = target.replace("._codebook._initialized", ".codebook.initialized")
|
||||
target = target.replace("._codebook.cluster_usage", ".codebook.cluster_usage")
|
||||
target = target.replace("._codebook.embedding_sum", ".codebook.embed_sum")
|
||||
mapped[target] = tensor
|
||||
return mapped
|
||||
|
||||
|
||||
class _StreamConv1d:
|
||||
"""Moshi ``RawStreamingConv1d``: causal left-context carry per call.
|
||||
|
||||
``pad_mode`` controls the stream-start left padding: SEANet convs use zeros
|
||||
(``constant``); the down/upsample resamplers use ``replicate`` (the first
|
||||
real sample), marked per row so elastic slot recycling re-primes correctly.
|
||||
"""
|
||||
|
||||
def __init__(self, conv: nn.Conv1d, pad_mode: str = "constant") -> None:
|
||||
self.conv = conv
|
||||
self.kernel = (conv.kernel_size[0] - 1) * conv.dilation[0] + 1
|
||||
self.stride = conv.stride[0]
|
||||
self.pad_mode = pad_mode
|
||||
self.prev: torch.Tensor | None = None
|
||||
self._fresh: torch.Tensor | None = None
|
||||
|
||||
def reset(self, batch_size: int, device, dtype) -> None:
|
||||
pad = self.kernel - self.stride
|
||||
self.prev = torch.zeros(batch_size, self.conv.in_channels, pad, device=device, dtype=dtype)
|
||||
self._fresh = torch.ones(batch_size, dtype=torch.bool)
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
self.prev[b].zero_()
|
||||
self._fresh[b] = True
|
||||
|
||||
def __call__(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.pad_mode == "replicate" and bool(self._fresh.any()):
|
||||
pad = self.prev.shape[-1]
|
||||
edge = x[..., 0:1].expand(-1, -1, pad)
|
||||
fresh = self._fresh.to(x.device).view(-1, 1, 1)
|
||||
self.prev = torch.where(fresh, edge.to(self.prev.dtype), self.prev)
|
||||
self._fresh[:] = False
|
||||
x = torch.cat([self.prev, x], dim=-1)
|
||||
t = x.shape[-1]
|
||||
num_frames = max(0, (t - self.kernel) // self.stride + 1)
|
||||
self.prev = x[..., num_frames * self.stride :]
|
||||
if num_frames == 0:
|
||||
return x.new_zeros(x.shape[0], self.conv.out_channels, 0)
|
||||
return self.conv(x[..., : (num_frames - 1) * self.stride + self.kernel])
|
||||
|
||||
|
||||
class _StreamConvTr1d:
|
||||
"""Moshi ``RawStreamingConvTranspose1d``: overlap-add tail carry per call."""
|
||||
|
||||
def __init__(self, conv: nn.ConvTranspose1d) -> None:
|
||||
self.conv = conv
|
||||
self.kernel = conv.kernel_size[0]
|
||||
self.stride = conv.stride[0]
|
||||
self.partial: torch.Tensor | None = None
|
||||
|
||||
def reset(self, batch_size: int, device, dtype) -> None:
|
||||
self.partial = torch.zeros(
|
||||
batch_size, self.conv.out_channels, self.kernel - self.stride, device=device, dtype=dtype
|
||||
)
|
||||
self._fresh = torch.ones(batch_size, dtype=torch.bool)
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
self.partial[b].zero_()
|
||||
self._fresh[b] = True
|
||||
|
||||
def __call__(self, x: torch.Tensor) -> torch.Tensor:
|
||||
out = self.conv(x)
|
||||
length = out.shape[-1]
|
||||
tail = self.kernel - self.stride
|
||||
pt = self.partial.shape[-1]
|
||||
merge = self.partial.clone()
|
||||
if self.conv.bias is not None:
|
||||
# The carried tail already includes the bias; the fresh output adds
|
||||
# it again, so subtract one copy -- except on a row's very first
|
||||
# frame, where the carry is zeros by construction.
|
||||
merge = merge - self.conv.bias[:, None]
|
||||
merge[self._fresh] = 0.0
|
||||
self._fresh[:] = False
|
||||
out[..., :pt] += merge
|
||||
self.partial = out[..., length - tail :].clone()
|
||||
return out[..., : length - tail]
|
||||
|
||||
|
||||
class _MimiTransformerLayer(nn.Module):
|
||||
"""Moshi mimi transformer layer: LayerNorm, LayerScale, GELU FFN, no biases."""
|
||||
|
||||
def __init__(self, dim: int = 512, num_heads: int = 8, ffn: int = 2048) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
self.in_proj_weight = nn.Parameter(torch.empty(3 * dim, dim))
|
||||
self.out_proj_weight = nn.Parameter(torch.empty(dim, dim))
|
||||
self.linear1 = nn.Parameter(torch.empty(ffn, dim))
|
||||
self.linear2 = nn.Parameter(torch.empty(dim, ffn))
|
||||
self.norm1 = nn.LayerNorm(dim)
|
||||
self.norm2 = nn.LayerNorm(dim)
|
||||
self.scale1 = nn.Parameter(torch.empty(dim))
|
||||
self.scale2 = nn.Parameter(torch.empty(dim))
|
||||
|
||||
def forward(self, x: torch.Tensor, kv: _RingKV, offset: torch.Tensor, context: int) -> torch.Tensor:
|
||||
B, T, _ = x.shape
|
||||
h = self.norm1(x)
|
||||
qkv = F.linear(h, self.in_proj_weight)
|
||||
qkv = qkv.view(B, T, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
|
||||
q, k, v = qkv[0], qkv[1], qkv[2]
|
||||
q, k = _apply_rope(q, k, offset)
|
||||
keys, values, pos_k = kv.complete(k, v)
|
||||
pos_k = pos_k.view(pos_k.shape[0], 1, pos_k.shape[1])
|
||||
pos_q = offset + torch.arange(T, device=q.device, dtype=torch.long).view(1, -1, 1)
|
||||
delta = pos_q - pos_k
|
||||
attn_bias = (pos_k >= 0) & (delta >= 0) & (delta < context)
|
||||
attn = F.scaled_dot_product_attention(q, keys, values, attn_bias.unsqueeze(1), dropout_p=0.0)
|
||||
attn = attn.transpose(1, 2).reshape(B, T, self.dim)
|
||||
x = x + self.scale1 * F.linear(attn, self.out_proj_weight)
|
||||
h = self.norm2(x)
|
||||
h = F.linear(F.gelu(F.linear(h, self.linear1)), self.linear2)
|
||||
return x + self.scale2 * h
|
||||
|
||||
|
||||
class _MimiStreamingTransformer(nn.Module):
|
||||
"""The 8-layer mimi encoder/decoder transformer as a stateful stepper."""
|
||||
|
||||
def __init__(self, num_layers: int = 8, dim: int = 512, num_heads: int = 8, context: int = 250) -> None:
|
||||
super().__init__()
|
||||
self.context = context
|
||||
self.layers = nn.ModuleList([_MimiTransformerLayer(dim, num_heads) for _ in range(num_layers)])
|
||||
self._kv: list[_RingKV] | None = None
|
||||
self._offset: torch.Tensor | None = None
|
||||
|
||||
def streaming_init(self, batch_size: int) -> None:
|
||||
p = next(self.parameters())
|
||||
heads = self.layers[0].num_heads
|
||||
hd = self.layers[0].head_dim
|
||||
self._kv = [_RingKV(batch_size, heads, hd, self.context, p.device, p.dtype) for _ in self.layers]
|
||||
self._offset = torch.zeros(1, device=p.device, dtype=torch.long)
|
||||
|
||||
def reset_streaming(self) -> None:
|
||||
for kv in self._kv:
|
||||
kv.reset()
|
||||
self._offset.zero_()
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
for kv in self._kv:
|
||||
kv.reset_slot(b)
|
||||
|
||||
def step(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""``x`` is ``[B, T, dim]`` (T = positions this frame, typically 2)."""
|
||||
for layer, kv in zip(self.layers, self._kv):
|
||||
x = layer(x, kv, self._offset, self.context)
|
||||
self._offset.add_(x.shape[1])
|
||||
return x
|
||||
|
||||
def load_weights(self, state_dict: dict[str, torch.Tensor], prefix: str) -> int:
|
||||
loaded = 0
|
||||
with torch.no_grad():
|
||||
for i, layer in enumerate(self.layers):
|
||||
base = f"{prefix}.transformer.layers.{i}"
|
||||
pairs = [
|
||||
(layer.in_proj_weight, f"{base}.self_attn.in_proj_weight"),
|
||||
(layer.out_proj_weight, f"{base}.self_attn.out_proj.weight"),
|
||||
(layer.linear1, f"{base}.linear1.weight"),
|
||||
(layer.linear2, f"{base}.linear2.weight"),
|
||||
(layer.norm1.weight, f"{base}.norm1.weight"),
|
||||
(layer.norm1.bias, f"{base}.norm1.bias"),
|
||||
(layer.norm2.weight, f"{base}.norm2.weight"),
|
||||
(layer.norm2.bias, f"{base}.norm2.bias"),
|
||||
(layer.scale1, f"{base}.layer_scale_1.scale"),
|
||||
(layer.scale2, f"{base}.layer_scale_2.scale"),
|
||||
]
|
||||
for param, name in pairs:
|
||||
param.data.copy_(state_dict[name].reshape(param.shape).to(param.dtype))
|
||||
loaded += 1
|
||||
return loaded
|
||||
|
||||
|
||||
def _walk_seanet(layers) -> list[tuple[str, object]]:
|
||||
"""Wrap a Hugging Face Mimi SEANet layer list with streaming conv state."""
|
||||
stages: list[tuple[str, object]] = []
|
||||
for layer in layers:
|
||||
kind = type(layer).__name__
|
||||
if kind == "MimiConv1d":
|
||||
stages.append(("conv", _StreamConv1d(layer.conv)))
|
||||
elif kind == "MimiConvTranspose1d":
|
||||
stages.append(("convtr", _StreamConvTr1d(layer.conv)))
|
||||
elif kind == "ELU":
|
||||
stages.append(("act", layer))
|
||||
elif kind == "MimiResnetBlock":
|
||||
block = (
|
||||
layer.block[0],
|
||||
_StreamConv1d(layer.block[1].conv),
|
||||
layer.block[2],
|
||||
_StreamConv1d(layer.block[3].conv),
|
||||
)
|
||||
stages.append(("res", block))
|
||||
else: # pragma: no cover - unexpected layer type
|
||||
raise ValueError(f"unhandled Mimi SEANet layer: {kind}")
|
||||
return stages
|
||||
|
||||
|
||||
class PersonaPlexMimiCodec(nn.Module):
|
||||
"""Streaming Mimi encode/decode at one 80 ms frame per call (moshi-free)."""
|
||||
|
||||
def __init__(self, hf_repo: str = DEFAULT_HF_REPO, checkpoint: str | None = None, device: str = "cuda") -> None:
|
||||
super().__init__()
|
||||
from huggingface_hub import hf_hub_download
|
||||
from safetensors.torch import load_file
|
||||
from transformers import MimiConfig, MimiModel
|
||||
|
||||
self.device = torch.device(device)
|
||||
|
||||
# The PersonaPlex repo ships the reference mimi checkpoint in the moshi
|
||||
# fused layout. Build the matching Transformers graph locally, then load
|
||||
# every codec tensor from that bundled checkpoint. Only the two HF
|
||||
# transformer stacks remain absent because the streaming implementations
|
||||
# below replace them with the checkpoint's fused QKV layout.
|
||||
if checkpoint is None:
|
||||
checkpoint = hf_hub_download(
|
||||
"nvidia/personaplex-7b-v1",
|
||||
"tokenizer-e351c8d8-checkpoint125.safetensors",
|
||||
)
|
||||
sd = load_file(checkpoint, device=str(self.device))
|
||||
self.model = MimiModel(MimiConfig())
|
||||
codec_state = _map_moshi_codec_weights(sd)
|
||||
incompatible = self.model.load_state_dict(codec_state, strict=False)
|
||||
expected_missing = {
|
||||
name
|
||||
for name in self.model.state_dict()
|
||||
if name.startswith(("encoder_transformer.", "decoder_transformer."))
|
||||
}
|
||||
if set(incompatible.missing_keys) != expected_missing or incompatible.unexpected_keys:
|
||||
raise RuntimeError(
|
||||
"PersonaPlex Mimi checkpoint did not exactly cover the local codec graph: "
|
||||
f"missing={sorted(set(incompatible.missing_keys) - expected_missing)}, "
|
||||
f"unexpected={sorted(incompatible.unexpected_keys)}"
|
||||
)
|
||||
# The fused streaming transformers below replace these unloaded HF
|
||||
# stacks. Drop their randomly initialized weights before moving Mimi to
|
||||
# the target device.
|
||||
del self.model.encoder_transformer, self.model.decoder_transformer
|
||||
self.model = self.model.to(self.device).eval()
|
||||
self.dtype = next(self.model.parameters()).dtype
|
||||
self.encoder_transformer = _MimiStreamingTransformer().to(self.device, self.dtype)
|
||||
self.decoder_transformer = _MimiStreamingTransformer().to(self.device, self.dtype)
|
||||
n_enc = self.encoder_transformer.load_weights(sd, "encoder_transformer")
|
||||
n_dec = self.decoder_transformer.load_weights(sd, "decoder_transformer")
|
||||
assert n_enc == n_dec == 80, (n_enc, n_dec)
|
||||
del sd
|
||||
|
||||
m = self.model
|
||||
self._enc_stages = _walk_seanet(m.encoder.layers)
|
||||
self._downsample = _StreamConv1d(m.downsample.conv, pad_mode="replicate")
|
||||
self._upsample = _StreamConvTr1d(m.upsample.conv)
|
||||
self._dec_stages = _walk_seanet(m.decoder.layers)
|
||||
self._batch_size: int | None = None
|
||||
|
||||
# -- streaming state ------------------------------------------------------
|
||||
|
||||
def _conv_states(self):
|
||||
for _, stage in (*self._enc_stages, *self._dec_stages):
|
||||
if isinstance(stage, (_StreamConv1d, _StreamConvTr1d)):
|
||||
yield stage
|
||||
elif isinstance(stage, tuple):
|
||||
yield stage[1]
|
||||
yield stage[3]
|
||||
yield self._downsample
|
||||
yield self._upsample
|
||||
|
||||
def streaming_init(self, batch_size: int) -> None:
|
||||
self._batch_size = batch_size
|
||||
for s in self._conv_states():
|
||||
s.reset(batch_size, self.device, self.dtype)
|
||||
self.encoder_transformer.streaming_init(batch_size)
|
||||
self.decoder_transformer.streaming_init(batch_size)
|
||||
|
||||
def reset_streaming(self) -> None:
|
||||
assert self._batch_size is not None
|
||||
for state in self._conv_states():
|
||||
for b in range(self._batch_size):
|
||||
state.reset_slot(b)
|
||||
self.encoder_transformer.reset_streaming()
|
||||
self.decoder_transformer.reset_streaming()
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
for s in self._conv_states():
|
||||
s.reset_slot(b)
|
||||
self.encoder_transformer.reset_slot(b)
|
||||
self.decoder_transformer.reset_slot(b)
|
||||
|
||||
# -- per-frame codec -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _run_stages(x: torch.Tensor, stages) -> torch.Tensor:
|
||||
for kind, stage in stages:
|
||||
if kind == "res":
|
||||
act0, conv1, act2, conv3 = stage
|
||||
x = x + conv3(act2(conv1(act0(x))))
|
||||
else:
|
||||
x = stage(x)
|
||||
return x
|
||||
|
||||
@torch.no_grad()
|
||||
def encode_frame(self, pcm: torch.Tensor) -> torch.Tensor:
|
||||
"""``[B, frame_size]`` float PCM -> ``[B, 8]`` codes."""
|
||||
x = pcm.to(self.device, self.dtype).view(-1, 1, FRAME_SIZE)
|
||||
x = self._run_stages(x, self._enc_stages)
|
||||
x = self.encoder_transformer.step(x.transpose(1, 2)).transpose(1, 2)
|
||||
x = self._downsample(x)
|
||||
codes = self.model.quantizer.encode(x) # [Q, B, T]
|
||||
return codes[:CODEBOOKS, :, 0].transpose(0, 1).contiguous()
|
||||
|
||||
@torch.no_grad()
|
||||
def decode_frame(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
"""``[B, 8]`` codes -> ``[B, frame_size]`` float PCM."""
|
||||
emb = self.model.quantizer.decode(codes.to(self.device).view(-1, CODEBOOKS, 1))
|
||||
emb = self._upsample(emb)
|
||||
emb = self.decoder_transformer.step(emb.transpose(1, 2)).transpose(1, 2)
|
||||
x = self._run_stages(emb, self._dec_stages)
|
||||
return x[:, 0, :]
|
||||
|
||||
@torch.no_grad()
|
||||
def decode_frames(self, codes: torch.Tensor) -> torch.Tensor:
|
||||
"""``[B, 8, F]`` codes -> ``[B, F * frame_size]`` float PCM."""
|
||||
emb = self.model.quantizer.decode(codes.to(self.device))
|
||||
emb = self._upsample(emb)
|
||||
emb = self.decoder_transformer.step(emb.transpose(1, 2)).transpose(1, 2)
|
||||
x = self._run_stages(emb, self._dec_stages)
|
||||
return x[:, 0, :]
|
||||
@@ -0,0 +1,639 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PersonaPlex talker: the temporal transformer as a vLLM-native omni AR stage.
|
||||
|
||||
This is the stage-0 (``LLM_AR``) model the OmniGPUModelRunner drives. It composes
|
||||
three pieces, each verified in isolation against Moshi:
|
||||
|
||||
* the Helium temporal transformer (:class:`HeliumModel`) on vLLM paged attention,
|
||||
consuming per-frame ``inputs_embeds`` and producing the per-frame hidden state;
|
||||
* the input embeddings (:class:`PersonaPlexInputEmbeddings`, ``embed_codes``) that
|
||||
build those ``inputs_embeds`` from the delayed 17-row token stack;
|
||||
* the depformer (:class:`PersonaPlexDepformer`) that, conditioned on the temporal
|
||||
hidden state and the sampled text token, predicts the per-frame audio codes.
|
||||
|
||||
Per-frame protocol (OmniGPUModelRunner, gpu_model_runner.py):
|
||||
|
||||
1. ``compute_logits`` produces the text logits; the engine samples the text token.
|
||||
2. ``preprocess`` (per-request, with that request's ``additional_information``)
|
||||
carries Moshi's acoustic-delay cache and the precomputed user-audio code stream,
|
||||
builds the base ``inputs_embeds`` for the current frame, and exposes the
|
||||
previous frame's temporal hidden + text-step embedding via ``mtp_inputs``.
|
||||
3. ``talker_mtp`` (batched, stateless) runs the depformer to predict the agent
|
||||
codes and finishes the next frame's ``inputs_embeds``; the codes are stored
|
||||
under ``talker_mtp_output_key=("codes","audio")`` for the Mimi code2wav stage.
|
||||
|
||||
Phase 1 is turn-based: the user-audio rows (9..16) come from a precomputed Mimi
|
||||
encode of the input WAV (built in ``preprocess``); live duplex is Phase 2.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.personaplex.configuration_personaplex import (
|
||||
PersonaPlexConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.modeling_helium import HeliumModel
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_depformer import (
|
||||
PersonaPlexDepformer,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_embeddings import (
|
||||
PersonaPlexInputEmbeddings,
|
||||
)
|
||||
|
||||
__all__ = ["PersonaPlexTalkerForConditionalGeneration"]
|
||||
|
||||
|
||||
class PersonaPlexTalkerForConditionalGeneration(nn.Module):
|
||||
"""vLLM-native PersonaPlex talker (temporal transformer + depformer).
|
||||
|
||||
Plain ``nn.Module`` (not ``SupportsPP``): inheriting a vLLM Protocol puts
|
||||
``Protocol`` in the MRO and breaks vLLM's runtime ``isinstance`` check for
|
||||
``VllmModelForTextGeneration``, which would misclassify the talker as a
|
||||
non-generate model. Qwen3-TTS's talker is plain ``nn.Module`` for the same
|
||||
reason; PersonaPlex does not need pipeline parallelism.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
config: PersonaPlexConfig = vllm_config.model_config.hf_config # type: ignore[assignment]
|
||||
self.config = config
|
||||
self.temporal_config = config.temporal_config
|
||||
hidden = self.temporal_config.hidden_size
|
||||
self._dtype = getattr(vllm_config.model_config, "dtype", torch.bfloat16)
|
||||
|
||||
# Temporal backbone on vLLM paged attention (consumes inputs_embeds).
|
||||
self.model = HeliumModel(
|
||||
vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
config=self.temporal_config,
|
||||
)
|
||||
# Text head (Moshi text_linear -> lm_head), vocab = text_vocab_size (32000).
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.text_vocab_size,
|
||||
hidden,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.text_vocab_size)
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
# Verified custom components: embed_codes + depformer.
|
||||
self.input_embeddings = PersonaPlexInputEmbeddings(config)
|
||||
self.depformer = PersonaPlexDepformer(
|
||||
config.depformer_config,
|
||||
temporal_hidden_size=hidden,
|
||||
text_card=config.text_vocab_size,
|
||||
)
|
||||
|
||||
# Omni AR runner contract.
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = True
|
||||
self.has_postprocess = True # capture each frame's hidden for the next depformer step
|
||||
self.requires_full_prefix_cached_hidden_states = False
|
||||
# Keep the per-frame "last" hidden on GPU (avoids a CPU round-trip each step).
|
||||
self.gpu_resident_buffer_keys: set[tuple[str, str]] = {("hidden_states", "last")}
|
||||
self.mtp_hidden_size = hidden
|
||||
self.talker_mtp_output_key = ("codes", "audio")
|
||||
# dep_q audio codebooks per frame; only cb 0..num_active are vocoded.
|
||||
self.dep_q = config.depformer_config.dep_q
|
||||
self.num_active_codebooks = config.depformer_config.num_active_codebooks
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Core forward / logits
|
||||
# ------------------------------------------------------------------
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
"""Placeholder embedding for vLLM's ``VllmModel`` protocol.
|
||||
|
||||
The talker is driven by precomputed ``inputs_embeds`` (built in
|
||||
``preprocess`` via ``embed_codes``); ``input_ids`` are only in-vocab
|
||||
bookkeeping placeholders. Return a zero embedding of the temporal hidden
|
||||
size — the runner replaces it with the real per-frame ``inputs_embeds``.
|
||||
"""
|
||||
return torch.zeros(
|
||||
(input_ids.shape[0], self.mtp_hidden_size),
|
||||
device=input_ids.device,
|
||||
dtype=self._dtype,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**_: Any,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor | OmniOutput,
|
||||
sampling_metadata: Any = None,
|
||||
) -> torch.Tensor | None:
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if hidden_states is None:
|
||||
return None
|
||||
if hidden_states.dim() == 3:
|
||||
b, s, h = hidden_states.shape
|
||||
logits = self.logits_processor(self.lm_head, hidden_states.reshape(b * s, h))
|
||||
return None if logits is None else logits.reshape(b, s, -1)
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
hidden = model_outputs
|
||||
info_dicts = kwargs.get("model_intermediate_buffer") or kwargs.get("runtime_additional_information") or []
|
||||
audio_codes_list: list[torch.Tensor] = []
|
||||
for info in info_dicts:
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
ac = info.get("codes", {}).get("audio")
|
||||
if isinstance(ac, torch.Tensor):
|
||||
audio_codes_list.append(ac)
|
||||
if not audio_codes_list:
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
|
||||
audio_codes = torch.cat(audio_codes_list, dim=0)
|
||||
hidden = hidden[: int(audio_codes.shape[0])]
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={"codes": {"audio": audio_codes}})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Per-frame omni AR protocol: preprocess (stateful) + talker_mtp (batched)
|
||||
# ------------------------------------------------------------------
|
||||
def _initial_frame_embed(self, device: torch.device) -> torch.Tensor:
|
||||
"""``embed_codes`` of Moshi's initial token frame (text+audio SOS)."""
|
||||
text_init = self.config.text_vocab_size # text_initial_token_id
|
||||
audio_init = self.config.audio_vocab_size # card == initial audio token
|
||||
n_q = self.config.num_audio_codebooks
|
||||
stack = torch.empty((1, 1 + n_q, 1), dtype=torch.long, device=device)
|
||||
stack[:, 0] = text_init
|
||||
stack[:, 1:] = audio_init
|
||||
return self.input_embeddings(stack).reshape(1, -1) # [1, hidden]
|
||||
|
||||
def _build_frame_embed(
|
||||
self,
|
||||
text_token: torch.Tensor,
|
||||
last_agent: torch.Tensor | None,
|
||||
prev_agent: torch.Tensor | None,
|
||||
device: torch.device,
|
||||
user_d0: torch.Tensor | None = None,
|
||||
user_d1: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Full ``inputs_embeds`` for a decode frame (Moshi cache read at offset-1).
|
||||
|
||||
Acoustic-delay decomposition (delays ``[0, 0,1x7, 0,1x7]``): after a
|
||||
depformer step, Moshi writes every unforced agent codebook into the
|
||||
current cache target. The next temporal input therefore reads the whole
|
||||
previous effective agent frame. User writes are delayed by codebook, so
|
||||
user cb0 and cb1..7 still come from ``user_d0`` and ``user_d1``.
|
||||
Unfilled positions default to the initial token (card).
|
||||
"""
|
||||
n_q = self.config.num_audio_codebooks # 16 (8 agent + 8 user)
|
||||
n_user = n_q // 2 # 8
|
||||
audio_init = self.config.audio_vocab_size # card == initial audio token (Moshi cache default)
|
||||
# No-history / not-yet-generated delayed positions = the initial token (card),
|
||||
# NOT 0 (verified against Moshi's per-frame input stack).
|
||||
stack = torch.full((1, 1 + n_q, 1), audio_init, dtype=torch.long, device=device)
|
||||
stack[:, 0] = text_token.reshape(1)
|
||||
# The full effective agent frame was stored at the previous cache target
|
||||
# after applying any teacher forcing. The next cache read sees all eight
|
||||
# codebooks from that frame.
|
||||
if last_agent is not None and last_agent.numel() >= n_user:
|
||||
stack[0, 1 : 1 + n_user, 0] = last_agent.reshape(-1)[:n_user].to(device)
|
||||
# user cb0 (row 9) delay-0; user cb1..7 (rows 10..16) delay-1.
|
||||
user_base = 1 + n_user # row index 9
|
||||
if user_d0 is not None and user_d0.numel() >= 1:
|
||||
stack[0, user_base, 0] = user_d0.reshape(-1)[0].to(device)
|
||||
if user_d1 is not None and user_d1.numel() >= n_user:
|
||||
stack[0, user_base + 1 : user_base + n_user, 0] = user_d1.reshape(-1)[1:n_user].to(device)
|
||||
return self.input_embeddings(stack).reshape(1, -1) # [1, hidden]
|
||||
|
||||
def _build_prefill_embed(
|
||||
self,
|
||||
prefill_text: torch.Tensor,
|
||||
offset: int,
|
||||
span: int,
|
||||
device: torch.device,
|
||||
silence: torch.Tensor | None = None,
|
||||
user_sine: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Per-frame embeds for the persona system-prompt prefill.
|
||||
|
||||
Each frame forces the persona/silence text token (row 0). The agent rows
|
||||
use Mimi-encoded silence, while the user rows optionally use the encoded
|
||||
sine frame from PersonaPlex's native text-prompt prefill. Without an
|
||||
explicit user frame, both sides fall back to silence.
|
||||
"""
|
||||
n_q = self.config.num_audio_codebooks
|
||||
n_user = n_q // 2
|
||||
zero_text = 3 # Moshi LMGen.zero_text_code (silence/pad text)
|
||||
sil = (
|
||||
silence.reshape(-1).to(device) if isinstance(silence, torch.Tensor) and silence.numel() >= n_user else None
|
||||
)
|
||||
user = (
|
||||
user_sine.reshape(-1).to(device)
|
||||
if isinstance(user_sine, torch.Tensor) and user_sine.numel() >= n_user
|
||||
else sil
|
||||
)
|
||||
total = int(prefill_text.numel())
|
||||
rows = []
|
||||
for i in range(span):
|
||||
pos = offset + i
|
||||
text_tok = int(prefill_text[pos].item()) if pos < total else zero_text
|
||||
stack = torch.zeros((1, 1 + n_q, 1), dtype=torch.long, device=device)
|
||||
stack[:, 0] = text_tok
|
||||
if sil is not None:
|
||||
stack[0, 1 : 1 + n_user, 0] = sil[:n_user] # agent rows = encoded silence
|
||||
if user is not None:
|
||||
stack[0, 1 + n_user : 1 + 2 * n_user, 0] = user[:n_user]
|
||||
rows.append(self.input_embeddings(stack).reshape(1, -1))
|
||||
return torch.cat(rows, dim=0) # [span, hidden]
|
||||
|
||||
@staticmethod
|
||||
def _user_frame(info: dict[str, Any], frame_idx: int) -> torch.Tensor | None:
|
||||
"""Fetch the Mimi-encoded user codes for ``frame_idx`` (or None / out of range)."""
|
||||
uc = info.get("pplex_user_codes")
|
||||
if uc is None:
|
||||
return None
|
||||
if not isinstance(uc, torch.Tensor):
|
||||
uc = torch.as_tensor(uc, dtype=torch.long)
|
||||
if uc.ndim != 2 or frame_idx < 0 or frame_idx >= uc.shape[0]:
|
||||
return None
|
||||
return uc[frame_idx]
|
||||
|
||||
@staticmethod
|
||||
def _last_agent_codes(info: dict[str, Any]) -> torch.Tensor | None:
|
||||
codes = info.get("codes")
|
||||
if isinstance(codes, dict):
|
||||
ac = codes.get("audio")
|
||||
if isinstance(ac, torch.Tensor) and ac.numel() > 0:
|
||||
return (ac[-1] if ac.ndim == 2 else ac).reshape(-1)
|
||||
return None
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
input_embeds: torch.Tensor | None,
|
||||
**info_dict: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
|
||||
"""Build the current frame's ``inputs_embeds`` and the mtp inputs.
|
||||
|
||||
Prefill emits the initial-token frame embedding; each decode step builds the
|
||||
delayed base frame (everything but agent cb0, which ``talker_mtp`` adds).
|
||||
"""
|
||||
additional = info_dict.get("additional_information")
|
||||
if isinstance(additional, dict):
|
||||
merged = {k: v for k, v in info_dict.items() if k != "additional_information"}
|
||||
for k, v in additional.items():
|
||||
merged.setdefault(k, v)
|
||||
info_dict = merged
|
||||
meta = info_dict.get("meta", {}) if isinstance(info_dict.get("meta"), dict) else {}
|
||||
|
||||
device = input_ids.device
|
||||
span = int(input_ids.shape[0])
|
||||
is_prefill_raw = info_dict.get("_omni_is_prefill")
|
||||
if isinstance(is_prefill_raw, bool):
|
||||
is_prefill = is_prefill_raw
|
||||
else:
|
||||
is_prefill = span > 1
|
||||
|
||||
duplex = info_dict.get("duplex")
|
||||
if isinstance(duplex, dict) and duplex.get("data_plane") is True and is_prefill:
|
||||
prompt_len_raw = info_dict.get("duplex_prompt_len", span)
|
||||
try:
|
||||
prompt_len = int(prompt_len_raw)
|
||||
except (TypeError, ValueError):
|
||||
prompt_len = span
|
||||
prepared = self._duplex_stage0_runtime().prepare_append(
|
||||
duplex,
|
||||
prompt_len=prompt_len,
|
||||
request_id=(str(info_dict["request_id"]) if isinstance(info_dict.get("request_id"), str) else None),
|
||||
)
|
||||
offset_raw = info_dict.get("duplex_token_offset", 0)
|
||||
try:
|
||||
offset = max(0, int(offset_raw))
|
||||
except (TypeError, ValueError):
|
||||
offset = 0
|
||||
local_offset = offset - prepared.prompt_offset
|
||||
if local_offset < 0:
|
||||
raise ValueError(
|
||||
"PersonaPlex scheduled span precedes the current append: "
|
||||
f"offset={offset}, append_offset={prepared.prompt_offset}, "
|
||||
f"span={span}, prompt={prompt_len}"
|
||||
)
|
||||
req_embeds = prepared.inputs_embeds[local_offset : local_offset + span].to(
|
||||
device=device,
|
||||
dtype=self._dtype,
|
||||
)
|
||||
req_input_ids = prepared.input_ids[local_offset : local_offset + span].to(device=device)
|
||||
if req_embeds.shape[0] != span or req_input_ids.shape[0] != span:
|
||||
raise ValueError(
|
||||
"PersonaPlex duplex prompt slice is shorter than the scheduled span: "
|
||||
f"offset={offset}, span={span}, prompt={prompt_len}"
|
||||
)
|
||||
return req_input_ids, req_embeds, prepared.info_update
|
||||
|
||||
zero_hidden = torch.zeros((1, self.mtp_hidden_size), device=device, dtype=self._dtype)
|
||||
|
||||
prefill_text = info_dict.get("pplex_prefill_text")
|
||||
prefill_len = int(prefill_text.numel()) if isinstance(prefill_text, torch.Tensor) else 0
|
||||
|
||||
if is_prefill:
|
||||
offset = max(0, int(info_dict.get("_omni_num_computed_tokens", 0) or 0))
|
||||
if isinstance(prefill_text, torch.Tensor) and prefill_text.numel() > 0:
|
||||
# Persona system-prompt prefill (step_system_prompts analog).
|
||||
silence = info_dict.get("pplex_silence_codes")
|
||||
emb = self._build_prefill_embed(prefill_text, offset, span, device, silence)
|
||||
else:
|
||||
emb = self._initial_frame_embed(device)
|
||||
if span > 1:
|
||||
emb = emb.expand(span, -1).contiguous()
|
||||
ids_out = input_ids.clone()
|
||||
info_update = {
|
||||
"meta": {
|
||||
"pplex_frame": int(meta.get("pplex_frame", 0)) + span,
|
||||
"pplex_prefill_len": prefill_len,
|
||||
},
|
||||
"mtp_inputs": (zero_hidden, zero_hidden),
|
||||
}
|
||||
return ids_out, emb, info_update
|
||||
|
||||
# Decode: input_ids == the text token sampled last step (delay-0 text).
|
||||
text_token = input_ids.reshape(-1)[:1]
|
||||
# Agent acoustic delay: cb0 (delay 0) = gen[t-1] = the last stored codes;
|
||||
# cb1..7 (delay 1) = gen[t-2] = the codes carried from the previous frame.
|
||||
last_agent = self._last_agent_codes(info_dict)
|
||||
prev_agent = info_dict.get("pplex_prev_agent")
|
||||
if prev_agent is not None and not isinstance(prev_agent, torch.Tensor):
|
||||
prev_agent = torch.as_tensor(prev_agent, dtype=torch.long)
|
||||
# Real user stream (Phase-1 turn-based): user cb0 delay-0, cb1..7 delay-1.
|
||||
# Index into the user stream relative to the first decode frame (after the
|
||||
# persona prefill), so the user audio aligns with the agent's response.
|
||||
prefill_len = int(meta.get("pplex_prefill_len", prefill_len) or 0)
|
||||
# decode_frame = pplex_frame - prefill_len (the prefill frame already advanced
|
||||
# pplex_frame, so this is d+1 at decode step d). Verified vs Moshi's per-frame
|
||||
# stack: user cb0 (delay 0) = enc[decode_frame-1] = enc[d]; cb1..7 (delay 1) =
|
||||
# enc[decode_frame-2]. (The agent stream lags one more: agent cb0 = gen[d-1].)
|
||||
decode_frame = max(0, int(meta.get("pplex_frame", 0)) - prefill_len)
|
||||
user_d0 = self._user_frame(info_dict, decode_frame - 1)
|
||||
user_d1 = self._user_frame(info_dict, decode_frame - 2)
|
||||
base = self._build_frame_embed(text_token, last_agent, prev_agent, device, user_d0, user_d1)
|
||||
text_step = self.input_embeddings.text_emb(text_token.reshape(1, 1)).reshape(1, -1)
|
||||
# The previous frame's temporal hidden (written by postprocess into
|
||||
# hidden_states["last"]) conditions this frame's depformer. Falls back to
|
||||
# zeros only on the very first decode step (no prior hidden yet).
|
||||
hs = info_dict.get("hidden_states", {}) if isinstance(info_dict.get("hidden_states"), dict) else {}
|
||||
last_hidden = hs.get("last")
|
||||
if isinstance(last_hidden, torch.Tensor) and last_hidden.numel() > 0:
|
||||
last_hidden = last_hidden.reshape(1, -1).to(device=device, dtype=self._dtype)
|
||||
else:
|
||||
last_hidden = zero_hidden
|
||||
info_update = {
|
||||
"meta": {"pplex_frame": int(meta.get("pplex_frame", 0)) + 1},
|
||||
"mtp_inputs": (last_hidden, text_step),
|
||||
}
|
||||
# Carry this frame's gen[t-1] forward; next frame reads it as gen[t-2] (cb1..7).
|
||||
if last_agent is not None:
|
||||
info_update["pplex_prev_agent"] = last_agent.detach().to(torch.long).cpu()
|
||||
return input_ids, base, info_update
|
||||
|
||||
def _duplex_stage0_runtime(self):
|
||||
runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None)
|
||||
if runtime is not None:
|
||||
return runtime
|
||||
from vllm_omni.experimental.fullduplex.personaplex.stage0 import (
|
||||
PersonaPlexStage0DuplexRuntime,
|
||||
)
|
||||
|
||||
model_path = str(getattr(self.vllm_config.model_config, "model", ""))
|
||||
device = str(next(self.parameters()).device)
|
||||
runtime = PersonaPlexStage0DuplexRuntime(
|
||||
self,
|
||||
model_path=model_path,
|
||||
device=device,
|
||||
max_sessions=int(getattr(self.vllm_config.model_config, "duplex_max_sessions", 1)),
|
||||
)
|
||||
self._personaplex_duplex_stage0_runtime = runtime
|
||||
return runtime
|
||||
|
||||
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
|
||||
runtime = getattr(self, "_personaplex_duplex_stage0_runtime", None)
|
||||
if runtime is None:
|
||||
return
|
||||
for request_id in finished_req_ids:
|
||||
runtime.close_request(request_id)
|
||||
|
||||
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
|
||||
"""Capture this frame's last hidden for the next step's depformer (mtp)."""
|
||||
if hidden_states is None or hidden_states.numel() == 0:
|
||||
return {}
|
||||
return {"hidden_states": {"last": hidden_states[-1, :].detach()}}
|
||||
|
||||
def talker_mtp(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
input_embeds: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
text_step: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the depformer for the frame and finish its ``inputs_embeds``.
|
||||
|
||||
Returns ``(inputs_embeds, audio_codes[B, dep_q])``. ``audio_codes`` are
|
||||
stored under ``("codes","audio")``; ``inputs_embeds`` (base from preprocess
|
||||
+ the fresh agent cb0 embedding, the delay-0 acoustic token) feeds the next
|
||||
temporal forward.
|
||||
"""
|
||||
bsz = int(input_ids.shape[0])
|
||||
dtype = self._dtype
|
||||
text_token = input_ids.reshape(bsz).to(torch.long)
|
||||
hidden = last_talker_hidden.reshape(bsz, 1, -1).to(dtype)
|
||||
|
||||
codes = self.depformer(text_token, hidden) # [B, dep_q] == gen[t]
|
||||
|
||||
# The frame's inputs_embeds is built fully in preprocess (Moshi cache read at
|
||||
# offset-1, with agent cb0 = gen[t-1]); gen[t] feeds the NEXT frame, so pass
|
||||
# the embed through unchanged and just emit this frame's codes.
|
||||
inputs_embeds = input_embeds.reshape(bsz, -1).to(dtype)
|
||||
return inputs_embeds, codes.to(torch.long)
|
||||
|
||||
def post_sample_talker_mtp(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
req_ids: list[str],
|
||||
req_infos: list[dict[str, Any]],
|
||||
) -> torch.Tensor:
|
||||
"""Generate depformer codes for a one-token resumable duplex segment.
|
||||
|
||||
The normal runner invokes ``talker_mtp`` at the start of the next decode
|
||||
step. PersonaPlex's unified duplex request instead appends one audio
|
||||
frame and stops after one sampled text token, so there is no next decode
|
||||
step. Run the same depformer dependency immediately from the current
|
||||
sampled text token and temporal hidden state.
|
||||
"""
|
||||
bsz = int(input_ids.shape[0])
|
||||
if len(req_infos) != bsz:
|
||||
raise ValueError(
|
||||
f"PersonaPlex depformer request information does not match batch: {len(req_infos)} != {bsz}"
|
||||
)
|
||||
text_token = input_ids.reshape(bsz).to(torch.long)
|
||||
hidden = hidden_states.reshape(bsz, 1, -1).to(self._dtype)
|
||||
audio_tokens: list[torch.Tensor] = []
|
||||
audio_provided: list[torch.Tensor] = []
|
||||
for info in req_infos:
|
||||
tokens = info.get("pplex_depformer_audio_tokens")
|
||||
provided = info.get("pplex_depformer_audio_provided")
|
||||
if not isinstance(tokens, torch.Tensor) or not isinstance(provided, torch.Tensor):
|
||||
raise ValueError("PersonaPlex duplex depformer teacher-forcing state is missing")
|
||||
tokens = tokens.reshape(-1)
|
||||
provided = provided.reshape(-1)
|
||||
if tokens.shape != provided.shape:
|
||||
raise ValueError(
|
||||
"PersonaPlex depformer teacher-forcing token/mask shapes differ: "
|
||||
f"{tuple(tokens.shape)} != {tuple(provided.shape)}"
|
||||
)
|
||||
audio_tokens.append(tokens)
|
||||
audio_provided.append(provided)
|
||||
codes = self.depformer(
|
||||
text_token,
|
||||
hidden,
|
||||
audio_tokens=torch.stack(audio_tokens).to(
|
||||
device=hidden.device,
|
||||
dtype=torch.long,
|
||||
),
|
||||
audio_provided=torch.stack(audio_provided).to(
|
||||
device=hidden.device,
|
||||
dtype=torch.bool,
|
||||
),
|
||||
).to(torch.long)
|
||||
runtime = self._duplex_stage0_runtime()
|
||||
for row, request_id in enumerate(req_ids):
|
||||
runtime.record_sample(
|
||||
request_id=request_id,
|
||||
text_token=text_token[row],
|
||||
agent_codes=codes[row],
|
||||
)
|
||||
return codes
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Weight loading
|
||||
# ------------------------------------------------------------------
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Route the Moshi checkpoint into the three components.
|
||||
|
||||
* ``transformer.*`` / ``out_norm.alpha`` -> the Helium temporal backbone
|
||||
(same q/k-split + gate/up-split + alpha-squeeze map as HeliumForCausalLM).
|
||||
* ``text_linear.weight`` -> ``lm_head``.
|
||||
* ``emb.*`` / ``text_emb.weight`` -> input embeddings.
|
||||
* ``depformer*`` / ``linears.*`` -> depformer.
|
||||
"""
|
||||
weights = list(weights)
|
||||
params = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded: set[str] = set()
|
||||
|
||||
emb_w: list[tuple[str, torch.Tensor]] = []
|
||||
dep_w: list[tuple[str, torch.Tensor]] = []
|
||||
for name, w in weights:
|
||||
if name.startswith(("emb.", "text_emb.")):
|
||||
emb_w.append((name, w))
|
||||
elif name.startswith(("depformer.", "depformer_in.", "depformer_emb.", "depformer_text_emb", "linears.")):
|
||||
dep_w.append((name, w))
|
||||
elif name == "text_linear.weight":
|
||||
self._load_direct("lm_head.weight", w, params, loaded)
|
||||
else:
|
||||
loaded |= self._load_temporal(name, w, params)
|
||||
|
||||
# Delegate to the verified component loaders, prefixed into this module.
|
||||
for sub, sub_w in (("input_embeddings", emb_w), ("depformer", dep_w)):
|
||||
module = getattr(self, sub)
|
||||
for tgt in module.load_weights(sub_w):
|
||||
loaded.add(f"{sub}.{tgt}")
|
||||
return loaded
|
||||
|
||||
def _load_temporal(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
params: dict[str, nn.Parameter],
|
||||
) -> set[str]:
|
||||
out: set[str] = set()
|
||||
if name == "out_norm.alpha":
|
||||
self._load_direct("model.norm.weight", loaded_weight.squeeze(), params, out)
|
||||
return out
|
||||
prefix = "transformer.layers."
|
||||
if not name.startswith(prefix):
|
||||
return out
|
||||
rest = name.removeprefix(prefix)
|
||||
layer_index, _, suffix = rest.partition(".")
|
||||
if not layer_index.isdigit() or not suffix:
|
||||
return out
|
||||
base = f"model.layers.{layer_index}"
|
||||
if suffix == "self_attn.in_proj_weight":
|
||||
q, k, v = loaded_weight.chunk(3, dim=0)
|
||||
pname = f"{base}.self_attn.qkv_proj.weight"
|
||||
self._load_shard(pname, q, "q", params)
|
||||
self._load_shard(pname, k, "k", params)
|
||||
self._load_shard(pname, v, "v", params)
|
||||
out.add(pname)
|
||||
elif suffix == "self_attn.out_proj.weight":
|
||||
self._load_direct(f"{base}.self_attn.o_proj.weight", loaded_weight, params, out)
|
||||
elif suffix == "gating.linear_in.weight":
|
||||
gate, up = loaded_weight.chunk(2, dim=0)
|
||||
pname = f"{base}.mlp.gate_up_proj.weight"
|
||||
self._load_shard(pname, gate, 0, params)
|
||||
self._load_shard(pname, up, 1, params)
|
||||
out.add(pname)
|
||||
elif suffix == "gating.linear_out.weight":
|
||||
self._load_direct(f"{base}.mlp.down_proj.weight", loaded_weight, params, out)
|
||||
elif suffix == "norm1.alpha":
|
||||
self._load_direct(f"{base}.input_layernorm.weight", loaded_weight.squeeze(), params, out)
|
||||
elif suffix == "norm2.alpha":
|
||||
self._load_direct(f"{base}.post_attention_layernorm.weight", loaded_weight.squeeze(), params, out)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _load_direct(
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
params: dict[str, nn.Parameter],
|
||||
loaded: set[str],
|
||||
) -> None:
|
||||
if name not in params:
|
||||
return
|
||||
param = params[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded.add(name)
|
||||
|
||||
@staticmethod
|
||||
def _load_shard(
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str | int,
|
||||
params: dict[str, nn.Parameter],
|
||||
) -> None:
|
||||
if name not in params:
|
||||
return
|
||||
param = params[name]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
@@ -0,0 +1,257 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Streaming Helium temporal transformer for PersonaPlex (moshi-free, plain torch).
|
||||
|
||||
Frame-clocked duplex needs a stateful per-frame forward: one 80 ms step consumes
|
||||
one summed frame embedding and returns the temporal hidden + text logits, with a
|
||||
sliding-window KV carried across the conversation. vLLM's engine path serves the
|
||||
offline/staged flavor of this model (``modeling_helium.HeliumForCausalLM``); this
|
||||
module is the REAL-TIME flavor: no engine context, a fixed-capacity ring KV, and
|
||||
static shapes so the whole step is CUDA-graphable.
|
||||
|
||||
Semantics mirror the Moshi streaming transformer exactly (MIT; the reference for
|
||||
the ``nvidia/personaplex-7b-v1`` checkpoint), op for op, so greedy decoding is
|
||||
bit-comparable against recorded reference outputs:
|
||||
|
||||
- fp32 RMSNorm with ``alpha`` weights (norm1/norm2/out_norm)
|
||||
- fused in-proj attention, interleaved (non-neox) RoPE applied at the ABSOLUTE
|
||||
offset in fp32, relative-position causal mask over a ring KV with
|
||||
``context``-window truncation
|
||||
- SiLU gating MLP with fused ``linear_in`` (gate/up) and ``linear_out``
|
||||
- ``transformer_out`` is the POST-``out_norm`` hidden; ``text_logits`` is
|
||||
``text_linear`` of that hidden
|
||||
|
||||
Weights load directly from the moshi checkpoint's fused layout
|
||||
(``transformer.layers.{i}.self_attn.in_proj_weight`` etc.), keeping the exact
|
||||
computation order of the reference.
|
||||
|
||||
Per-slot elastic recycle (concurrent serving) uses the same per-row
|
||||
``start_offset`` masking verified for the moshi-hosted path: writes stay uniform
|
||||
because every slot ticks in lockstep; ``reset_slot`` only moves that row's
|
||||
valid-window start. The LM-specific "+1 sacrifice tick" bump is the CALLER's
|
||||
job (see the fullduplex PersonaPlex stage0), matching the split introduced when
|
||||
the +1 was found to be wrong for codec streams.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.model_executor.models.personaplex.personaplex_depformer import _rms_norm_f32
|
||||
|
||||
|
||||
def _apply_rope(q: torch.Tensor, k: torch.Tensor, offset: torch.Tensor, max_period: float = 10_000.0):
|
||||
"""Interleaved RoPE at absolute ``offset``, fp32 rotation (moshi ``apply_rope``)."""
|
||||
B, H, T, D = q.shape
|
||||
ds = torch.arange(D // 2, device=q.device, dtype=torch.float32)
|
||||
freqs = torch.exp(ds * (-math.log(max_period) * 2 / D))
|
||||
ts = offset.float() + torch.arange(T, device=q.device, dtype=torch.float32)
|
||||
ts = ts.view(1, -1, 1)
|
||||
|
||||
dims = q.shape[:-1]
|
||||
q = q.view(*dims, D // 2, 2)
|
||||
k = k.view(*dims, D // 2, 2)
|
||||
qr, qi = q[..., 0].float(), q[..., 1].float()
|
||||
kr, ki = k[..., 0].float(), k[..., 1].float()
|
||||
rotr = torch.cos(freqs * ts)
|
||||
roti = torch.sin(freqs * ts)
|
||||
qor = qr * rotr - qi * roti
|
||||
qoi = qr * roti + qi * rotr
|
||||
kor = kr * rotr - ki * roti
|
||||
koi = kr * roti + ki * rotr
|
||||
dtype = q.dtype
|
||||
qo = torch.stack([qor.to(dtype), qoi.to(dtype)], dim=-1)
|
||||
ko = torch.stack([kor.to(dtype), koi.to(dtype)], dim=-1)
|
||||
return qo.view(*dims, D), ko.view(*dims, D)
|
||||
|
||||
|
||||
class _RingKV:
|
||||
"""Fixed-capacity KV ring with per-row valid-window start (elastic recycle).
|
||||
|
||||
Uniform writes (all rows tick together) + a per-row mask via absolute
|
||||
positions; with ``start_offset == 0`` everywhere this is the plain streaming
|
||||
cache. All shapes static -> CUDA-graph safe.
|
||||
"""
|
||||
|
||||
def __init__(self, batch_size: int, num_heads: int, dim_per_head: int, capacity: int, device, dtype):
|
||||
self.capacity = capacity
|
||||
self.cache = torch.zeros((2, batch_size, num_heads, capacity, dim_per_head), device=device, dtype=dtype)
|
||||
self.end_offset = torch.zeros(1, device=device, dtype=torch.long)
|
||||
self.start_offset = torch.zeros(batch_size, device=device, dtype=torch.long)
|
||||
|
||||
def reset(self) -> None:
|
||||
self.end_offset.zero_()
|
||||
self.start_offset.zero_()
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
# Mask everything written so far for row b; the row's next write is its
|
||||
# first visible entry. (LM sacrifice-tick +1 is applied by the caller.)
|
||||
self.start_offset[b] = self.end_offset.clone()
|
||||
|
||||
def bump_slot_start(self, b: int) -> None:
|
||||
self.start_offset[b] += 1
|
||||
|
||||
def complete(self, k: torch.Tensor, v: torch.Tensor):
|
||||
B, H, T, D = k.shape
|
||||
indexes = torch.arange(T, device=self.end_offset.device, dtype=self.end_offset.dtype) + self.end_offset
|
||||
indexes = indexes % self.capacity
|
||||
self.cache[0].index_copy_(2, indexes, k)
|
||||
self.cache[1].index_copy_(2, indexes, v)
|
||||
self.end_offset.add_(T)
|
||||
|
||||
idx = torch.arange(self.capacity, device=self.end_offset.device, dtype=torch.long)
|
||||
invalid = idx >= self.end_offset
|
||||
end_index = self.end_offset % self.capacity
|
||||
delta = idx - end_index
|
||||
# `delta <= 0` (not `< 0`) is moshi's exact convention (transformer.py
|
||||
# RingKVCache.complete). It labels the just-past-newest slot as the future
|
||||
# write position, so once the ring has wrapped the single oldest in-window
|
||||
# cell is excluded and the effective window is capacity-1. This is inherited
|
||||
# verbatim from the reference and only shows after the window fills (Helium
|
||||
# ~3000 frames / 240 s); it costs one frame out of thousands. Do NOT change
|
||||
# this to `< 0`: it would diverge from moshi and break greedy bit-parity.
|
||||
positions = torch.where(delta <= 0, self.end_offset + delta, self.end_offset + delta - self.capacity)
|
||||
positions = torch.where(invalid, torch.full_like(positions, -1), positions)
|
||||
positions = positions.view(1, -1) # [1, capacity]
|
||||
below = positions < self.start_offset.view(-1, 1) # [B, capacity]
|
||||
positions = torch.where(below, torch.full_like(positions, -1), positions)
|
||||
return self.cache[0], self.cache[1], positions
|
||||
|
||||
|
||||
class _TemporalLayer(nn.Module):
|
||||
"""One Helium decoder layer in the checkpoint's fused layout."""
|
||||
|
||||
def __init__(self, dim: int, num_heads: int, hidden: int) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
self.in_proj_weight = nn.Parameter(torch.empty(3 * dim, dim))
|
||||
self.out_proj_weight = nn.Parameter(torch.empty(dim, dim))
|
||||
self.gating_in = nn.Parameter(torch.empty(2 * hidden, dim))
|
||||
self.gating_out = nn.Parameter(torch.empty(dim, hidden))
|
||||
self.norm1_alpha = nn.Parameter(torch.ones(1, 1, dim))
|
||||
self.norm2_alpha = nn.Parameter(torch.ones(1, 1, dim))
|
||||
|
||||
def forward(self, x: torch.Tensor, kv: _RingKV, offset: torch.Tensor, context: int) -> torch.Tensor:
|
||||
B, T, _ = x.shape
|
||||
h = _rms_norm_f32(x, self.norm1_alpha, 1e-8)
|
||||
qkv = F.linear(h, self.in_proj_weight)
|
||||
# Layout mirrors moshi's einops "b t (p h d) -> p b h t d" exactly, so the
|
||||
# downstream kernels see identical strides (bit-level replay agreement).
|
||||
qkv = qkv.view(B, T, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
|
||||
q, k, v = qkv[0], qkv[1], qkv[2]
|
||||
q, k = _apply_rope(q, k, offset)
|
||||
|
||||
keys, values, pos_k = kv.complete(k, v)
|
||||
pos_k = pos_k.view(pos_k.shape[0], 1, pos_k.shape[1]) # [B, 1, cap]
|
||||
pos_q = offset + torch.arange(T, device=q.device, dtype=torch.long).view(1, -1, 1)
|
||||
delta = pos_q - pos_k
|
||||
attn_bias = (pos_k >= 0) & (delta >= 0) & (delta < context)
|
||||
attn_bias = attn_bias.unsqueeze(1) # [B, 1, T, cap]
|
||||
attn = F.scaled_dot_product_attention(q, keys, values, attn_bias, dropout_p=0.0)
|
||||
attn = attn.transpose(1, 2).reshape(B, T, self.dim)
|
||||
x = x + F.linear(attn, self.out_proj_weight)
|
||||
|
||||
h = _rms_norm_f32(x, self.norm2_alpha, 1e-8)
|
||||
a, b = F.linear(h, self.gating_in).chunk(2, dim=-1)
|
||||
return x + F.linear(F.silu(a) * b, self.gating_out)
|
||||
|
||||
|
||||
class PersonaPlexTemporalStreaming(nn.Module):
|
||||
"""The 7B Helium temporal backbone as a stateful per-frame stepper.
|
||||
|
||||
``step(frame_embedding [B, 1, dim]) -> (transformer_out [B, 1, dim],
|
||||
text_logits [B, 1, 1, text_card])``, matching moshi's
|
||||
``LMModel.forward_embeddings`` contract so the existing native depformer and
|
||||
input embeddings compose unchanged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int = 4096,
|
||||
num_layers: int = 32,
|
||||
num_heads: int = 32,
|
||||
hidden: int = 11264,
|
||||
context: int = 3000,
|
||||
text_card: int = 32000,
|
||||
max_period: float = 10_000.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.context = context
|
||||
self.max_period = max_period
|
||||
self.layers = nn.ModuleList([_TemporalLayer(dim, num_heads, hidden) for _ in range(num_layers)])
|
||||
self.out_norm_alpha = nn.Parameter(torch.ones(1, 1, dim))
|
||||
self.text_linear = nn.Parameter(torch.empty(text_card, dim))
|
||||
self._kv: list[_RingKV] | None = None
|
||||
self._offset: torch.Tensor | None = None
|
||||
|
||||
# -- streaming state ----------------------------------------------------
|
||||
|
||||
def streaming_init(self, batch_size: int) -> None:
|
||||
p = next(self.parameters())
|
||||
heads = self.layers[0].num_heads
|
||||
head_dim = self.layers[0].head_dim
|
||||
# Capacity = context window (the mask truncates at `context` anyway).
|
||||
self._kv = [_RingKV(batch_size, heads, head_dim, self.context, p.device, p.dtype) for _ in self.layers]
|
||||
self._offset = torch.zeros(1, device=p.device, dtype=torch.long)
|
||||
|
||||
def reset_streaming(self) -> None:
|
||||
assert self._kv is not None, "call streaming_init first"
|
||||
for kv in self._kv:
|
||||
kv.reset()
|
||||
self._offset.zero_()
|
||||
|
||||
def reset_slot(self, b: int) -> None:
|
||||
assert self._kv is not None
|
||||
for kv in self._kv:
|
||||
kv.reset_slot(b)
|
||||
|
||||
def bump_slot_start(self, b: int) -> None:
|
||||
"""LM sacrifice-tick semantics: mask the one spurious post-recycle write."""
|
||||
assert self._kv is not None
|
||||
for kv in self._kv:
|
||||
kv.bump_slot_start(b)
|
||||
|
||||
# -- per-frame step -------------------------------------------------------
|
||||
|
||||
@torch.no_grad()
|
||||
def step(self, frame_embedding: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self._kv is not None, "call streaming_init first"
|
||||
x = frame_embedding
|
||||
for layer, kv in zip(self.layers, self._kv):
|
||||
x = layer(x, kv, self._offset, self.context)
|
||||
self._offset.add_(x.shape[1])
|
||||
out = _rms_norm_f32(x, self.out_norm_alpha, 1e-8)
|
||||
text_logits = F.linear(out, self.text_linear)
|
||||
return out, text_logits[:, None]
|
||||
|
||||
# -- weights --------------------------------------------------------------
|
||||
|
||||
def load_weights(self, state_dict: dict[str, torch.Tensor]) -> int:
|
||||
"""Load from the moshi checkpoint layout (fused, exact tensor reuse)."""
|
||||
loaded = 0
|
||||
with torch.no_grad():
|
||||
for i, layer in enumerate(self.layers):
|
||||
base = f"transformer.layers.{i}"
|
||||
pairs = [
|
||||
(layer.in_proj_weight, f"{base}.self_attn.in_proj_weight"),
|
||||
(layer.out_proj_weight, f"{base}.self_attn.out_proj.weight"),
|
||||
(layer.gating_in, f"{base}.gating.linear_in.weight"),
|
||||
(layer.gating_out, f"{base}.gating.linear_out.weight"),
|
||||
(layer.norm1_alpha, f"{base}.norm1.alpha"),
|
||||
(layer.norm2_alpha, f"{base}.norm2.alpha"),
|
||||
]
|
||||
for param, name in pairs:
|
||||
src = state_dict[name]
|
||||
param.data.copy_(src.reshape(param.shape).to(param.dtype))
|
||||
loaded += 1
|
||||
self.out_norm_alpha.data.copy_(state_dict["out_norm.alpha"].reshape(self.out_norm_alpha.shape))
|
||||
self.text_linear.data.copy_(state_dict["text_linear.weight"].to(self.text_linear.dtype))
|
||||
loaded += 2
|
||||
return loaded
|
||||
@@ -0,0 +1,75 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PersonaPlex pipeline: Talker (AR decode -> Mimi codebooks) -> Code2Wav (codebooks -> 24 kHz PCM).
|
||||
|
||||
PersonaPlex is a Moshi finetune (full-duplex speech-to-speech). For offline/batch
|
||||
runs it is served as a 2-stage vllm-omni audio->audio pipeline, reusing the
|
||||
Qwen3-TTS staged topology:
|
||||
|
||||
* Stage 0 (``personaplex``) is the AR talker: the Helium temporal transformer
|
||||
plus the depformer (both built by the lead). It emits the per-frame audio
|
||||
codebooks as latents.
|
||||
* Stage 1 (``code2wav``) wraps the external Mimi codec and decodes the active
|
||||
codebooks (``cb 0..7``) into PCM.
|
||||
|
||||
The inter-stage input processors named below live in
|
||||
``vllm_omni.model_executor.stage_input_processors.personaplex`` and are part of
|
||||
the talker<->code2wav seam; they are built by the lead alongside the talker.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
# Talker<->Code2Wav input processors (built by lead; coupled to talker output).
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.personaplex"
|
||||
|
||||
PERSONAPLEX_PIPELINE = PipelineConfig(
|
||||
model_type="personaplex",
|
||||
# Pipeline-level default; the code2wav stage overrides per-stage below.
|
||||
model_arch="PersonaPlexTalkerForConditionalGeneration",
|
||||
default_deploy_config_name="personaplex.yaml",
|
||||
duplex_runtime_extension=(
|
||||
"vllm_omni.experimental.fullduplex.personaplex.runtime_extension.PersonaPlexDuplexRuntimeExtension"
|
||||
),
|
||||
duplex_serving_adapter=(
|
||||
"vllm_omni.experimental.fullduplex.personaplex.serving_adapter.PersonaPlexServingRuntimeAdapter"
|
||||
),
|
||||
duplex_control_enabled=True,
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="personaplex",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
# built by lead: talker emits a 17-row Mimi token stack per frame;
|
||||
# these processors keep the PCM-bearing rows (cb 0..7) and flatten
|
||||
# them codebook-major for the Code2Wav stage.
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
custom_process_next_stage_input_func=f"{_PROC}.talker2code2wav_full_payload",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
model_arch="PersonaPlexCode2Wav",
|
||||
# In sync (non-async-chunk) mode the only pre-stage input proc is a
|
||||
# length-only placeholder; the bulk codec payload ships via the
|
||||
# worker connector from stage 0's full-payload producer.
|
||||
# built by lead.
|
||||
sync_process_input_func=f"{_PROC}.talker2code2wav_token_only",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -164,6 +164,18 @@ _OMNI_MODELS = {
|
||||
"qwen3_tts_code2wav",
|
||||
"Qwen3TTSCode2Wav",
|
||||
),
|
||||
## personaplex (Moshi finetune, full-duplex S2S; 2-stage audio->audio offline pipeline)
|
||||
# built by lead: personaplex_talker module + PersonaPlexTalkerForConditionalGeneration.
|
||||
"PersonaPlexTalkerForConditionalGeneration": (
|
||||
"personaplex",
|
||||
"personaplex_talker",
|
||||
"PersonaPlexTalkerForConditionalGeneration",
|
||||
),
|
||||
"PersonaPlexCode2Wav": (
|
||||
"personaplex",
|
||||
"personaplex_code2wav",
|
||||
"PersonaPlexCode2Wav",
|
||||
),
|
||||
## higgs-audio v2
|
||||
"HiggsAudioV2ForConditionalGeneration": (
|
||||
"higgs_audio_v2",
|
||||
|
||||
Reference in new issue
Block a user