[Model] Add Gepard-1.0 native-AR FSQ/NanoCodec TTS (offline inference) (#5666)

Signed-off-by: Mengjie Zhao <zmj0129@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-08-14 22:06:05 -07:00
1 parent d671e6b0db
commit 35cacf020c
17 files changed
+2727

No files matched your search

@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# NOTE: Do not import model classes in this file. Importing any
# submodule in this package triggers __init__.py execution, and
# both the model registry and pipeline registry import submodules
# directly — heavy imports here would be loaded as a side effect
# even though nothing depends on these re-exports.
@@ -0,0 +1,215 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Config for Gepard-1.0, a single-stage autoregressive TTS.
Text tokens -> one 32-code FSQ audio frame per step -> NeMo NanoCodec ->
waveform. The backbone is a vLLM-native ``Qwen3_5ForCausalLM``; the codebook
heads, binary stop head and voice-clone ref_compressor are Gepard additions.
Parses the model's ``gepard_config.json`` sidecar, which nests the LM
parameters under ``backbone_config`` and carries audio-head cardinalities,
special tokens, codec settings and the short-text repetition layout.
"""
from __future__ import annotations
from transformers import AutoConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
_BACKBONE_MODEL_TYPE = "qwen3_5_text"
# Stripped from the backbone config: the model uses standard 1D RoPE, and these
# would flip vLLM's uses_mrope() to True.
_MROPE_KEYS = ("mrope_section", "mrope_interleaved")
class GepardConfig(PretrainedConfig):
"""Configuration for the Gepard-1.0 native-AR TTS model.
Args mirror ``gepard_config.json``. Defaults match the trained
``nineninesix/gepard-1.0`` checkpoint so an instance built with no
arguments (e.g. dummy/profiling loads) is still self-consistent.
"""
model_type = "gepard"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
backbone_config: dict | None = None,
audio_heads: dict | None = None,
audio_embed_dim: int = 32,
special_tokens: dict | None = None,
text_repetition: dict | None = None,
codec: dict | None = None,
voice_cloning: dict | None = None,
stop_threshold: float = 0.5,
stop_loss_weight: float = 2.0,
stop_pos_weight: float = 25.0,
temperature: float = 0.3,
**kwargs,
):
# Everything is assigned before super().__init__() at the bottom:
# transformers 5.x runs validators there that call get_text_config().
self.backbone_config = self._normalize_backbone(backbone_config)
self.audio_head_levels = self._parse_audio_heads(audio_heads)
self.num_audio_heads = len(self.audio_head_levels)
self.audio_embed_dim = audio_embed_dim
# head0 is the token vLLM samples; STOP is a synthetic sentinel one past
# its valid range.
self.head0_vocab_size = self.audio_head_levels[0] if self.audio_head_levels else 8
self.stop_token = self.head0_vocab_size
st = special_tokens or {}
self.start_of_text = st.get("start_of_text", 248073)
self.end_of_text = st.get("end_of_text", 248074)
self.start_of_speech = st.get("start_of_speech", 248070)
self.end_of_speech = st.get("end_of_speech", 248071)
self.tts_pad = st.get("tts_pad", 248076)
# The speaker placeholder slots begin at tokeniser_length.
self.tokeniser_length = st.get("tokeniser_length", 248077)
self.speaker_token_base = self.tokeniser_length
# Prompt layout, read by prompt.py. These must match the training
# layout, so they come from the checkpoint rather than a literal.
tr = text_repetition or {}
self.text_repetition_enabled = tr.get("enabled", True)
self.text_repetition_target_tokens = tr.get("target_text_tokens", 16)
self.text_repetition_apply_below = tr.get("apply_below", 13)
self.text_repetition_max_repeats = tr.get("max_repeats", 8)
# NeMo NanoCodec, which runs outside vLLM.
cc = codec or {}
self.codec_id = cc.get("codec_id", "nvidia/nemo-nano-codec-22khz-1.89kbps-21.5fps")
self.codec_sample_rate = cc.get("sample_rate", 22050)
self.codec_frame_rate_hz = cc.get("frame_rate_hz", 21.5)
# Per-group FSQ levels; num_codec_groups * len(fsq_levels) == 32.
self.fsq_levels = cc.get("fsq_levels", [8, 7, 6, 6])
self.num_codec_groups = cc.get("num_layers", 8)
self.codec_do_unfold = cc.get("do_unfold", True)
self.stop_threshold = stop_threshold
self.stop_loss_weight = stop_loss_weight
self.stop_pos_weight = stop_pos_weight
# head0 goes through vLLM's SamplingParams; the other 31 heads use this
# in-model. There is no top_p: Gumbel-max has no nucleus step.
self.temperature = temperature
# Carried for the cloning follow-up; zero-shot uses null_prefix only.
vc = voice_cloning or {}
self.voice_cloning_enabled = vc.get("enabled", True)
comp = vc.get("compressor", {}) or {}
self.num_speaker_prefix = comp.get("num_queries", 8)
self.ref_compressor_num_blocks = comp.get("num_layers", 2)
self.ref_compressor_num_heads = comp.get("num_heads", 8)
self.ref_compressor_d_model = comp.get("d_model", 1024)
self.ref_compressor_ffn_mult = comp.get("ffn_hidden_size_multiplier", 4)
# Last on purpose — triggers transformers-5.x validators that call
# get_text_config(); every attribute they touch must already exist.
super().__init__(**kwargs)
@classmethod
def from_checkpoint(
cls,
model: str,
backbone_config: dict | None = None,
revision: str | None = None,
) -> GepardConfig:
"""Build the full config for a checkpoint that self-identifies as the
bare backbone: audio fields from the sidecar, backbone fields from the
loaded config.
``revision`` must be the one the weights came from — a revision that
moves the audio-head cardinalities or the special tokens moves the
prompt layout and the STOP sentinel with them.
"""
sidecar: dict = {}
try:
from vllm.transformers_utils.config import get_hf_file_to_dict
sidecar = dict(get_hf_file_to_dict("gepard_config.json", model, revision=revision) or {})
except (OSError, ValueError) as e:
# A missing or malformed sidecar is expected; the defaults match the
# trained checkpoint. Anything else must surface.
logger.warning(
"GepardConfig: could not read gepard_config.json from %s (%s: %s); "
"audio fields fall back to trained-checkpoint defaults.",
model,
type(e).__name__,
e,
)
if backbone_config:
bb = dict(backbone_config)
# The loaded config carries the hf_overrides-patched identity; the
# backbone must stay qwen3_5_text or get_text_config() recurses.
bb["model_type"] = _BACKBONE_MODEL_TYPE
bb.pop("architectures", None)
sidecar["backbone_config"] = bb
return cls(**sidecar)
@staticmethod
def _parse_audio_heads(audio_heads: dict | None) -> list[int]:
"""``{level_audio_0: 8, level_audio_1: 7, ...}`` -> ``[8, 7, ...]``.
Sorted by the numeric suffix, so channel order does not depend on dict
insertion order.
"""
if not audio_heads:
return [8, 7, 6, 6] * 8
return [audio_heads[k] for k in sorted(audio_heads, key=lambda s: int(str(s).rsplit("_", 1)[-1]))]
@classmethod
def _normalize_backbone(cls, backbone_config: dict | None) -> dict:
"""Return the backbone dict with vestigial mRoPE fields removed.
They are template leftovers that the trained checkpoint's own
``config.json`` does not carry. Left in, they flip vLLM's
``uses_mrope()`` to True and wire ``positions`` wrongly.
"""
bb = dict(backbone_config or {})
rope = bb.get("rope_parameters")
if isinstance(rope, dict) and any(k in rope for k in _MROPE_KEYS):
rope = {k: v for k, v in rope.items() if k not in _MROPE_KEYS}
bb["rope_parameters"] = rope
logger.info(
"GepardConfig: stripped vestigial mRoPE keys from backbone rope_parameters "
"(model uses standard 1D default RoPE)."
)
return bb
def get_text_config(self, **kwargs) -> PretrainedConfig:
"""Return the Qwen3.5 backbone config, which vLLM runs the backbone on."""
cached = getattr(self, "_text_config", None)
if cached is not None:
return cached
bb = dict(self.backbone_config)
# pop, not setdefault: for_model()'s first positional is named
# model_type, so leaving the key in **bb passes it twice.
model_type = bb.pop("model_type", _BACKBONE_MODEL_TYPE)
try:
text_config = AutoConfig.for_model(model_type, **bb)
except (KeyError, ValueError) as e:
# Only an unknown model_type is a legitimate fallback. A TypeError
# means the kwargs are wrong and must not degrade silently.
text_config = PretrainedConfig(**bb)
logger.warning(
"GepardConfig: AutoConfig.for_model(%s) failed (%s: %s); using a plain "
"PretrainedConfig for the backbone. Verify vLLM still resolves "
"the Qwen3.5 model.",
model_type,
type(e).__name__,
e,
)
self._text_config = text_config
return text_config
AutoConfig.register("gepard", GepardConfig)
__all__ = ["GepardConfig"]
@@ -0,0 +1,526 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Gepard-1.0 native-AR talker.
Single-stage AR TTS on a vLLM-native Qwen3.5 backbone. Each step samples one
32-channel FSQ frame; head0 is the vLLM-facing token and the other 31 are
side-channel. Generation ends on a learned binary stop head, not an EOS token.
The NeMo NanoCodec decodes committed frames to a waveform outside vLLM.
Zero-shot only, and ``enforce_eager``; CUDA graph is a perf follow-up.
"""
from __future__ import annotations
import dataclasses
import os
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.qwen3_5 import Qwen3_5ForCausalLM
from vllm.model_executor.models.utils import AutoWeightsLoader, maybe_prefix
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig
from vllm_omni.model_executor.models.gepard.nanocodec import NanoCodec
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
# Decode cadence: a short first chunk for early audio, then larger ones to
# amortize the codec call. Does not affect the samples produced.
FIRST_CHUNK_FRAMES = int(os.environ.get("VLLM_GEPARD_FIRST_CHUNK_FRAMES", "10"))
CHUNK_FRAMES = int(os.environ.get("VLLM_GEPARD_CHUNK_FRAMES", "86"))
@dataclasses.dataclass(frozen=True)
class _ReqInfo:
"""Scheduler state the runner threads into ``preprocess``.
The hook key strings are read only here, so a rename fails in one place.
"""
request_id: str
is_prefill: bool
prompt_len: int
num_computed_tokens: int
max_tokens: int | None
seed: int | None
@classmethod
def from_hook_kwargs(cls, kwargs: dict[str, Any], *, span: int) -> _ReqInfo:
# Defaults cover direct calls, which thread no scheduler state.
max_tokens = kwargs.get("_omni_max_tokens")
seed = kwargs.get("_omni_seed")
return cls(
request_id=str(kwargs.get("request_id", "default")),
is_prefill=bool(kwargs.get("_omni_is_prefill", span > 1)),
prompt_len=int(kwargs.get("_omni_prompt_len") or 0),
num_computed_tokens=int(kwargs.get("_omni_num_computed_tokens") or 0),
max_tokens=None if max_tokens is None else int(max_tokens),
seed=None if seed is None else int(seed),
)
@dataclasses.dataclass
class _GepardState:
"""Per-request generation state, keyed by request_id."""
request_id: str
# Embedding of the last frame's 32 codes; the next step's input.
curr_embed_for_next: torch.Tensor | None = None
# Frame history (T, 32) handed to the codec.
frames: list[torch.Tensor] = dataclasses.field(default_factory=list)
emitted_frames: int = 0
# None falls back to null_prefix; the cloning follow-up sets it.
speaker_prefix: torch.Tensor | None = None
# None means the request carried no seed and draws from the global RNG.
generator: torch.Generator | None = None
frame_count: int = 0
past_first_step: bool = False
is_stopping: bool = False
class GepardTalkerForConditionalGeneration(nn.Module):
"""Gepard native-AR TTS talker (Qwen3.5 backbone + 32 FSQ heads + stop)."""
have_multimodal_outputs = True
has_preprocess = True
has_postprocess = False
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.vllm_config = vllm_config
cfg = vllm_config.model_config.hf_config
if not isinstance(cfg, GepardConfig):
# hf_overrides patches the loaded config instance rather than
# re-classing it, so what arrives here is the backbone config.
cfg = GepardConfig.from_checkpoint(
vllm_config.model_config.model,
backbone_config=cfg.to_dict(),
revision=vllm_config.model_config.revision,
)
self.config = cfg
self._device = current_omni_platform.get_torch_device()
# Deterministic argmax sampling, for reproducible comparisons.
self._greedy = os.environ.get("VLLM_GEPARD_GREEDY", "0") == "1"
self.vocab_sizes: list[int] = list(cfg.audio_head_levels) # [8,7,6,6]*8
self.num_heads: int = cfg.num_audio_heads
self.head0_vocab: int = cfg.head0_vocab_size
self.stop_token: int = cfg.stop_token # sentinel, one past head0's range
self.total_vocab: int = sum(self.vocab_sizes)
self.stop_threshold: float = cfg.stop_threshold
self.temperature: float = cfg.temperature
hidden = self.config.get_text_config().hidden_size
self.fused_codebook_head = nn.Linear(hidden, self.total_vocab, bias=True)
self.stop_head = nn.Linear(hidden, 1, bias=True)
self.audio_embeddings = nn.ModuleList([nn.Embedding(v, cfg.audio_embed_dim) for v in self.vocab_sizes])
self.audio_embed_proj = nn.Sequential(
nn.Linear(self.num_heads * cfg.audio_embed_dim, hidden),
nn.GELU(),
nn.Linear(hidden, hidden),
nn.LayerNorm(hidden, elementwise_affine=False),
)
self.register_buffer("audio_embed_scale", torch.tensor(1.0))
# Learned default-voice fallback.
self.null_prefix = nn.Parameter(torch.zeros(cfg.num_speaker_prefix, hidden))
gather_idx, mask = self._build_gather_mask(self.vocab_sizes)
self.register_buffer("_cb_gather_idx", gather_idx, persistent=False)
self.register_buffer("_cb_mask", mask, persistent=False)
self.model = Qwen3_5ForCausalLM(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
# Cheap and NeMo-free to construct; load() does the heavy import.
self._codec = NanoCodec(codec_id=cfg.codec_id, sample_rate=cfg.codec_sample_rate)
self._max_model_len: int = int(vllm_config.model_config.max_model_len or 0)
self._active_states: dict[str, _GepardState] = {}
# These queues stay index-aligned across preprocess -> forward ->
# compute_logits: preprocess appends in input_batch.req_ids order.
# (req_id, span, samples_frame, is_last_token)
self._pending_requests: list[tuple[str, int, bool, bool]] = []
# (req_id, head0, do_stop); head0 < 0 marks a discarded placeholder row.
self._results_queue: list[tuple[str, int, bool]] = []
self._audio_queue: list[tuple[str, torch.Tensor | None]] = []
self._deferred_cleanup_ids: set[str] = set()
self._sample_rate: int = cfg.codec_sample_rate
@staticmethod
def _build_gather_mask(vocab_sizes: list[int]) -> tuple[torch.Tensor, torch.Tensor]:
"""Gather indices + -inf pad mask so one argmax samples all 31 heads."""
num_heads = len(vocab_sizes)
max_cb_vocab = max(vocab_sizes[1:]) # == 8 for Gepard; do NOT hardcode 7
gather_rows, mask_rows = [], []
cum = vocab_sizes[0]
for i in range(1, num_heads):
v = vocab_sizes[i]
gather_rows.append(list(range(cum, cum + v)) + [cum] * (max_cb_vocab - v))
mask_rows.append([0.0] * v + [float("-inf")] * (max_cb_vocab - v))
cum += v
return (
torch.tensor(gather_rows, dtype=torch.long),
torch.tensor(mask_rows, dtype=torch.float32),
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Fuse codebook_heads.{0..31} into fused_codebook_head; delegate the rest."""
cb_w: dict[str, torch.Tensor] = {}
rest: list[tuple[str, torch.Tensor]] = []
for name, tensor in weights:
if name.startswith(("aligner.", "supcon_head.")):
continue # training-only
if name.startswith("codebook_heads."):
cb_w[name] = tensor
continue
if name.startswith(("model.", "lm_head.")):
# We nest the backbone one level deeper than the checkpoint
# keys assume, so prepend the extra hop.
name = "model." + name
rest.append((name, tensor))
w_parts, b_parts = [], []
for i in range(self.num_heads):
if (wk := f"codebook_heads.{i}.weight") in cb_w:
w_parts.append(cb_w[wk])
if (bk := f"codebook_heads.{i}.bias") in cb_w:
b_parts.append(cb_w[bk])
if w_parts:
rest.append(("fused_codebook_head.weight", torch.cat(w_parts, dim=0)))
if b_parts:
rest.append(("fused_codebook_head.bias", torch.cat(b_parts, dim=0)))
# NanoCodec loads lazily on first decode, so load_format=dummy and
# NeMo-less environments never pay for it.
loader = AutoWeightsLoader(self, skip_prefixes=["mtp.", "ref_compressor."])
return loader.load_weights(iter(rest))
def _get_or_create_state(self, request_id: str) -> _GepardState:
st = self._active_states.get(request_id)
if st is None:
st = _GepardState(request_id=request_id)
self._active_states[request_id] = st
return st
def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:
# Fires BEFORE forward -> defer the actual free until after forward runs,
# else we'd drop state the in-flight forward still reads.
self._deferred_cleanup_ids.update(str(r) for r in finished_req_ids)
def _flush_deferred_cleanup(self) -> None:
# Cleanup only. Audio queued here is never delivered — the id has left
# the output batch by now — so the tail is emitted in forward instead,
# and anything still pending here is genuinely lost. Say so.
for req_id in self._deferred_cleanup_ids:
state = self._active_states.pop(req_id, None)
if state is not None and state.frame_count > state.emitted_frames:
logger.warning(
"Gepard request %s ended with %d undelivered frame(s): it finished "
"without the stop head firing and without reaching its token budget "
"(aborted or preempted). Its audio is truncated.",
req_id,
state.frame_count - state.emitted_frames,
)
self._deferred_cleanup_ids.clear()
def _is_last_output_token(self, info: _ReqInfo, *, span: int, samples_frame: bool) -> bool:
"""Whether the token sampled this step is the request's budget-final one.
That step is the last one on which the request is still in the output
batch, so it is also the last chance to ship its audio. Counted off the
engine's cursor rather than a local tally, so a re-prefill after
preemption cannot desync it.
"""
if not samples_frame:
# A partial prefill chunk's token is discarded, so it costs no budget.
return False
budget = info.max_tokens
if self._max_model_len:
len_cap = self._max_model_len - info.prompt_len
budget = len_cap if budget is None else min(budget, len_cap)
if budget is None or budget <= 0:
return False
produced = info.num_computed_tokens + span - info.prompt_len + 1
return produced >= budget
def _emit_audio(self, state: _GepardState, *, is_final: bool) -> None:
"""Decode this request's un-emitted frames and queue the new samples.
Queues deltas: each call contributes only samples not queued before.
"""
pending = state.frame_count - state.emitted_frames
if pending <= 0:
return
if not is_final:
threshold = FIRST_CHUNK_FRAMES if state.emitted_frames == 0 else CHUNK_FRAMES
if pending < threshold:
return
frames = torch.stack(state.frames) # (T, 32)
if not self._codec.is_loaded:
# The first decode pays the codec load; warmup is a perf follow-up.
self._codec.load(frames.device)
audio = self._codec.decode_stream(frames, start_idx=state.emitted_frames, is_final=is_final)
if audio is None:
return
self._audio_queue.append((state.request_id, audio))
state.emitted_frames = state.frame_count
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@staticmethod
def _gumbel_noise(ref: torch.Tensor, generators: list[torch.Generator | None] | None) -> torch.Tensor:
"""Gumbel noise shaped like ``ref``; row i uses ``generators[i]`` if set.
A seeded row draws its own noise of a fixed shape, so the noise it sees
does not depend on who shared its batch. Only the noise: the logits come
from a bf16 matmul that is not row-wise reproducible, so a batched
request can still sample differently from a solo one.
"""
if not generators or all(g is None for g in generators):
u = torch.rand_like(ref)
else:
u = torch.stack(
[
torch.rand(ref.shape[1:], generator=g, device=ref.device, dtype=ref.dtype)
if g is not None
else torch.rand(ref.shape[1:], device=ref.device, dtype=ref.dtype)
for g in generators
]
)
return -torch.log(-torch.log(u.clamp_min_(1e-20)))
def _sample_frame(
self,
hidden: torch.Tensor,
generators: list[torch.Generator | None] | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""(N, hidden) -> head0 (N,), heads_1_31 (N, 31), do_stop (N,).
``generators`` is row-aligned with ``hidden``. The 32 heads are sampled
here rather than by vLLM's sampler, so a request's seed only reaches
the audio through this.
"""
fused = self.fused_codebook_head(hidden)
gathered = fused[:, self._cb_gather_idx].float() + self._cb_mask # (N,31,8)
h0_logits = fused[:, : self.head0_vocab].float()
if self._greedy:
heads = gathered.argmax(dim=-1)
head0 = h0_logits.argmax(dim=-1)
else:
heads = (gathered / self.temperature + self._gumbel_noise(gathered, generators)).argmax(dim=-1)
head0 = (h0_logits / self.temperature + self._gumbel_noise(h0_logits, generators)).argmax(dim=-1)
p_stop = torch.sigmoid(self.stop_head(hidden)).squeeze(-1)
do_stop = p_stop > self.stop_threshold
return head0, heads, do_stop
def _audio_frame_embed(self, head0: torch.Tensor, heads_1_31: torch.Tensor) -> torch.Tensor:
"""Rebuild the next step's input embedding from the 32 sampled codes."""
codes = torch.cat([head0.clamp(max=self.head0_vocab - 1).unsqueeze(-1), heads_1_31], dim=-1)
parts = [self.audio_embeddings[i](codes[:, i]) for i in range(self.num_heads)]
return self.audio_embed_proj(torch.cat(parts, dim=-1)) * self.audio_embed_scale
def preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor | None = None, **req_infos: Any):
# Prefill injects the speaker prefix at the placeholder slots. The
# prompt ends on SOS, so forward samples the first frame there.
dev = input_ids.device
span = int(input_ids.shape[0])
info = _ReqInfo.from_hook_kwargs(req_infos, span=span)
req_id = info.request_id
is_prefill = info.is_prefill
prompt_len = info.prompt_len
computed = info.num_computed_tokens
# A prefill chunk that does not finish the prompt commits no frame:
# vLLM discards its sampled token.
samples_frame = (not is_prefill) or prompt_len <= 0 or computed + span >= prompt_len
if is_prefill:
state = self._get_or_create_state(req_id)
# An aborted id can be resubmitted, and the runner treats the two as
# distinct requests. Drop the pending free, or the cleanup at the
# end of this forward takes the new request's state with it.
resubmitted = req_id in self._deferred_cleanup_ids
self._deferred_cleanup_ids.discard(req_id)
if resubmitted:
# Stand in for the cleanup just cancelled.
undelivered = state.frame_count - state.emitted_frames
if undelivered > 0:
logger.warning(
"Gepard request %s ended with %d undelivered frame(s) and its id has been "
"resubmitted; the first request's audio is truncated.",
req_id,
undelivered,
)
elif state.frame_count:
# Not a resubmit, so this is a preemption recompute, and it
# cannot be recovered: a frame's 31 side-channel codes are not
# in the token stream, so the replayed prompt cannot rebuild the
# KV that produced the audio already delivered. Generation
# restarts from frame 0 and the caller gets two unrelated halves.
logger.warning(
"Gepard request %s was preempted after %d frame(s) and is being recomputed; "
"its audio will not continue from where it stopped. Raise the stage's "
"gpu_memory_utilization or lower max_num_seqs to keep requests from being "
"preempted.",
req_id,
state.frame_count,
)
state.curr_embed_for_next = None
state.frames.clear()
state.emitted_frames = 0
state.frame_count = 0
state.past_first_step = False
state.is_stopping = False
# SamplingParams.seed only reaches vLLM's sampler, so it never
# touches the audio unless it is threaded into a generator here.
state.generator = None
if info.seed is not None:
state.generator = torch.Generator(device=dev)
state.generator.manual_seed(info.seed)
embeds = self.model.embed_input_ids(input_ids)
prefix = state.speaker_prefix if state.speaker_prefix is not None else self.null_prefix
base = self.config.speaker_token_base
spk = (input_ids >= base) & (input_ids < base + self.config.num_speaker_prefix)
if bool(spk.any()):
embeds[spk] = prefix[input_ids[spk] - base].to(dtype=embeds.dtype)
else:
state = self._active_states.get(req_id)
curr = state.curr_embed_for_next if state is not None else None
if curr is not None:
embeds = curr.to(device=dev).reshape(1, -1)
else:
# Defensive only: a decode step always follows a forward that
# stashed curr_embed_for_next.
hidden = self.config.get_text_config().hidden_size
embeds = torch.zeros(1, hidden, device=dev, dtype=self.null_prefix.dtype)
is_last_token = self._is_last_output_token(info, span=span, samples_frame=samples_frame)
self._pending_requests.append((req_id, span, samples_frame, is_last_token))
return input_ids, embeds, {}
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: Any,
) -> torch.Tensor | OmniOutput:
# positions is plain 1D: Gepard uses default RoPE, not mRoPE. input_ids
# passes through only when no embeds were spliced (the profile run).
hidden = self.model(
input_ids=input_ids if inputs_embeds is None else None,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
# Empty _pending_requests means the engine-init profile run: skip.
if self._pending_requests:
# One sampling row per request: the last scheduled position of its
# span. Same row set compute_logits sees at logits_indices.
ends, generators, offset = [], [], 0
for req_id, span, _samples_frame, _is_last in self._pending_requests:
offset += span
ends.append(offset - 1)
state = self._active_states.get(req_id)
generators.append(state.generator if state is not None else None)
rows = hidden[torch.tensor(ends, device=hidden.device)]
head0, heads_1_31, want_stop = self._sample_frame(rows, generators)
frame_embeds = self._audio_frame_embed(head0, heads_1_31)
head0_l = head0.tolist()
stop_l = want_stop.tolist()
for i, (req_id, _span, samples_frame, is_last_token) in enumerate(self._pending_requests):
if not samples_frame:
# Partial prefill chunk: vLLM discards this row's token.
self._results_queue.append((req_id, -1, False))
continue
state = self._get_or_create_state(req_id)
if state.is_stopping:
# Async scheduling runs one more step after a stop token —
# only the max_tokens finish is guarded against it. This
# step's output is discarded, so a frame committed here
# would be stranded and reported as lost audio.
self._results_queue.append((req_id, int(head0_l[i]), True))
continue
# Stop may only fire past the first audio frame, so the
# prefill-sampled frame and the one after it always commit.
do_stop = bool(stop_l[i]) and state.past_first_step
if do_stop:
state.is_stopping = True
else:
state.frames.append(torch.cat([head0[i : i + 1], heads_1_31[i]]))
state.frame_count += 1
state.curr_embed_for_next = frame_embeds[i]
state.past_first_step = True
self._results_queue.append((req_id, int(head0_l[i]), do_stop))
# Both ways a request ends have to flush here: this is the last
# step on which it is still in the output batch, and a payload
# queued after it is dropped by routing.
self._emit_audio(state, is_final=do_stop or is_last_token)
self._pending_requests.clear()
self._flush_deferred_cleanup()
return hidden
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None):
# Vocab-wide logits carrying head0, or STOP for a stopped row. Requires
# the pipeline's sampling_constraints, without which vLLM never acts on
# the sentinel.
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if hidden_states is None:
return None
bsz = hidden_states.shape[0]
vocab = self.config.get_text_config().vocab_size
logits = torch.full((bsz, vocab), float("-inf"), device=hidden_states.device, dtype=hidden_states.dtype)
if not self._results_queue:
# Engine-init dummy sampler run: neutral one-hot logits so KV-cache
# profiling can complete.
logits[:, 0] = 0.0
return logits
# Row i belongs to _results_queue[i]. The pipeline pins temperature=0.0,
# so vLLM's sampler argmaxes the single finite entry.
for i, (_req_id, head0, do_stop) in enumerate(self._results_queue):
if i >= bsz:
break
if head0 < 0:
logits[i, 0] = 0.0 # partial-prefill placeholder; vLLM discards it
continue
logits[i, self.stop_token if do_stop else head0] = 0.0
self._results_queue.clear()
return logits
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
by_req: dict[str, torch.Tensor] = {}
for req_id, audio in self._audio_queue:
if audio is None:
continue
by_req[req_id] = (
torch.cat([by_req[req_id].reshape(-1), audio.reshape(-1)], dim=0) if req_id in by_req else audio
)
self._audio_queue.clear()
# The sparse marker is always carried, even on a step that decoded
# nothing: without it the runner ships the pooler payload instead, whose
# hidden states land under this stage's "audio" key.
ready_req_ids = list(by_req)
sr = torch.tensor(self._sample_rate, dtype=torch.int32)
mm: dict[str, Any] = {
"model_outputs": [by_req[r].reshape(-1) for r in ready_req_ids],
"sr": [sr for _ in ready_req_ids],
"meta": {"req_id": ready_req_ids, "sparse_audio": ["1"]},
}
# Deltas, not the whole history: callers concatenate across steps.
return OmniOutput(text_hidden_states=model_outputs, multimodal_outputs=mm)
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NeMo NanoCodec wrapper for Gepard — per-dimension FSQ decode.
Gepard's 32 FSQ heads emit per-dimension codes (8 groups x 4 dims), while stock
NeMo decode wants composed per-group mixed-radix indices. ``decode_from_codes``
dequantizes per-dim instead and hands the result to stock ``decode_audio`` — a
few lines on public NeMo API, no fork.
NeMo is a heavy optional dependency, imported lazily inside ``load``. Decode
only; ``encode_to_dequantized`` belongs to the voice-cloning follow-up.
"""
from __future__ import annotations
import os
import torch
import torch.nn as nn
from vllm.logger import init_logger
logger = init_logger(__name__)
# Left-context frames prepended to every streaming decode window. NanoCodec is
# convolutional, so decoding a bare chunk boundary yields a discontinuity; the
# extra frames are decoded for receptive-field context and trimmed afterwards.
# Mirrors the reference server's TTS_LOOKBACK_FRAMES.
LOOKBACK_FRAMES = int(os.environ.get("VLLM_GEPARD_LOOKBACK_FRAMES", "8"))
# End-of-clip shaping. The codec leaves the waveform on a non-zero sample and
# the hard step to silence is an audible click; a linear fade ramps it out.
# Both default to 0.0 (off), matching the reference server's defaults.
END_OF_SPEECH_FADE_MS = float(os.environ.get("VLLM_GEPARD_END_FADE_MS", "0.0"))
END_OF_SPEECH_SILENCE_MS = float(os.environ.get("VLLM_GEPARD_END_SILENCE_MS", "0.0"))
def apply_end_of_speech_tail(
audio: torch.Tensor,
sample_rate: int,
fade_ms: float = END_OF_SPEECH_FADE_MS,
silence_ms: float = END_OF_SPEECH_SILENCE_MS,
) -> torch.Tensor:
"""Fade the final ``fade_ms`` to zero, then append ``silence_ms`` of pad.
The fade is what removes the end-of-clip click; padding alone does not.
"""
if audio.numel() == 0 or (fade_ms <= 0 and silence_ms <= 0):
return audio
out = audio
if fade_ms > 0:
n = min(out.shape[0], int(sample_rate * fade_ms / 1000.0))
if n > 0:
# Copy: the caller's tensor is still referenced by the audio queue.
out = out.clone()
out[-n:] *= torch.linspace(1.0, 0.0, n, dtype=out.dtype, device=out.device)
if silence_ms > 0:
pad = torch.zeros(int(sample_rate * silence_ms / 1000.0), dtype=out.dtype, device=out.device)
out = torch.cat([out, pad], dim=0)
return out
def _make_unfolded_codec_cls():
"""Build the ``UnfoldedCodecModel`` subclass. Imports NeMo (optional dep).
Defined lazily so importing this module never requires NeMo. The subclass
is OUR code layered on stock ``AudioCodecModel`` — not a NeMo fork.
"""
try:
from nemo.collections.tts.models import AudioCodecModel # noqa: PLC0415
from omegaconf import open_dict # noqa: PLC0415
except ImportError as e: # pragma: no cover - env-dependent
raise ImportError(
"Gepard TTS needs NeMo for the NanoCodec decoder. Install it with "
"`pip install nemo_toolkit[tts]` (heavy; NVIDIA Open Model License, "
"commercial use OK), or gate the model behind that extra."
) from e
class UnfoldedCodecModel(AudioCodecModel):
"""AudioCodecModel + direct decode from per-dimension FSQ codes."""
def __init__(self, cfg, trainer=None):
# SLMDiscriminator pulls microsoft/wavlm-base-plus (~360 MB) and is
# training-only — strip it from the config before init.
with open_dict(cfg):
disc = cfg.get("discriminator", None)
if disc is not None and "discriminators" in disc:
disc.discriminators = [d for d in disc.discriminators if "SLM" not in d._target_]
super().__init__(cfg, trainer)
def decode_from_codes(self, codes: torch.Tensor, codes_len: torch.Tensor):
"""Decode per-dimension discrete FSQ codes to a waveform.
codes: (B, D, T) per-dim discrete values, D = num_groups * dims.
codes_len: (B,) valid frame count per batch element.
Returns (audio (B, T_audio), audio_len (B,)).
"""
num_levels = self.vector_quantizer.fsqs[0].num_levels.squeeze()
scale = (num_levels // 2).float().to(codes.device) # e.g. [4,3,3,3]
groups = codes.chunk(self.vector_quantizer.num_groups, dim=1)
dequantized = torch.cat(
[(g - scale[None, :, None]) / scale[None, :, None] for g in groups],
dim=1,
)
return self.decode_audio(inputs=dequantized, input_len=codes_len)
return UnfoldedCodecModel
class NanoCodec(nn.Module):
"""Thin wrapper owning the (optional) NeMo NanoCodec decoder.
Constructing this is cheap and NeMo-free; ``load`` does the heavy import +
weight load so ``load_format=dummy`` and NeMo-less environments don't pay
for it at model construction.
"""
def __init__(self, codec_id: str, sample_rate: int) -> None:
super().__init__()
self.codec_id = codec_id
self.sample_rate = sample_rate
self._codec: nn.Module | None = None
@property
def is_loaded(self) -> bool:
return self._codec is not None
def load(self, device: torch.device) -> None:
"""Import NeMo, build the UnfoldedCodecModel, move to device, eval()."""
cls = _make_unfolded_codec_cls()
codec = cls.from_pretrained(self.codec_id)
codec.to(device=device)
codec.eval()
self._codec = codec
logger.info("Gepard NanoCodec loaded from %s on %s", self.codec_id, device)
@torch.inference_mode()
def decode_from_codes(self, codes: torch.Tensor, codes_len: torch.Tensor) -> torch.Tensor:
"""(B, 32, T) per-dim codes -> waveform (B, T_audio). Returns audio only."""
if self._codec is None:
raise RuntimeError("NanoCodec.decode_from_codes called before load()")
audio, _audio_len = self._codec.decode_from_codes(codes, codes_len)
return audio
@torch.inference_mode()
def decode_stream(
self,
frames: torch.Tensor,
*,
start_idx: int,
lookback: int = LOOKBACK_FRAMES,
is_final: bool = False,
) -> torch.Tensor | None:
"""Decode the tail of a ``(T, 32)`` code history past ``start_idx``.
The window starts ``lookback`` frames earlier for receptive-field
context and those samples are trimmed after, so successive calls
concatenate to a single whole-history decode. None if nothing is new.
"""
count = int(frames.shape[0])
if count <= start_idx:
return None
decode_start = max(0, start_idx - lookback)
window = frames[decode_start:count] # (T_win, 32)
codes = window.t().unsqueeze(0).contiguous().long() # (1, 32, T_win)
codes_len = torch.full((1,), codes.shape[-1], dtype=torch.long, device=codes.device)
audio = self.decode_from_codes(codes, codes_len).reshape(-1)
# Derive samples_per_frame from this decode rather than assuming it.
samples_per_frame = audio.shape[0] // (count - decode_start)
audio = audio[(start_idx - decode_start) * samples_per_frame :]
if is_final:
audio = apply_end_of_speech_tail(audio, self.sample_rate)
return audio
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Gepard-1.0 pipeline topology.
Single-stage AR TTS. The Qwen3.5 backbone runs under paged attention, the 32
codebook heads and the binary stop head are sampled inside the talker, and the
NeMo NanoCodec decodes the accumulated frames outside vLLM.
``compute_logits`` scatters head0 into vLLM's vocab-wide logits; head0's valid
range is 0..7, so the synthetic STOP sentinel is 8. ``stop_token_ids=[8]``
couples it to vLLM's stop machinery, with ``detokenize=False`` because head0 is
an audio code. Same idiom as VoxCPM2/MOSS.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
# head0 valid range is 0..7 (its FSQ cardinality is 8); STOP is the sentinel
# one past that range. Mirrors the reference config.py STOP_TOKEN = 8.
_GEPARD_STOP_TOKEN_ID = 8
GEPARD_PIPELINE = PipelineConfig(
model_type="gepard",
default_deploy_config_name="gepard.yaml",
model_arch="GepardTalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="gepard",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="audio",
owns_tokenizer=True,
engine_output_type="audio",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [_GEPARD_STOP_TOKEN_ID],
},
),
),
)
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Gepard-1.0 prompt layout.
``preprocess`` only consumes this layout — it injects the speaker prefix at the
placeholder slots and samples the first audio frame from the SOS position. The
layout itself is built here, by whichever entry point is assembling a request.
"""
from __future__ import annotations
import math
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from vllm_omni.model_executor.models.gepard.configuration_gepard import GepardConfig
def build_gepard_prompt_ids(text_token_ids: list[int], *, config: GepardConfig) -> list[int]:
"""Assemble ``[speaker_slots] + [SOT, *text, EOT]*(R-1) + [SOT, *text, EOT, SOS]``.
Only the final copy carries SOS, the learned "audio starts now" gate, so
the earlier copies are read as context and never voiced. Short texts repeat
because a couple of text tokens carry too little mass against the speaker
prefix. This must match the training layout or WER collapses, which is why
the thresholds come from the checkpoint's ``text_repetition`` block rather
than from literals here.
Raises ``ValueError`` on empty text: the layout would still be structurally
valid and the model would voice it as arbitrary audio, so an empty request
has to fail loudly. This is the only entry point in the offline path — there
is no adapter above it to check first.
"""
n = len(text_token_ids)
if n == 0:
raise ValueError("text_token_ids is empty; Gepard needs at least one text token to voice")
if (
not config.text_repetition_enabled
or n >= config.text_repetition_apply_below
or n >= config.text_repetition_target_tokens
):
repeats = 1
else:
repeats = max(
1,
min(
math.ceil(config.text_repetition_target_tokens / n),
config.text_repetition_max_repeats,
),
)
block = [config.start_of_text, *text_token_ids, config.end_of_text]
text_region = block * (repeats - 1) + [*block, config.start_of_speech]
speaker_slots = [config.speaker_token_base + i for i in range(config.num_speaker_prefix)]
return speaker_slots + text_region
@@ -295,6 +295,12 @@ _OMNI_MODELS = {
"fish_speech_dac_decoder",
"FishSpeechDACDecoder",
),
## Gepard-1.0 (nineninesix) — single-stage native-AR FSQ/NanoCodec TTS
"GepardTalkerForConditionalGeneration": (
"gepard",
"gepard_talker",
"GepardTalkerForConditionalGeneration",
),
## VoxCPM2
"VoxCPM2TalkerForConditionalGeneration": (
"voxcpm2",