[TTS][New Model] support bosonai/higgs-audio-v3-tts-4b (#4169)

Signed-off-by: Yuekai Zhang <zhangyuekai@foxmail.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yuekai ZhangandYueqian Lin authored and GitHub committed 2026-06-06 15:59:12 +00:00
1 parent 963ba1ab7a
commit 82e74ce972
22 files changed
+4295

No files matched your search

@@ -0,0 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vllm-omni integration for boson-ai's higgs-audio v3 (two-stage TTS)."""
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
__all__ = ["HiggsAudioV3Config"]
@@ -0,0 +1,182 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration class for higgs-audio v3 (HiggsMultimodalQwen3) in vllm-omni.
``HiggsAudioV3Config.from_pretrained(model_path)`` returns a config with
``tts_token_id``, ``text_token_id``, ``audio_continuation_id``, and
``eos_token_id`` already resolved from the checkpoint tokenizer. If the
tokenizer is unavailable or missing required specials, the load raises.
"""
from __future__ import annotations
import logging
import os
from typing import Any
import transformers
from transformers import AutoConfig, PretrainedConfig
__all__ = ["HiggsAudioV3Config"]
logger = logging.getLogger(__name__)
_QWEN3_ROPE_THETA = 1_000_000
_REQUIRED_SPECIALS = ("<|tts|>", "<|text|>", "<|audio|>")
def _build_text_config(raw: Any) -> PretrainedConfig:
if isinstance(raw, PretrainedConfig):
return raw
cfg = dict(raw or {})
model_type = cfg.get("model_type", "qwen3")
if model_type == "qwen3" and cfg.get("rope_theta") is None:
cfg["rope_theta"] = _QWEN3_ROPE_THETA
try:
cfg_cls = transformers.CONFIG_MAPPING[model_type]
except KeyError as exc:
raise ValueError(f"Unknown text backbone model_type {model_type!r}") from exc
return cfg_cls(**cfg)
def _resolve_model_dir(pretrained_model_name_or_path: str) -> str | None:
"""Resolve a model name/path to a local directory containing tokenizer files."""
if os.path.isdir(pretrained_model_name_or_path):
return pretrained_model_name_or_path
try:
from huggingface_hub import try_to_load_from_cache
cached = try_to_load_from_cache(repo_id=pretrained_model_name_or_path, filename="tokenizer_config.json")
if isinstance(cached, str) and os.path.isfile(cached):
return os.path.dirname(cached)
except Exception:
pass
try:
from huggingface_hub.constants import HF_HUB_CACHE
safe = pretrained_model_name_or_path.replace("/", "--")
snapshots = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
if os.path.isdir(snapshots):
for rev in os.listdir(snapshots):
candidate = os.path.join(snapshots, rev)
if os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
return candidate
except Exception:
pass
return None
class HiggsAudioV3Config(PretrainedConfig):
"""Typed config for higgs-audio v3 (HiggsMultimodalQwen3).
``from_pretrained()`` automatically resolves ``<|tts|>``, ``<|text|>``,
``<|audio|>`` and ``eos_token_id`` from the checkpoint tokenizer.
"""
model_type: str = "higgs_multimodal_qwen3"
is_composition = True
def __init__(
self,
audio_encoder_config: dict[str, Any] | None = None,
text_config: dict[str, Any] | PretrainedConfig | None = None,
audio_token_id: int = -100,
mel_per_sample: int = 8,
num_codebooks: int = 8,
codebook_size: int = 1026,
audio_stream_bos_id: int = 1024,
audio_stream_eos_id: int = 1025,
sample_rate: int = 24000,
frame_rate: int = 25,
tts_token_id: int | None = None,
text_token_id: int | None = None,
audio_continuation_id: int | None = None,
**kwargs: Any,
) -> None:
self.audio_token_id = audio_token_id
self.mel_per_sample = mel_per_sample
if audio_encoder_config is None:
audio_encoder_config = {
"encoder_type": "discrete",
"num_codebooks": num_codebooks,
"vocab_size": codebook_size,
"tie_word_embeddings": True,
}
self.audio_encoder_config = audio_encoder_config
self.num_codebooks = int(audio_encoder_config.get("num_codebooks", num_codebooks))
if self.num_codebooks <= 0:
raise ValueError(f"num_codebooks must be > 0, got {self.num_codebooks}")
self.codebook_size = int(audio_encoder_config.get("vocab_size", codebook_size))
self.tie_modality_embeddings = bool(audio_encoder_config.get("tie_word_embeddings", True))
self.audio_stream_bos_id = audio_stream_bos_id
self.audio_stream_eos_id = audio_stream_eos_id
self.sample_rate = sample_rate
self.frame_rate = frame_rate
self.text_config = _build_text_config(text_config)
self.audio_hidden_size = int(audio_encoder_config.get("out_dim", self.text_config.hidden_size))
self.tts_token_id = tts_token_id
self.text_token_id = text_token_id
self.audio_continuation_id = audio_continuation_id
super().__init__(**kwargs)
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any) -> HiggsAudioV3Config:
"""Load config and resolve special token IDs from the checkpoint tokenizer.
Passes the original ``pretrained_model_name_or_path`` (local dir or
HF repo ID) directly to ``AutoTokenizer.from_pretrained()`` so it can
handle cache hits, downloads, and local paths uniformly. Raises if the
tokenizer is missing required specials.
"""
config = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
config.resolve_special_tokens(pretrained_model_name_or_path)
return config
def get_text_config(self, decoder: bool = False) -> PretrainedConfig:
del decoder
return self.text_config
@property
def num_real_codes(self) -> int:
return self.audio_stream_bos_id
@property
def hidden_size(self) -> int:
return self.text_config.hidden_size
def resolve_special_tokens(self, model_path: str) -> None:
"""Resolve <|tts|>, <|text|>, <|audio|> and eos from the HF tokenizer.
Raises ``ValueError`` if any of the 3 required specials is missing
from the tokenizer's added vocabulary.
"""
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
vocab = dict(tokenizer.get_added_vocab())
missing = [t for t in _REQUIRED_SPECIALS if t not in vocab]
if missing:
raise ValueError(
f"Tokenizer at {model_path} is missing required Higgs TTS v3 "
f"special tokens: {missing}. Available added tokens: "
f"{sorted(vocab.keys())[:20]}"
)
self.tts_token_id = vocab["<|tts|>"]
self.text_token_id = vocab["<|text|>"]
self.audio_continuation_id = vocab["<|audio|>"]
if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None:
self.eos_token_id = int(tokenizer.eos_token_id)
AutoConfig.register("higgs_multimodal_qwen3", HiggsAudioV3Config)
@@ -0,0 +1,524 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stage 1 (codec decoder) for higgs-audio v3.
Reuses higgs-audio-v2's RVQ + DAC codec decoder but loads weights from the
v3 checkpoint's bundled codec (``tied.embedding.modality_embeddings.0.model.*``
prefix) rather than from a standalone tokenizer repo.
"""
from __future__ import annotations
import os
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_decoder import (
HiggsAudioRVQ,
build_boson_dac_decoder,
build_higgs_audio_acoustic_decoder,
load_higgs_audio_codec,
)
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
__all__ = [
"HiggsAudioV3Code2Wav",
"HiggsAudioV3Code2WavForConditionalGeneration",
]
logger = init_logger(__name__)
# Prefix under which codec weights are stored in the v3 checkpoint.
_CODEC_PREFIX = "tied.embedding.modality_embeddings.0.model."
class HiggsAudioV3Code2Wav(nn.Module):
"""Stage-1 codec decoder for higgs-audio v3.
The codec architecture is identical to v2 (HiggsAudioRVQ + fc2 +
BosonDacDecoder); only the weight loading path differs. V3 bundles
codec weights inside the main checkpoint under a known prefix, while
v2 uses a standalone tokenizer repo.
For backward compatibility and simplicity, we delegate to the v2
``load_higgs_audio_codec`` function when the audio tokenizer repo
is available. When not (or as primary path), we extract codec weights
from the v3 safetensors via the engine's weight iterator.
"""
input_modalities = "audio"
def __init__(
self,
config: HiggsAudioV3Config | None = None,
*,
vllm_config: VllmConfig | None = None,
prefix: str = "",
):
super().__init__()
if vllm_config is not None:
hf_config = vllm_config.model_config.hf_config
if isinstance(hf_config, HiggsAudioV3Config):
self.config = hf_config
else:
self.config = HiggsAudioV3Config(**hf_config.to_dict())
self._model_path: str | None = vllm_config.model_config.model
self.vllm_config: VllmConfig | None = vllm_config
else:
if config is None:
raise TypeError("HiggsAudioV3Code2Wav: provide either `config` or `vllm_config`.")
self.config = config
self._model_path = None
self.vllm_config = None
self.sample_rate: int = int(self.config.sample_rate)
self.num_codebooks: int = int(self.config.num_codebooks)
self.num_real_codes: int = int(self.config.num_real_codes)
self.hop_length: int = 960
# Engine-runner hooks
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
# Codec modules (populated by load_weights)
self.quantizer: HiggsAudioRVQ | None = None
self.fc2: nn.Linear | None = None
self.acoustic_decoder: nn.Module | None = None
self._loaded: bool = False
# Do NOT eagerly load from standalone tokenizer here.
# load_weights() will try bundled V3 codec first, then fall back.
def _load_from_tokenizer_repo(self) -> None:
"""Load codec from the standalone higgs-audio-v2-tokenizer repo."""
tokenizer_id = "bosonai/higgs-audio-v2-tokenizer"
# Try to resolve from HF cache
from huggingface_hub import try_to_load_from_cache
cached = try_to_load_from_cache(repo_id=tokenizer_id, filename="config.json")
if isinstance(cached, str) and os.path.isfile(cached):
tokenizer_dir = os.path.dirname(cached)
else:
from huggingface_hub.constants import HF_HUB_CACHE
safe = tokenizer_id.replace("/", "--")
snapshots_dir = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
if os.path.isdir(snapshots_dir):
tokenizer_dir = None
for rev in os.listdir(snapshots_dir):
candidate = os.path.join(snapshots_dir, rev)
if os.path.isfile(os.path.join(candidate, "config.json")):
tokenizer_dir = candidate
break
if tokenizer_dir is None:
raise FileNotFoundError(f"No cached snapshot for {tokenizer_id}")
else:
raise FileNotFoundError(f"No cached snapshot for {tokenizer_id}")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
quantizer, fc2, acoustic_decoder, _cfg = load_higgs_audio_codec(tokenizer_dir, device)
self.quantizer = quantizer
self.fc2 = fc2
self.acoustic_decoder = acoustic_decoder
self._loaded = True
logger.info("Loaded HiggsAudioV3Code2Wav from standalone tokenizer repo.")
# ------------------------------------------------------------------ engine hooks
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(self, hidden_states: Any, sampling_metadata: Any = None) -> None:
return None
# ------------------------------------------------------------------ load
def load_weights(self, weights_or_model_dir, device: torch.device | None = None):
if self._loaded:
if not isinstance(weights_or_model_dir, (str, bytes, os.PathLike)):
for _ in weights_or_model_dir:
pass
return {name for name, _ in self.named_parameters()}
if not isinstance(weights_or_model_dir, (str, bytes, os.PathLike)):
# Engine path: try to extract V3 bundled codec weights first
codec_state: dict[str, torch.Tensor] = {}
for name, tensor in weights_or_model_dir:
if name.startswith(_CODEC_PREFIX):
stripped = name[len(_CODEC_PREFIX) :]
codec_state[stripped] = tensor
if codec_state:
# Bundled keys found — load strictly, no fallback on error.
self._load_from_bundled_state(codec_state, device)
logger.info(
"Loaded HiggsAudioV3Code2Wav from bundled V3 checkpoint (%d codec keys consumed).",
len(codec_state),
)
return {name for name, _ in self.named_parameters()}
# No bundled keys — fallback to standalone tokenizer repo
if not self._loaded:
try:
self._load_from_tokenizer_repo()
except (FileNotFoundError, OSError) as exc2:
raise RuntimeError(
"HiggsAudioV3Code2Wav: could not load codec weights. "
"Neither V3 bundled codec keys (tied.embedding."
"modality_embeddings.0.model.*) were found in the "
"checkpoint, nor is bosonai/higgs-audio-v2-tokenizer "
f"cached in HF_HOME. Last error: {exc2}"
) from exc2
return {name for name, _ in self.named_parameters()}
# File-based path
if not self._loaded:
self._load_from_tokenizer_repo()
return None
def _load_from_bundled_state(
self,
codec_state: dict[str, torch.Tensor],
device: torch.device | None = None,
) -> None:
"""Load codec from V3 checkpoint's bundled weights.
Keys arrive with the ``tied.embedding.modality_embeddings.0.model.``
prefix already stripped. We remap to the same layout that
``load_higgs_audio_codec`` expects and build the modules.
"""
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_decoder import (
_remap_boson_model_pth_state_dict,
)
if device is None:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# The bundled codec may use boson-ai key naming. Try remapping.
needs_remap = any(k.startswith("quantizer.vq.layers.") for k in codec_state)
if needs_remap:
codec_state = _remap_boson_model_pth_state_dict(codec_state)
# Discover parameters from the state dict
proj_key = "quantizer.quantizers.0.project_out.weight"
if proj_key not in codec_state:
raise KeyError(
f"Bundled codec state dict missing '{proj_key}'. "
f"Available keys (first 10): {list(codec_state.keys())[:10]}"
)
hidden_size = codec_state[proj_key].shape[0]
codebook_dim = 64
codebook_size = 1024
required_quantizers = self.num_codebooks
# Build RVQ — require exactly num_codebooks quantizers
quantizer = HiggsAudioRVQ(
num_quantizers=required_quantizers,
codebook_size=codebook_size,
codebook_dim=codebook_dim,
hidden_size=hidden_size,
).to(device)
for i in range(required_quantizers):
prefix = f"quantizer.quantizers.{i}"
# Require codebook weight
embed_key = None
for key_suffix in [".codebook.embed", ".codebook.weight"]:
candidate = f"{prefix}{key_suffix}"
if candidate in codec_state:
embed_key = candidate
break
if embed_key is None:
raise KeyError(
f"Bundled codec missing codebook weight for quantizer {i}. "
f"Expected {prefix}.codebook.embed or {prefix}.codebook.weight"
)
quantizer.quantizers[i].codebook.weight.data.copy_(codec_state[embed_key].to(device))
# Require projector weights
proj_w = f"{prefix}.project_out.weight"
proj_b = f"{prefix}.project_out.bias"
if proj_w not in codec_state:
raise KeyError(f"Bundled codec missing {proj_w}")
if proj_b not in codec_state:
raise KeyError(f"Bundled codec missing {proj_b}")
quantizer.quantizers[i].project_out.weight.data.copy_(codec_state[proj_w].to(device))
quantizer.quantizers[i].project_out.bias.data.copy_(codec_state[proj_b].to(device))
# Build fc2
fc2_w = codec_state.get("fc2.weight")
fc2_b = codec_state.get("fc2.bias")
if fc2_w is None or fc2_b is None:
raise KeyError(
"Bundled codec state dict missing fc2.weight and/or fc2.bias. "
f"Available keys (first 10): {list(codec_state.keys())[:10]}"
)
fc2 = nn.Linear(fc2_w.shape[1], fc2_w.shape[0]).to(device)
fc2.weight.data.copy_(fc2_w.to(device))
fc2.bias.data.copy_(fc2_b.to(device))
# Build acoustic decoder — detect layout from key names.
# V3 checkpoint uses OmniVoice layout (block.N/conv1/conv2/snake1),
# not boson-ai standalone layout (model.0/model.1).
ad_keys = {
k[len("acoustic_decoder.") :]: v for k, v in codec_state.items() if k.startswith("acoustic_decoder.")
}
is_boson_layout = any(k.startswith("model.") for k in ad_keys)
if is_boson_layout:
acoustic_decoder = build_boson_dac_decoder(device)
else:
# OmniVoice layout — need the v2 tokenizer config for DacConfig
tokenizer_config = {
"acoustic_model_config": {
"codebook_dim": 8,
"codebook_size": 1024,
"decoder_hidden_size": 1024,
"downsampling_ratios": [8, 5, 4, 2, 3],
"encoder_hidden_size": 64,
"hidden_size": 256,
"hop_length": 960,
"model_type": "dac",
"n_codebooks": 9,
"sampling_rate": 16000,
"upsampling_ratios": [8, 5, 4, 2, 3],
}
}
acoustic_decoder = build_higgs_audio_acoustic_decoder(tokenizer_config, device)
if not ad_keys:
raise KeyError(
"Bundled codec state dict has no acoustic_decoder.* keys. "
f"Available keys (first 10): {list(codec_state.keys())[:10]}"
)
load_report = acoustic_decoder.load_state_dict(ad_keys, strict=False)
if load_report.missing_keys:
raise RuntimeError(
f"Bundled DAC decoder missing {len(load_report.missing_keys)} keys: {load_report.missing_keys[:5]}..."
)
if load_report.unexpected_keys:
raise RuntimeError(
f"Bundled DAC decoder has {len(load_report.unexpected_keys)} unexpected keys: "
f"{load_report.unexpected_keys[:5]}..."
)
acoustic_decoder.eval()
# Verify required decoder-side codec keys were consumed. The V3
# checkpoint also bundles encoder-side keys (acoustic_encoder.*,
# encoder_semantic.*, decoder_semantic.*, semantic_model.*, fc.*, fc1.*)
# that the DAC decoder does not need — those are allowed but ignored.
consumed: set[str] = set()
for i in range(required_quantizers):
prefix = f"quantizer.quantizers.{i}"
for suffix in [".codebook.embed", ".codebook.weight"]:
if f"{prefix}{suffix}" in codec_state:
consumed.add(f"{prefix}{suffix}")
consumed.add(f"{prefix}.project_out.weight")
consumed.add(f"{prefix}.project_out.bias")
consumed.add("fc2.weight")
consumed.add("fc2.bias")
for k in codec_state:
if k.startswith("acoustic_decoder."):
consumed.add(k)
# Known unused prefixes: encoder-side modules and quantizer training
# artifacts (cluster_size, embed_avg, inited, project_in) that are
# bundled in the checkpoint but not needed for inference decoding.
_KNOWN_UNUSED_PREFIXES = (
"acoustic_encoder.",
"encoder_semantic.",
"decoder_semantic.",
"semantic_model.",
"fc.",
"fc1.",
)
# Quantizer training-only suffixes (per-codebook)
_QUANTIZER_TRAINING_SUFFIXES = (
".codebook.cluster_size",
".codebook.embed_avg",
".codebook.inited",
".project_in.weight",
".project_in.bias",
)
unconsumed = set()
for k in codec_state:
if k in consumed:
continue
if any(k.startswith(p) for p in _KNOWN_UNUSED_PREFIXES):
continue
if any(k.endswith(s) for s in _QUANTIZER_TRAINING_SUFFIXES):
continue
unconsumed.add(k)
if unconsumed:
raise RuntimeError(
f"Bundled codec has {len(unconsumed)} unexpected decoder-side keys: {sorted(unconsumed)[:10]}..."
)
self.quantizer = quantizer
self.fc2 = fc2
self.acoustic_decoder = acoustic_decoder
self._loaded = True
# ------------------------------------------------------------------ decode
@torch.inference_mode()
def decode_codes(self, audio_codes: torch.Tensor) -> torch.Tensor:
"""Decode [B, num_codebooks=8, T] codes to PCM [B, 1, T*960]."""
if not self._loaded:
raise RuntimeError("HiggsAudioV3Code2Wav not loaded.")
codes = self._validate_codes(audio_codes)
rvq_codes = codes.transpose(0, 1).long()
quantized = self.quantizer.decode(rvq_codes)
quantized = quantized.to(dtype=self.fc2.weight.dtype)
quantized = self.fc2(quantized.transpose(1, 2)).transpose(1, 2)
first_param = next(self.acoustic_decoder.parameters(), None)
if first_param is not None and quantized.dtype != first_param.dtype:
quantized = quantized.to(dtype=first_param.dtype)
audio = self.acoustic_decoder(quantized)
if audio.dim() == 2:
audio = audio.unsqueeze(1)
return audio.to(codes.device)
@torch.inference_mode()
def forward_chunk(
self,
audio_codes: torch.Tensor,
*,
left_context_size: int = 0,
right_holdback_size: int = 0,
hop_length: int | None = None,
) -> torch.Tensor:
# ``left_context_size`` frames trimmed off the front (already-emitted
# prefix). ``right_holdback_size`` frames trimmed off the end
# (deferred to the next chunk so the codec has future context for
# the emit region's trailing samples).
hop = int(hop_length) if hop_length is not None else self.hop_length
pcm = self.decode_codes(audio_codes)
right_trim = right_holdback_size * hop
left_trim = left_context_size * hop
if right_trim > 0:
if pcm.shape[-1] <= right_trim:
return pcm[..., :0]
pcm = pcm[..., :-right_trim]
if left_trim == 0:
return pcm
if pcm.shape[-1] <= left_trim:
return pcm[..., :0]
return pcm[..., left_trim:]
# ------------------------------------------------------------------ runtime forward
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> torch.Tensor | OmniOutput:
if isinstance(input_ids, torch.Tensor) and input_ids.ndim == 3:
return self.decode_codes(input_ids)
sr_val = int(self.sample_rate)
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
empty = torch.zeros((0,), dtype=torch.float32)
if input_ids is None or input_ids.numel() == 0:
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
)
ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
left_context_size = [0] * len(request_ids_list)
right_holdback_size = [0] * len(request_ids_list)
if runtime_additional_information is not None:
for i, info in enumerate(runtime_additional_information):
if i >= len(left_context_size):
break
meta = info.get("meta", {}) if isinstance(info, dict) else {}
if "left_context_size" in meta:
left_context_size[i] = int(meta["left_context_size"])
if "right_holdback_size" in meta:
right_holdback_size[i] = int(meta["right_holdback_size"])
wavs: list[torch.Tensor] = []
for i, req_ids in enumerate(request_ids_list):
n = int(req_ids.numel())
if n == 0:
wavs.append(empty)
continue
if n % self.num_codebooks != 0:
logger.warning(
"HiggsAudioV3Code2Wav: flat code length %d not divisible by %d",
n,
self.num_codebooks,
)
wavs.append(empty)
continue
frames = n // self.num_codebooks
codes_qf = req_ids.reshape(self.num_codebooks, frames)
codes_bqf = codes_qf.unsqueeze(0)
try:
pcm = self.forward_chunk(
codes_bqf,
left_context_size=left_context_size[i],
right_holdback_size=right_holdback_size[i],
hop_length=self.hop_length,
)
except ValueError as exc:
logger.warning("HiggsAudioV3Code2Wav: decode skipped (%s)", exc)
wavs.append(empty)
continue
wavs.append(pcm.squeeze(0).squeeze(0).to(torch.float32).cpu())
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"model_outputs": wavs,
"sr": [sr_tensor] * len(wavs),
},
)
# ------------------------------------------------------------------ helpers
def _validate_codes(self, audio_codes: torch.Tensor) -> torch.Tensor:
if audio_codes.ndim != 3:
raise ValueError(
f"audio_codes must have shape [B, {self.num_codebooks}, T]; got {tuple(audio_codes.shape)}"
)
if int(audio_codes.shape[1]) != self.num_codebooks:
raise ValueError(f"dim 1 must equal num_codebooks={self.num_codebooks}; got {int(audio_codes.shape[1])}")
# ``.item()`` triggers a GPU->CPU sync and is illegal inside CUDA-graph
# capture (the dummy run before warmup hits this path). Skip the
# range check while a graph is being captured; the per-step validator
# still fires at real inference time.
if audio_codes.numel() > 0 and not torch.cuda.is_current_stream_capturing():
max_val = int(audio_codes.max().item())
min_val = int(audio_codes.min().item())
if max_val >= self.num_real_codes or min_val < 0:
raise ValueError(
f"audio_codes out of range: min={min_val}, max={max_val}; expected [0, {self.num_real_codes - 1}]"
)
return audio_codes
@staticmethod
def _split_request_ids(ids: torch.Tensor, seq_token_counts: list[int] | None = None) -> list[torch.Tensor]:
if seq_token_counts is not None and len(seq_token_counts) > 1:
boundaries = [0]
for count in seq_token_counts:
boundaries.append(boundaries[-1] + int(count))
n = int(ids.numel())
return [ids[boundaries[i] : min(boundaries[i + 1], n)] for i in range(len(seq_token_counts))]
return [ids]
HiggsAudioV3Code2WavForConditionalGeneration = HiggsAudioV3Code2Wav
@@ -0,0 +1,958 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stage-0 talker for higgs-audio v3 (Qwen3 backbone, fused multi-codebook).
Architecture:
- Backbone: Qwen3 (~4B, 36 layers, 2560 hidden, GQA 32/8). No DualFFN.
- Fused multi-codebook embedding: [N*V, D] weight, offset lookup, sum across N
- Fused multi-codebook head: same weight (tied), reshape to [L, N, V]
- MusicGen-style delay pattern [0,1,...,7] with BOC/EOC
- Audio feedback: replace continuation-token embedding with fused codebook embed
Weight loading maps from the HF checkpoint's prefixes:
tied.embedding.text_embedding. -> model.embed_tokens.
body.layers. -> model.layers.
body.norm. -> model.norm.
tied.head.text_head. -> lm_head.
tied.embedding.modality_embeddings.0.embedding. -> multimodal_embedding.
tied.embedding.modality_embeddings.0.model.* -> skipped (codec for code2wav)
tied.head.modality_heads.0.* -> skipped when tied
"""
from __future__ import annotations
import copy
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.qwen3 import Qwen3Model
from vllm_omni.model_executor.models.higgs_audio_v3.configuration_higgs_audio_v3 import (
HiggsAudioV3Config,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
__all__ = ["HiggsAudioV3TalkerForConditionalGeneration"]
logger = init_logger(__name__)
# Delay pattern constants
BOC_ID = 1024 # beginning of codebook
EOC_ID = 1025 # end of codebook
# Checkpoint prefix mapping
_BACKBONE_PREFIX_MAP = {
"tied.embedding.text_embedding.": "model.embed_tokens.",
"body.layers.": "model.layers.",
"body.norm.": "model.norm.",
"tied.head.text_head.": "lm_head.",
}
_MODALITY_EMBEDDING_PREFIX = "tied.embedding.modality_embeddings.0.embedding."
_MODALITY_HEAD_PREFIX = "tied.head.modality_heads.0."
_CODEC_PREFIX = "tied.embedding.modality_embeddings.0.model."
class HiggsFusedMultiTextEmbedding(nn.Module):
"""Fused multi-codebook embedding: [N*V, D] weight + offset lookup."""
def __init__(self, num_codebooks: int, vocab_size: int, hidden_size: int):
super().__init__()
self.weight = nn.Parameter(torch.empty(num_codebooks * vocab_size, hidden_size))
self.num_codebooks = num_codebooks
self.vocab_size = vocab_size
def forward(self, codes: torch.Tensor) -> torch.Tensor:
N = self.num_codebooks
V = self.vocab_size
offsets = torch.arange(N, device=codes.device, dtype=codes.dtype) * V
fused_ids = codes + offsets
return F.embedding(fused_ids, self.weight).sum(dim=-2)
class HiggsFusedMultiTextHead(nn.Module):
"""Fused multi-codebook head: [L, D] -> [L, N, V] via one linear."""
def __init__(self, num_codebooks: int, vocab_size: int, hidden_size: int):
super().__init__()
self.weight = nn.Parameter(torch.empty(num_codebooks * vocab_size, hidden_size))
self.num_codebooks = num_codebooks
self.vocab_size = vocab_size
def generate(self, hidden: torch.Tensor) -> torch.Tensor:
logits = F.linear(hidden, self.weight)
return logits.reshape(hidden.shape[0], self.num_codebooks, self.vocab_size)
class HiggsAudioV3TalkerForConditionalGeneration(nn.Module):
"""Stage-0 talker for higgs-audio v3.
Wraps Qwen3Model backbone + fused multi-codebook modules for TTS generation
with MusicGen-style delay pattern sampling and audio feedback embedding.
"""
# Tell the AR runner to call model.sample() instead of the stock sampler.
prefer_model_sampler: bool = True
# Tell the runner to call postprocess() to emit per-step audio codes.
have_multimodal_outputs: bool = True
has_postprocess: bool = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
hf_config = vllm_config.model_config.hf_config
if isinstance(hf_config, HiggsAudioV3Config):
self.config = hf_config
else:
self.config = HiggsAudioV3Config(**hf_config.to_dict())
self.vllm_config = vllm_config
self.num_codebooks = int(self.config.num_codebooks)
self.codebook_size = int(self.config.codebook_size)
hidden_size = int(self.config.audio_hidden_size)
self.tie_modality = self.config.tie_modality_embeddings
# Fused multi-codebook modules
self.multimodal_embedding = HiggsFusedMultiTextEmbedding(self.num_codebooks, self.codebook_size, hidden_size)
self.modality_head = HiggsFusedMultiTextHead(self.num_codebooks, self.codebook_size, hidden_size)
if self.tie_modality:
self.modality_head.weight = self.multimodal_embedding.weight
# Qwen3 backbone
self._backbone_config = self.config.text_config
backbone_vllm_config = copy.copy(vllm_config)
backbone_model_config = copy.copy(vllm_config.model_config)
backbone_model_config.hf_config = self._backbone_config
backbone_vllm_config.model_config = backbone_model_config
self.model = Qwen3Model(
vllm_config=backbone_vllm_config,
prefix=f"{prefix}.model" if prefix else "model",
)
if self._backbone_config.tie_word_embeddings:
self.lm_head = self.model.embed_tokens
else:
self.lm_head = ParallelLMHead(
self._backbone_config.vocab_size,
self._backbone_config.hidden_size,
prefix=f"{prefix}.lm_head" if prefix else "lm_head",
)
self.logits_processor = LogitsProcessor(self._backbone_config.vocab_size)
# Audio continuation token ID — resolved lazily from tokenizer.
# This is the <|audio|> token that serves as the LM-level continuation
# marker during audio generation (equivalent to v2's audio_token_id).
self._audio_continuation_id: int | None = None
self._eos_token_id: int | None = None
self._resolved_tokens = False
# Per-request audio state keyed by batch row index.
# Reset per slot via _slot_output_len tracking (same pattern as v2).
self._audio_state: dict[int, dict[str, Any]] = {}
self._slot_output_len: dict[int, int] = {}
self._last_logits_hidden: torch.Tensor | None = None
self._last_step_input_ids: torch.Tensor | None = None
self._last_step_query_start_loc: torch.Tensor | None = None
self._last_first_audio_after_start: torch.Tensor | None = None
self._last_audio_codes: torch.Tensor | None = None
self._postprocess_cursor: int = 0
# Pre-allocated decode-step audio feedback buffers (CUDA-graph safe).
# Populated by sample(), read by forward() via torch.where (no dict).
max_bs = 64 # safe upper bound; will grow if needed
self._decode_last_codes = torch.zeros(max_bs, self.num_codebooks, dtype=torch.long)
self._decode_has_codes = torch.zeros(max_bs, dtype=torch.bool)
# PrefixCache opt-outs (mirror qwen3_tts pattern):
# 1. The talker only consumes the last token's hidden state, so the
# runner can skip the per-step full hidden-state GPU->CPU merge
# that PrefixCache otherwise does.
# 2. Per-step ``codes.audio`` rows stay GPU-resident; defer the CPU
# write of the prefix-cache mm-output copy to request finish so
# the per-step bookkeeping does not block batching. Stage 0 can
# then set ``enable_prefix_caching: true`` without the regression
# observed in qwen3_tts (#3665).
self.requires_full_prefix_cached_hidden_states = False
self.deferred_prefix_cache_mm_keys = {"codes.audio"}
def _resolve_token_ids(self) -> None:
"""Resolve <|audio|> and eos token IDs.
Prefers config's pre-resolved IDs (from ``resolve_special_tokens()``),
falls back to loading the HF tokenizer directly.
"""
if self._resolved_tokens:
return
self._resolved_tokens = True
# Try config first (populated by resolve_special_tokens or from_pretrained)
cfg_audio = getattr(self.config, "audio_continuation_id", None)
cfg_eos = getattr(self.config, "eos_token_id", None)
if cfg_audio is not None:
self._audio_continuation_id = int(cfg_audio)
if cfg_eos is not None:
self._eos_token_id = int(cfg_eos)
if self._audio_continuation_id is not None:
logger.info(
"Resolved v3 token IDs from config: audio_continuation=%s, eos=%s",
self._audio_continuation_id,
self._eos_token_id,
)
return
# Fallback: load tokenizer directly
model_path = getattr(self.vllm_config.model_config, "model", None)
if model_path is None:
return
try:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
vocab = dict(tokenizer.get_added_vocab())
if "<|audio|>" in vocab:
self._audio_continuation_id = vocab["<|audio|>"]
if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None:
self._eos_token_id = int(tokenizer.eos_token_id)
logger.info(
"Resolved v3 token IDs from tokenizer: audio_continuation=%s, eos=%s",
self._audio_continuation_id,
self._eos_token_id,
)
except Exception as exc:
logger.warning("Failed to resolve token IDs from tokenizer: %s", exc)
# ------------------------------------------------------------------ forward
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Any | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: Any,
) -> torch.Tensor:
if inputs_embeds is None:
# Mask -100 placeholders to 0 before embedding. Use torch.where
# (no Python data-dependent branch) so this is CUDA-graph safe.
safe_ids = torch.where(input_ids < 0, torch.zeros_like(input_ids), input_ids)
hidden_states = self.model.embed_tokens(safe_ids)
else:
hidden_states = inputs_embeds
if input_ids is not None:
self._last_step_input_ids = input_ids
# Stash query_start_loc for _apply_audio_mode_bias
try:
from vllm.forward_context import get_forward_context
attn_metadata = get_forward_context().attn_metadata
if isinstance(attn_metadata, dict) and attn_metadata:
attn = next(iter(attn_metadata.values()))
else:
attn = attn_metadata
qsl = getattr(attn, "query_start_loc", None)
if isinstance(qsl, torch.Tensor):
self._last_step_query_start_loc = qsl.detach().clone()
else:
self._last_step_query_start_loc = None
except Exception:
self._last_step_query_start_loc = None
# Prefill-only operations: ref audio substitution and audio feedback
# require Python dict/list ops that break CUDA graph capture.
# Detect prefill (sequence length > batch size heuristic) vs decode.
is_prefill = input_ids is not None and inputs_embeds is None and int(input_ids.numel()) > 1
if is_prefill:
# Voice clone: replace -100 placeholder positions with ref audio embeddings
info_dicts = kwargs.get("model_intermediate_buffer")
if info_dicts is None:
info_dicts = kwargs.get("runtime_additional_information")
hidden_states = self._apply_ref_audio_substitution(hidden_states, input_ids, info_dicts)
# Audio feedback at decode: replace continuation token embeddings
if input_ids is not None and inputs_embeds is None:
hidden_states = self._apply_audio_feedback(hidden_states, input_ids)
residual: torch.Tensor | None = None
for layer in self.model.layers:
hidden_states, residual = layer(positions, hidden_states, residual)
norm_out = self.model.norm(hidden_states, residual)
if isinstance(norm_out, tuple):
norm_out = norm_out[0]
return norm_out
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata: Any = None) -> torch.Tensor:
self._last_logits_hidden = hidden_states
return self.logits_processor(self.lm_head, hidden_states, sampling_metadata)
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
safe_ids = input_ids
if input_ids is not None and (input_ids < 0).any():
safe_ids = torch.where(input_ids < 0, torch.zeros_like(input_ids), input_ids)
text_embed = self.model.embed_tokens(safe_ids)
return self._apply_audio_feedback(text_embed, input_ids)
# ------------------------------------------------------------------ ref audio substitution
def _apply_ref_audio_substitution(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
info_dicts: list[dict[str, Any]] | None,
) -> torch.Tensor:
"""Replace -100 placeholder positions with fused multi-codebook embeddings
of the delay-pattern-encoded reference audio codes.
Called at prefill to inject voice clone reference. ``info_dicts`` is a
list of per-request dicts from ``model_intermediate_buffer``, each
containing ``audio_input_ids`` ([T, N] delayed codes) and
``audio_input_ids_mask`` ([T] bool mask).
"""
if not info_dicts:
return hidden_states
PLACEHOLDER = -100
flat_ids = input_ids.reshape(-1)
placeholder_mask = flat_ids == PLACEHOLDER
if not placeholder_mask.any():
return hidden_states
# Use query_start_loc to map placeholders to per-request spans
q_start = self._last_step_query_start_loc
if not isinstance(q_start, torch.Tensor) or q_start.numel() < 2:
# Fallback: single-request batch
q_start_list = [0, int(flat_ids.numel())]
else:
q_start_list = q_start.detach().to("cpu").tolist()
new_hidden: torch.Tensor | None = None
num_requests = min(len(info_dicts), len(q_start_list) - 1)
for i in range(num_requests):
info = info_dicts[i]
if not isinstance(info, dict):
continue
codes = info.get("audio_input_ids")
mask = info.get("audio_input_ids_mask")
# Handle msgspec serialization (may be list-wrapped)
if isinstance(codes, list):
codes = codes[0] if codes else None
if isinstance(mask, list):
mask = mask[0] if mask else None
if not isinstance(codes, torch.Tensor):
continue
# codes shape: [T, num_codebooks] delayed reference codes
if codes.ndim == 3:
codes = codes[0]
if codes.ndim != 2:
continue
if isinstance(mask, torch.Tensor):
if mask.ndim == 2:
mask = mask[0]
codes = codes[mask.to(dtype=torch.bool)]
if codes.numel() == 0:
continue
# Find placeholder positions in this request's span
s = int(q_start_list[i])
e = int(q_start_list[i + 1])
if e - s <= 1:
continue # Decode step, skip
span_mask = placeholder_mask[s:e]
placeholders = span_mask.nonzero(as_tuple=True)[0]
n_codes = int(codes.shape[0])
if int(placeholders.numel()) < n_codes:
continue # Mismatch
# Embed delayed codes via fused multi-codebook embedding
target = placeholders[:n_codes] + s
codes_device = codes.to(device=hidden_states.device, dtype=torch.long)
embeds = self.multimodal_embedding(codes_device) # [n_codes, hidden]
if new_hidden is None:
new_hidden = hidden_states.clone()
flat_hidden = new_hidden.reshape(-1, new_hidden.shape[-1])
flat_hidden[target] = embeds.to(new_hidden.dtype)
return new_hidden if new_hidden is not None else hidden_states
# ------------------------------------------------------------------ audio feedback
def _apply_audio_feedback(self, hidden_states: torch.Tensor, input_ids: torch.Tensor) -> torch.Tensor:
"""Replace decode-step embeddings with audio feedback from pre-allocated buffers.
CUDA-graph safe: reads from pre-allocated _decode_last_codes and
_decode_has_codes tensors using torch.where (no Python dict lookup).
The buffers are populated by sample() after each step.
For decode steps (1 token per request): position i maps to row i.
For prefill: audio feedback is not needed (ref audio substitution
handles the prefill path separately).
"""
bs = hidden_states.shape[0]
# Ensure buffers are on the right device and large enough
if self._decode_last_codes.device != hidden_states.device:
self._decode_last_codes = self._decode_last_codes.to(hidden_states.device)
self._decode_has_codes = self._decode_has_codes.to(hidden_states.device)
if bs > self._decode_last_codes.shape[0]:
new_size = max(bs, self._decode_last_codes.shape[0] * 2)
self._decode_last_codes = torch.zeros(
new_size, self.num_codebooks, dtype=torch.long, device=hidden_states.device
)
self._decode_has_codes = torch.zeros(new_size, dtype=torch.bool, device=hidden_states.device)
# Compute audio embeddings from last_codes for ALL rows (graph-safe)
codes_slice = self._decode_last_codes[:bs] # [bs, N]
has_codes = self._decode_has_codes[:bs].unsqueeze(-1) # [bs, 1]
audio_embeds = self.multimodal_embedding(codes_slice) # [bs, D]
audio_embeds = audio_embeds.to(dtype=hidden_states.dtype)
# Select: where has_codes, use audio embed; else keep text embed
return torch.where(has_codes, audio_embeds, hidden_states)
# ------------------------------------------------------------------ sampling
def sample(self, logits: torch.Tensor, sampling_metadata: Any) -> Any:
"""Model-owned sampler with delay-pattern audio dispatch.
Mirrors v2's pattern: bias LM logits to force audio continuation,
sample multi-codebook codes via the fused head, apply delay pattern,
and accumulate per-request state.
"""
self._resolve_token_ids()
sampler = getattr(self, "_stock_sampler", None)
if sampler is None:
from vllm.v1.sample.sampler import Sampler
sampler = Sampler()
self._stock_sampler = sampler
audio_id = self._audio_continuation_id
num_codebooks = self.num_codebooks
# Slot reuse detection: check output_token_ids length for each row.
# If a slot's output length decreased (fresh request reusing a finished
# slot), drop stale _audio_state for that row.
output_ids = getattr(sampling_metadata, "output_token_ids", None)
if output_ids is not None:
for i in range(len(output_ids)):
cur_len = len(output_ids[i]) if output_ids[i] else 0
prev_len = self._slot_output_len.get(i, 0)
if cur_len < prev_len:
# Fresh request reusing this slot — clear stale state
self._audio_state.pop(i, None)
self._slot_output_len[i] = 0
self._slot_output_len[i] = cur_len
# Bias LM logits for audio continuation
self._apply_audio_mode_bias(logits, sampling_metadata)
sampler_output = sampler(logits=logits, sampling_metadata=sampling_metadata)
hidden = self._last_logits_hidden
self._last_logits_hidden = None
if hidden is None or audio_id is None:
self._last_audio_codes = None
return sampler_output
sampled = getattr(sampler_output, "sampled_token_ids", None)
if sampled is None:
self._last_audio_codes = None
return sampler_output
sampled_flat = sampled.reshape(-1)
if int(sampled_flat.numel()) != int(hidden.shape[0]):
self._last_audio_codes = None
return sampler_output
is_audio = sampled_flat == audio_id
# Handle first-after-<|audio|> transition: initialize state but do NOT
# skip sampling. Unlike v2 (which seeds an all-BOC frame and skips the
# first audio step), v3/sglang samples codebook 0 on the very first
# step with delay_count=0 (codebooks 1-7 masked to BOC).
first_after_start = self._last_first_audio_after_start
self._last_first_audio_after_start = None
if isinstance(first_after_start, torch.Tensor) and first_after_start.numel() == is_audio.shape[0]:
first_after_start = first_after_start.to(is_audio.device)
init_rows = first_after_start & is_audio
if bool(init_rows.any()):
boc_frame = torch.full((num_codebooks,), BOC_ID, dtype=torch.long, device=hidden.device)
for bi in torch.nonzero(init_rows, as_tuple=False).reshape(-1).tolist():
bi = int(bi)
self._slot_output_len[bi] = 0
self._audio_state[bi] = {
"num_delay": 0,
"num_remaining_delays": None,
"audio_out_ids": None,
"last_codes": boc_frame.clone(),
"should_terminate": False,
}
# Update CUDA-graph-safe feedback buffers
self._decode_last_codes[bi] = boc_frame.to(self._decode_last_codes.device)
self._decode_has_codes[bi] = True
# Do NOT remove init_rows from is_audio — let them be sampled
if not bool(is_audio.any()):
self._last_audio_codes = None
return sampler_output
audio_row_indices = torch.nonzero(is_audio, as_tuple=False).reshape(-1).tolist()
# Per-codebook logits at audio positions
cb_logits = self._audio_codebook_logits(hidden, is_audio)
# Apply delay pattern masking BEFORE sampling
self._apply_delay_pattern_masking(cb_logits, audio_row_indices)
# Sample per-codebook
cb_logits_2d = cb_logits.reshape(-1, cb_logits.shape[-1])
codes_2d = self._sample_audio_codes(cb_logits_2d)
codes_flat = codes_2d.view(cb_logits.shape[0], cb_logits.shape[1]).to(torch.long)
# Update delay pattern state
eos_stream = EOC_ID
bos = BOC_ID
new_codes_flat: list[torch.Tensor] = []
for local_i, batch_i in enumerate(audio_row_indices):
state = self._audio_state.setdefault(
int(batch_i),
{
"num_delay": 0,
"num_remaining_delays": None,
"audio_out_ids": None,
"last_codes": torch.full((num_codebooks,), bos, dtype=torch.long, device=hidden.device),
"should_terminate": False,
},
)
num_delay: int = state["num_delay"]
num_remaining_delays: int | None = state["num_remaining_delays"]
this_codes = codes_flat[local_i].clone()
# Leading delay-pattern BOS pad
if num_delay + 1 < num_codebooks:
this_codes[num_delay + 1 :] = bos
num_delay += 1
# Trailing eos ramp-down
if num_remaining_delays is not None:
this_codes[: num_codebooks - num_remaining_delays] = eos_stream
num_remaining_delays -= 1
else:
eos_positions = (this_codes == eos_stream).nonzero(as_tuple=False).reshape(-1)
if eos_positions.numel() > 0:
last_eos_idx = int(eos_positions[-1].item())
this_codes[: last_eos_idx + 1] = eos_stream
num_remaining_delays = num_codebooks - last_eos_idx - 1
if num_remaining_delays is not None and num_remaining_delays <= 0:
# Ramp-down complete — terminate
state["num_delay"] = 0
state["num_remaining_delays"] = None
state["should_terminate"] = True
new_codes_flat.append(torch.full_like(this_codes, -1))
continue
state["num_delay"] = num_delay
state["num_remaining_delays"] = num_remaining_delays
state["last_codes"] = this_codes.clone()
# Update CUDA-graph-safe feedback buffers
self._decode_last_codes[int(batch_i)] = this_codes.to(self._decode_last_codes.device)
self._decode_has_codes[int(batch_i)] = True
if state["audio_out_ids"] is None:
state["audio_out_ids"] = this_codes.unsqueeze(-1).clone()
else:
state["audio_out_ids"] = torch.cat([state["audio_out_ids"], this_codes.unsqueeze(-1)], dim=-1)
new_codes_flat.append(this_codes)
# Build full codes tensor [batch_size, num_codebooks]
# -1 marks "no audio code at this position"
codes_full = torch.full(
(int(sampled_flat.numel()), num_codebooks),
-1,
dtype=torch.long,
device=hidden.device,
)
if new_codes_flat:
stacked = torch.stack(new_codes_flat, dim=0).to(hidden.device)
codes_full[is_audio] = stacked
self._last_audio_codes = codes_full
self._postprocess_cursor = 0
return sampler_output
# ------------------------------------------------------------------ postprocess
def postprocess(
self,
hidden_states_slice: torch.Tensor,
multimodal_outputs: Any = None,
**req_infos: Any,
) -> dict[str, Any]:
"""Publish per-request audio codes into model_intermediate_buffer.
Called once per request in batch order. Indexes _last_audio_codes
by a running cursor (one row per request per step).
"""
_ = multimodal_outputs
codes_full = self._last_audio_codes
if codes_full is None:
return {}
cursor = int(self._postprocess_cursor)
if cursor >= int(codes_full.shape[0]):
self._postprocess_cursor = 0
return {}
slice_codes = codes_full[cursor : cursor + 1]
self._postprocess_cursor = cursor + 1
# Drop placeholder rows (-1)
audio_rows = slice_codes[:, 0] >= 0
if not bool(audio_rows.any()):
return {}
new_codes = slice_codes[audio_rows].to(torch.int32)
return {"codes": {"audio": new_codes}}
# ------------------------------------------------------------------ helpers
def _audio_codebook_logits(self, hidden_states: torch.Tensor, audio_mask: torch.Tensor) -> torch.Tensor:
mask = audio_mask.reshape(-1).to(hidden_states.device)
hidden_flat = hidden_states.reshape(-1, hidden_states.shape[-1])
if not mask.any():
return torch.empty(
(0, self.num_codebooks, self.codebook_size),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
return self.modality_head.generate(hidden_flat[mask])
def _apply_delay_pattern_masking(self, cb_logits: torch.Tensor, audio_row_indices: list[int]) -> None:
"""Mask per-codebook logits according to delay pattern state, in-place.
During delay phase: codebooks beyond delay_count only allow BOC.
During ramp-down: locked codebooks only allow EOC.
Normal generation: BOC disallowed; only cb0 allows EOC.
"""
bos_pre = BOC_ID
eos_pre = EOC_ID
num_codebooks = self.num_codebooks
for local_i, batch_i in enumerate(audio_row_indices):
state = self._audio_state.get(int(batch_i))
num_delay = int(state["num_delay"]) if state else 0
num_rem = state.get("num_remaining_delays") if state else None
if num_rem is not None:
lock_until = num_codebooks - int(num_rem)
for q in range(num_codebooks):
row = cb_logits[local_i, q]
if q < lock_until:
mask = torch.full_like(row, float("-inf"))
mask[eos_pre] = row[eos_pre]
cb_logits[local_i, q] = mask
else:
cb_logits[local_i, q, bos_pre] = float("-inf")
cb_logits[local_i, q, eos_pre] = float("-inf")
else:
for q in range(num_codebooks):
row = cb_logits[local_i, q]
if q > num_delay:
mask = torch.full_like(row, float("-inf"))
mask[bos_pre] = row[bos_pre]
cb_logits[local_i, q] = mask
else:
cb_logits[local_i, q, bos_pre] = float("-inf")
if q != 0:
cb_logits[local_i, q, eos_pre] = float("-inf")
def _sample_audio_codes(self, logits_2d: torch.Tensor) -> torch.Tensor:
"""Replicate upstream sampling: temperature → top-k → top-p → multinomial."""
x = logits_2d.float()
top_k = 50
top_p = 0.95
if 0 < top_k < x.shape[-1]:
kth = x.topk(top_k, dim=-1).values[..., -1:]
x = x.masked_fill(x < kth, float("-inf"))
if 0.0 < top_p < 1.0:
sorted_logits, sorted_idx = x.sort(dim=-1, descending=True)
cumprobs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)
sorted_mask = cumprobs > top_p
sorted_mask[..., 1:] = sorted_mask[..., :-1].clone()
sorted_mask[..., 0] = False
mask = torch.zeros_like(x, dtype=torch.bool)
mask.scatter_(-1, sorted_idx, sorted_mask)
x = x.masked_fill(mask, float("-inf"))
# Detect all-masked rows BEFORE softmax (softmax of all-inf yields NaN).
has_finite = torch.isfinite(x).any(dim=-1)
all_masked = ~has_finite
if all_masked.any():
# For all-masked rows, fall back to argmax (picks least-negative).
fallback = x.argmax(dim=-1)
if has_finite.any():
probs = x[has_finite].softmax(dim=-1)
sampled = torch.multinomial(probs, num_samples=1).squeeze(-1)
result = fallback.clone()
result[has_finite] = sampled
return result
return fallback
probs = x.softmax(dim=-1)
return torch.multinomial(probs, num_samples=1).squeeze(-1)
def _apply_audio_mode_bias(self, logits: torch.Tensor, sampling_metadata: Any) -> None:
"""Detect <|audio|> transition, force continuation, force eos at ramp-down.
Mirrors v2's _apply_audio_mode_bias: walks per-request to find the
previous token, and if it was <|audio|> or the continuation token,
forces the next emit to <|audio|>. On ramp-down completion, forces eos.
"""
if logits is None or logits.ndim != 2:
return
audio_id = self._audio_continuation_id
eos_id = self._eos_token_id
if audio_id is None:
return
num_rows = int(logits.shape[0])
prompt_ids = getattr(sampling_metadata, "prompt_token_ids", None)
output_ids = getattr(sampling_metadata, "output_token_ids", None)
# Fallback prev-token source from stashed input_ids
stash_ids = self._last_step_input_ids
stash_tail: list[int] | None = None
if isinstance(stash_ids, torch.Tensor) and stash_ids.numel() > 0:
q_start = self._last_step_query_start_loc
if isinstance(q_start, torch.Tensor) and int(q_start.numel()) == num_rows + 1:
q_start_cpu = q_start.detach().to("cpu").tolist()
tail_idx = [max(0, int(q_start_cpu[i + 1]) - 1) for i in range(num_rows)]
flat_ids = stash_ids.detach().to("cpu").tolist()
stash_tail = [int(flat_ids[idx]) if idx < len(flat_ids) else -1 for idx in tail_idx]
elif int(stash_ids.numel()) >= num_rows:
stash_tail = stash_ids[-num_rows:].detach().to("cpu").tolist()
first_after_start = torch.zeros(num_rows, dtype=torch.bool, device=logits.device)
for i in range(num_rows):
prev: int | None = None
if output_ids is not None and i < len(output_ids):
hist = output_ids[i]
if hist:
prev = int(hist[-1])
if prev is None and prompt_ids is not None:
try:
p_i = prompt_ids[i]
if hasattr(p_i, "tolist"):
p_i = p_i.tolist()
if p_i:
prev = int(p_i[-1])
except (IndexError, TypeError):
prev = None
if prev is None and stash_tail is not None and i < len(stash_tail):
prev = int(stash_tail[i])
if prev is None:
continue
# Only bias if previous token was <|audio|> (the continuation token)
if prev != audio_id:
continue
# Check if this is the FIRST step after <|audio|> appears
# (i.e., transitioning from prompt to audio generation)
audio_state = self._audio_state.get(i)
if audio_state is None or audio_state.get("should_terminate"):
# No state yet, or stale state from a finished prior request
# reusing this slot — treat as first audio step
self._audio_state.pop(i, None)
# Clear CUDA-graph feedback buffers for this slot
self._decode_has_codes[i] = False
first_after_start[i] = True
# Check for ramp-down termination
should_terminate = bool(isinstance(audio_state, dict) and audio_state.get("should_terminate"))
if should_terminate and eos_id is not None and 0 <= eos_id < int(logits.shape[-1]):
row = logits[i]
mask = torch.full_like(row, float("-inf"))
mask[eos_id] = row[eos_id]
logits[i].copy_(mask)
audio_state["should_terminate"] = False
continue
# Force audio continuation token
row = logits[i]
mask = torch.full_like(row, float("-inf"))
if 0 <= audio_id < row.shape[-1]:
mask[audio_id] = row[audio_id]
logits[i].copy_(mask)
self._last_first_audio_after_start = first_after_start
# ------------------------------------------------------------------ omni output
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
hidden = model_outputs
info_dicts = kwargs.get("model_intermediate_buffer")
if info_dicts is None:
info_dicts = kwargs.get("runtime_additional_information")
if info_dicts is None:
info_dicts = []
audio_codes_list: list[torch.Tensor] = []
any_nonempty = False
for info in info_dicts:
ac: torch.Tensor | None = None
if isinstance(info, dict):
codes_field = info.get("codes")
if isinstance(codes_field, dict):
ac = codes_field.get("audio")
else:
ac = info.get("audio_codes")
if isinstance(ac, torch.Tensor) and ac.numel() > 0:
audio_codes_list.append(ac)
any_nonempty = True
else:
audio_codes_list.append(torch.empty(0, dtype=torch.long))
if any_nonempty:
return OmniOutput(
text_hidden_states=hidden,
multimodal_outputs={"codes": {"audio": audio_codes_list}},
)
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=None)
# ------------------------------------------------------------------ weight loading
# Per-layer suffixes from the actual V3 checkpoint (results/higgs_v3_checkpoint_analysis.txt)
_V3_LAYER_SUFFIXES = (
"input_layernorm.weight",
"mlp.down_proj.weight",
"mlp.gate_proj.weight",
"mlp.up_proj.weight",
"post_attention_layernorm.weight",
"self_attn.k_norm.weight",
"self_attn.k_proj.weight",
"self_attn.o_proj.weight",
"self_attn.q_norm.weight",
"self_attn.q_proj.weight",
"self_attn.v_proj.weight",
)
@classmethod
def _build_required_keys(cls, num_layers: int) -> set[str]:
"""Build the exact set of required V3 checkpoint keys."""
keys = {
"tied.embedding.text_embedding.weight",
"body.norm.weight",
f"{_MODALITY_EMBEDDING_PREFIX}weight",
}
for i in range(num_layers):
for suffix in cls._V3_LAYER_SUFFIXES:
keys.add(f"body.layers.{i}.{suffix}")
return keys
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
backbone_weights: list[tuple[str, torch.Tensor]] = []
loaded_params: set[str] = set()
own_params = dict(self.named_parameters())
seen_checkpoint_keys: set[str] = set()
for name, tensor in weights:
seen_checkpoint_keys.add(name)
mapped = self._map_weight_name(name)
if mapped is None:
continue
if mapped.startswith("model.") or mapped.startswith("lm_head."):
backbone_weights.append((mapped, tensor))
elif mapped in own_params:
param = own_params[mapped]
if param.shape != tensor.shape:
raise ValueError(
f"Shape mismatch for {mapped}: expected {param.shape}, "
f"got {tensor.shape} (checkpoint key: {name})"
)
param.data.copy_(tensor.to(param.dtype))
loaded_params.add(mapped)
if backbone_weights:
backbone_module = _BackboneWrapper(self.model, self.lm_head, self._backbone_config)
loaded = backbone_module.load_weights(iter(backbone_weights))
loaded_params.update(loaded)
# Resolve special token IDs from the tokenizer
model_path = getattr(self.vllm_config.model_config, "model", None)
if model_path:
self.config.resolve_special_tokens(model_path)
self._resolve_token_ids()
# Verify every required checkpoint key was seen.
num_layers = int(self._backbone_config.num_hidden_layers)
required = self._build_required_keys(num_layers)
missing = required - seen_checkpoint_keys
if missing:
raise RuntimeError(
f"HiggsAudioV3Talker: {len(missing)} required checkpoint keys missing: {sorted(missing)[:5]}..."
)
logger.info(
"HiggsAudioV3Talker: loaded %d params, modality_embedding=%s, tied=%s",
len(loaded_params),
tuple(self.multimodal_embedding.weight.shape),
self.tie_modality,
)
return loaded_params
def _map_weight_name(self, name: str) -> str | None:
if name.startswith(_CODEC_PREFIX):
return None
if name.startswith(_MODALITY_HEAD_PREFIX):
if self.tie_modality:
return None
return name.replace(_MODALITY_HEAD_PREFIX, "modality_head.")
if name.startswith(_MODALITY_EMBEDDING_PREFIX):
return name.replace(_MODALITY_EMBEDDING_PREFIX, "multimodal_embedding.")
for ckpt_prefix, model_prefix in _BACKBONE_PREFIX_MAP.items():
if name.startswith(ckpt_prefix):
return name.replace(ckpt_prefix, model_prefix, 1)
# Reject unexpected non-codec Higgs checkpoint keys
raise ValueError(
f"Unexpected checkpoint key with no known mapping: {name!r}. "
f"Known prefixes: {list(_BACKBONE_PREFIX_MAP.keys())}, "
f"{_MODALITY_EMBEDDING_PREFIX!r}, {_MODALITY_HEAD_PREFIX!r}, {_CODEC_PREFIX!r}"
)
class _BackboneWrapper(nn.Module):
"""Wrapper to use AutoWeightsLoader for Qwen3 backbone."""
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, model, lm_head, config):
super().__init__()
self.model = model
self.lm_head = lm_head
self.config = config
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
from vllm.model_executor.models.utils import AutoWeightsLoader
skip = ["lm_head."] if getattr(self.config, "tie_word_embeddings", False) else None
loader = AutoWeightsLoader(self, skip_prefixes=skip)
return loader.load_weights(weights)
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prompt builder for higgs-audio v3 TTS.
Prompt formats:
Zero-shot:
<|tts|> <|text|> {text tokens} <|audio|>
Voice clone (no ref text):
<|tts|> <|ref_audio|> [-100]×N <|text|> {text tokens} <|audio|>
Voice clone (with ref text):
<|tts|> <|ref_text|> {ref text tokens} <|ref_audio|> [-100]×N <|text|> {text tokens} <|audio|>
-100 placeholders are replaced at prefill time with fused multi-codebook
embeddings of the delay-pattern-encoded reference audio codes.
"""
from __future__ import annotations
from typing import Any
import numpy as np
import torch
__all__ = [
"HiggsAudioV3TokenizerAdapter",
"AUDIO_PLACEHOLDER_ID",
"encode_reference_audio",
"apply_delay_pattern",
]
AUDIO_PLACEHOLDER_ID = -100
_REQUIRED_SPECIALS: tuple[str, ...] = (
"<|tts|>",
"<|text|>",
"<|audio|>",
)
# Codec stream special token IDs (inside the codebook vocab, not the LM vocab)
BOC_ID = 1024
EOC_ID = 1025
def apply_delay_pattern(codes_tn: torch.Tensor) -> torch.Tensor:
"""Apply MusicGen-style delay pattern to raw codes.
Input: [T, N] raw codes (T frames, N codebooks).
Output: [T + N - 1, N] delayed codes with BOC/EOC padding.
Codebook c is delayed by c positions: rows 0..c-1 get BOC,
rows c..c+T-1 get real codes, rows c+T..T+N-2 get EOC.
"""
T, N = codes_tn.shape
L = T + N - 1
delayed = torch.full((L, N), EOC_ID, dtype=codes_tn.dtype)
for c in range(N):
delayed[:c, c] = BOC_ID
delayed[c : c + T, c] = codes_tn[:, c]
return delayed
def encode_reference_audio(
wav: np.ndarray,
sr: int,
) -> torch.Tensor:
"""Encode a reference audio clip to codec codes [T, num_codebooks].
Uses the same HiggsAudioV2TokenizerModel as v2 (same codec).
Returns raw codes before delay pattern application.
"""
from vllm_omni.model_executor.models.higgs_audio_v2.higgs_audio_v2_tokenizer import (
_encode_ref_audio_codes,
)
# _encode_ref_audio_codes returns [num_codebooks, T_raw]
codes_qt = _encode_ref_audio_codes(wav, sr)
# Transpose to [T_raw, num_codebooks]
return codes_qt.transpose(0, 1).contiguous()
class HiggsAudioV3TokenizerAdapter:
"""Wraps the HF tokenizer and builds TTS prompts."""
def __init__(self, tokenizer: Any) -> None:
self._tok = tokenizer
vocab = dict(tokenizer.get_added_vocab())
missing = [t for t in _REQUIRED_SPECIALS if t not in vocab]
if missing:
raise ValueError(f"Tokenizer is missing Higgs TTS v3 specials: {missing}")
self.tts_id: int = vocab["<|tts|>"]
self.text_id: int = vocab["<|text|>"]
self.audio_id: int = vocab["<|audio|>"]
self.ref_audio_id: int | None = vocab.get("<|ref_audio|>")
self.ref_text_id: int | None = vocab.get("<|ref_text|>")
@property
def tokenizer(self) -> Any:
return self._tok
def build_prompt(
self,
text: str,
*,
num_ref_tokens: int = 0,
reference_text: str | None = None,
) -> list[int]:
"""Build a TTS prompt.
Args:
text: Target text to synthesize.
num_ref_tokens: Number of delay-pattern reference code rows.
0 means zero-shot (no voice clone).
reference_text: Optional transcript of the reference audio.
"""
if not text or not text.strip():
raise ValueError("Text input must be non-empty for TTS")
if num_ref_tokens < 0:
raise ValueError(f"num_ref_tokens must be >= 0, got {num_ref_tokens}")
ids: list[int] = [self.tts_id]
# Voice clone: optional ref text + ref audio placeholders
if reference_text and num_ref_tokens > 0 and self.ref_text_id is not None:
ids.append(self.ref_text_id)
ids.extend(self._tok.encode(reference_text, add_special_tokens=False))
if num_ref_tokens > 0:
if self.ref_audio_id is None:
raise ValueError("Tokenizer missing <|ref_audio|> for voice clone")
ids.append(self.ref_audio_id)
ids.extend([AUDIO_PLACEHOLDER_ID] * num_ref_tokens)
ids.append(self.text_id)
ids.extend(self._tok.encode(text, add_special_tokens=False))
ids.append(self.audio_id)
return ids
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""higgs-audio v3 pipeline: Talker (text -> 8-codebook codec) -> Code2Wav (codec -> 24 kHz PCM).
Two delivery modes are wired here:
* ``sync_process_input_func`` runs when the deploy YAML has
``async_chunk: false``. The orchestrator collects the entire Stage-0 emit,
reverts the delay pattern once, and hands a single payload to Stage 1.
* ``async_chunk_process_next_stage_input_func`` runs when the deploy YAML
has ``async_chunk: true``. Stage 0 dispatches per AR step; the streaming
adapter buffers raw delay-pattern rows, slides a window with left context
and right holdback, and emits codec-ready frames per chunk. Stage 1
trims the overlap on both ends so the client sees a coherent PCM stream.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.higgs_audio_v3"
HIGGS_AUDIO_V3_PIPELINE = PipelineConfig(
model_type="higgs_multimodal_qwen3",
model_arch="HiggsAudioV3TalkerForConditionalGeneration",
hf_architectures=("HiggsMultimodalQwen3ForConditionalGeneration",),
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="higgs_audio_v3",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
# stop_token_ids: the model-owned sampler forces eos at ramp-down
# completion. Safety stops from the actual V3 checkpoint:
# 151643 = <|endoftext|> (eos_token_id from config.json)
# 151671 = <|audio_end|> (audio generation end marker)
sampling_constraints={
"detokenize": False,
"stop_token_ids": [151643, 151671],
},
async_chunk_process_next_stage_input_func=f"{_PROC}.talker2code2wav_async_chunk",
),
StagePipelineConfig(
stage_id=1,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
model_arch="HiggsAudioV3Code2WavForConditionalGeneration",
sync_process_input_func=f"{_PROC}.talker2code2wav",
sampling_constraints={"detokenize": True},
),
),
)
@@ -119,6 +119,22 @@ _OMNI_MODELS = {
"higgs_audio_v2_code2wav",
"HiggsAudioV2Code2WavForConditionalGeneration",
),
## higgs-audio v3
"HiggsMultimodalQwen3ForConditionalGeneration": (
"higgs_audio_v3",
"higgs_audio_v3_talker",
"HiggsAudioV3TalkerForConditionalGeneration",
),
"HiggsAudioV3TalkerForConditionalGeneration": (
"higgs_audio_v3",
"higgs_audio_v3_talker",
"HiggsAudioV3TalkerForConditionalGeneration",
),
"HiggsAudioV3Code2WavForConditionalGeneration": (
"higgs_audio_v3",
"higgs_audio_v3_code2wav",
"HiggsAudioV3Code2WavForConditionalGeneration",
),
## mimo_audio
"MiMoAudioModel": (
"mimo_audio",