[OmniVoice] Add two-stage TTS serving support (#2463)

Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-04-03 20:41:20 +00:00
1 parent 10db95f9a9
commit 0e83ebe1d4
24 files changed
+2737 -1

No files matched your search

@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OmniVoice configuration for vLLM-Omni two-stage pipeline."""
from transformers.configuration_utils import PretrainedConfig
class OmniVoiceConfig(PretrainedConfig):
"""Configuration for OmniVoice model in vLLM-Omni.
This mirrors the HuggingFace OmniVoiceConfig but adds fields needed
for the two-stage serving pipeline.
"""
model_type = "omnivoice"
def get_text_config(self, **kwargs):
"""Return self so vLLM uses our top-level config (which has
num_attention_heads etc.) instead of trying to extract a sub-config."""
return self
def __init__(self, **kwargs):
# HF repos (e.g. k2-fsa/OmniVoice) may nest generation hyperparameters.
gen_cfg = kwargs.pop("generation_config", None)
if isinstance(gen_cfg, dict):
for k, v in gen_cfg.items():
kwargs.setdefault(k, v)
super().__init__(**kwargs)
# Audio codec params (prefer values set by PretrainedConfig from config.json)
self.audio_vocab_size = getattr(self, "audio_vocab_size", 1025)
self.audio_mask_id = getattr(self, "audio_mask_id", 1024)
self.num_audio_codebook = getattr(self, "num_audio_codebook", 8)
self.audio_codebook_weights = getattr(
self,
"audio_codebook_weights",
[8, 8, 6, 6, 4, 4, 2, 2],
)
# LLM backbone params (Qwen3-0.6B defaults from HF config)
llm_config = getattr(self, "llm_config", None) or {}
if isinstance(llm_config, PretrainedConfig):
llm_config = llm_config.to_dict()
elif not isinstance(llm_config, dict):
llm_config = {}
self.llm_hidden_size = llm_config.get("hidden_size", 1024)
self.llm_num_hidden_layers = llm_config.get("num_hidden_layers", 28)
self.llm_num_attention_heads = llm_config.get("num_attention_heads", 16)
self.llm_num_key_value_heads = llm_config.get("num_key_value_heads", 8)
self.llm_intermediate_size = llm_config.get("intermediate_size", 3072)
self.llm_vocab_size = llm_config.get("vocab_size", 151676)
self.llm_max_position_embeddings = llm_config.get("max_position_embeddings", 40960)
self.llm_rope_theta = llm_config.get("rope_theta", 1000000.0)
self.llm_rms_norm_eps = llm_config.get("rms_norm_eps", 1e-6)
self.llm_head_dim = llm_config.get("head_dim", self.llm_hidden_size // self.llm_num_attention_heads)
# Expose LLM params at top level for vLLM ModelConfig compatibility
# (vLLM expects num_attention_heads, hidden_size, etc. on the config)
self.num_attention_heads = self.llm_num_attention_heads
self.num_key_value_heads = self.llm_num_key_value_heads
self.num_hidden_layers = self.llm_num_hidden_layers
self.hidden_size = self.llm_hidden_size
self.head_dim = self.llm_head_dim
if not hasattr(self, "vocab_size"):
self.vocab_size = self.llm_vocab_size
# Generation params (defaults from OmniVoiceGenerationConfig)
self.num_step = getattr(self, "num_step", 32)
self.guidance_scale = getattr(self, "guidance_scale", 2.0)
self.t_shift = getattr(self, "t_shift", 0.1)
self.layer_penalty_factor = getattr(self, "layer_penalty_factor", 5.0)
self.position_temperature = getattr(self, "position_temperature", 5.0)
self.class_temperature = getattr(self, "class_temperature", 0.0)
# Audio output
self.sample_rate = getattr(self, "sample_rate", 24000)
self.frame_rate = getattr(self, "frame_rate", 25)
# Serving
self.speculative_config = None
@@ -0,0 +1,281 @@
#!/usr/bin/env python3
# Copyright 2026 Xiaomi Corp. (authors: Han Zhu)
#
# See ../../LICENSE for clarification regarding multiple authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Text duration estimation for TTS generation.
Provides ``RuleDurationEstimator``, which estimates audio duration from text
using character phonetic weights across 600+ languages. Used by
``OmniVoice.generate()`` to determine output length when no duration is specified.
"""
import bisect
import unicodedata
from functools import lru_cache
class RuleDurationEstimator:
def __init__(self):
# ==========================================
# 1. Phonetic Weights Table
# ==========================================
# The weight represents the relative speaking time compared to
# a standard Latin letter.
# Benchmark: 1.0 = One Latin Character (~40-50ms)
self.weights = {
# --- Logographic (1 char = full syllable/word) ---
"cjk": 3.0, # Chinese, Japanese Kanji, etc.
# --- Syllabic / Blocks
"hangul": 2.5, # Korean Hangul
"kana": 2.2, # Japanese Hiragana/Katakana
"ethiopic": 3.0, # Amharic/Ge'ez
"yi": 3.0, # Yi script
# --- Abugida (Consonant-Vowel complexes) ---
"indic": 1.8, # Hindi, Bengali, Tamil, etc.
"thai_lao": 1.5, # Thai, Lao
"khmer_myanmar": 1.8, # Khmer, Myanmar
# --- Abjad (Consonant-heavy) ---
"arabic": 1.5, # Arabic, Persian, Urdu
"hebrew": 1.5, # Hebrew
# --- Alphabet (Segmental) ---
"latin": 1.0, # English, Spanish, French, Vietnamese, etc. (Baseline)
"cyrillic": 1.0, # Russian, Ukrainian
"greek": 1.0, # Greek
"armenian": 1.0, # Armenian
"georgian": 1.0, # Georgian
# --- Symbols & Misc ---
"punctuation": 0.5, # Pause capability
"space": 0.2, # Word boundary/Breath (0.05 / 0.22)
"digit": 3.5, # Numbers
"mark": 0.0, # Diacritics/Accents (Silent modifiers)
"default": 1.0, # Fallback for unknown scripts
}
# ==========================================
# 2. Unicode Range Mapping
# ==========================================
# Format: (End_Codepoint, Type_Key)
# Used for fast binary search (bisect).
self.ranges = [
(0x02AF, "latin"), # Latin (Basic, Supplement, Ext, IPA)
(0x03FF, "greek"), # Greek & Coptic
(0x052F, "cyrillic"), # Cyrillic
(0x058F, "armenian"), # Armenian
(0x05FF, "hebrew"), # Hebrew
(0x077F, "arabic"), # Arabic, Syriac, Arabic Supplement
(0x089F, "arabic"), # Arabic Extended-B (+ Syriac Supp)
(0x08FF, "arabic"), # Arabic Extended-A
(0x097F, "indic"), # Devanagari
(0x09FF, "indic"), # Bengali
(0x0A7F, "indic"), # Gurmukhi
(0x0AFF, "indic"), # Gujarati
(0x0B7F, "indic"), # Oriya
(0x0BFF, "indic"), # Tamil
(0x0C7F, "indic"), # Telugu
(0x0CFF, "indic"), # Kannada
(0x0D7F, "indic"), # Malayalam
(0x0DFF, "indic"), # Sinhala
(0x0EFF, "thai_lao"), # Thai & Lao
(0x0FFF, "indic"), # Tibetan (Abugida)
(0x109F, "khmer_myanmar"), # Myanmar
(0x10FF, "georgian"), # Georgian
(0x11FF, "hangul"), # Hangul Jamo
(0x137F, "ethiopic"), # Ethiopic
(0x139F, "ethiopic"), # Ethiopic Supplement
(0x13FF, "default"), # Cherokee
(0x167F, "default"), # Canadian Aboriginal Syllabics
(0x169F, "default"), # Ogham
(0x16FF, "default"), # Runic
(0x171F, "default"), # Tagalog (Baybayin)
(0x173F, "default"), # Hanunoo
(0x175F, "default"), # Buhid
(0x177F, "default"), # Tagbanwa
(0x17FF, "khmer_myanmar"), # Khmer
(0x18AF, "default"), # Mongolian
(0x18FF, "default"), # Canadian Aboriginal Syllabics Ext
(0x194F, "indic"), # Limbu
(0x19DF, "indic"), # Tai Le & New Tai Lue
(0x19FF, "khmer_myanmar"), # Khmer Symbols
(0x1A1F, "indic"), # Buginese
(0x1AAF, "indic"), # Tai Than
(0x1B7F, "indic"), # Balinese
(0x1BBF, "indic"), # Sundanese
(0x1BFF, "indic"), # Batak
(0x1C4F, "indic"), # Lepcha
(0x1C7F, "indic"), # Ol Chiki (Santali)
(0x1C8F, "cyrillic"), # Cyrillic Extended-C
(0x1CBF, "georgian"), # Georgian Extended
(0x1CCF, "indic"), # Sundanese Supplement
(0x1CFF, "indic"), # Vedic Extensions
(0x1D7F, "latin"), # Phonetic Extensions
(0x1DBF, "latin"), # Phonetic Extensions Supplement
(0x1DFF, "default"), # Combining Diacritical Marks Supplement
(0x1EFF, "latin"), # Latin Extended Additional (Vietnamese)
(0x309F, "kana"), # Hiragana
(0x30FF, "kana"), # Katakana
(0x312F, "cjk"), # Bopomofo (Pinyin)
(0x318F, "hangul"), # Hangul Compatibility Jamo
(0x9FFF, "cjk"), # CJK Unified Ideographs (Main)
(0xA4CF, "yi"), # Yi Syllables
(0xA4FF, "default"), # Lisu
(0xA63F, "default"), # Vai
(0xA69F, "cyrillic"), # Cyrillic Extended-B
(0xA6FF, "default"), # Bamum
(0xA7FF, "latin"), # Latin Extended-D
(0xA82F, "indic"), # Syloti Nagri
(0xA87F, "default"), # Phags-pa
(0xA8DF, "indic"), # Saurashtra
(0xA8FF, "indic"), # Devanagari Extended
(0xA92F, "indic"), # Kayah Li
(0xA95F, "indic"), # Rejang
(0xA97F, "hangul"), # Hangul Jamo Extended-A
(0xA9DF, "indic"), # Javanese
(0xA9FF, "khmer_myanmar"), # Myanmar Extended-B
(0xAA5F, "indic"), # Cham
(0xAA7F, "khmer_myanmar"), # Myanmar Extended-A
(0xAADF, "indic"), # Tai Viet
(0xAAFF, "indic"), # Meetei Mayek Extensions
(0xAB2F, "ethiopic"), # Ethiopic Extended-A
(0xAB6F, "latin"), # Latin Extended-E
(0xABBF, "default"), # Cherokee Supplement
(0xABFF, "indic"), # Meetei Mayek
(0xD7AF, "hangul"), # Hangul Syllables
(0xFAFF, "cjk"), # CJK Compatibility
(0xFDFF, "arabic"), # Arabic Presentation Forms-A
(0xFE6F, "default"), # Variation Selectors
(0xFEFF, "arabic"), # Arabic Presentation Forms-B
(0xFFEF, "latin"), # Fullwidth Latin
]
self.breakpoints = [r[0] for r in self.ranges]
@lru_cache(maxsize=4096)
def _get_char_weight(self, char):
"""Determines the weight of a single character."""
code = ord(char)
if (65 <= code <= 90) or (97 <= code <= 122):
return self.weights["latin"]
if code == 32:
return self.weights["space"]
# Ignore arabic Tatweel
if code == 0x0640:
return self.weights["mark"]
category = unicodedata.category(char)
if category.startswith("M"):
return self.weights["mark"]
if category.startswith("P") or category.startswith("S"):
return self.weights["punctuation"]
if category.startswith("Z"):
return self.weights["space"]
if category.startswith("N"):
return self.weights["digit"]
# 3. Binary search for Unicode Block (此时区间里绝不会再混进标点符号)
idx = bisect.bisect_left(self.breakpoints, code)
if idx < len(self.ranges):
script_type = self.ranges[idx][1]
return self.weights.get(script_type, self.weights["default"])
# 4. Handle upper planes (CJK Ext B/C/D, Historic scripts)
if code > 0x20000:
return self.weights["cjk"]
return self.weights["default"]
def calculate_total_weight(self, text):
"""Sums up the normalized weights for a string."""
return sum(self._get_char_weight(c) for c in text)
def estimate_duration(
self,
target_text: str,
ref_text: str,
ref_duration: float,
low_threshold: float | None = 50,
boost_strength: float = 3,
) -> float:
"""
Args:
target_text (str): The text for which we want to estimate the duration.
ref_text (str): The reference text that was used to measure
the ref_duration.
ref_duration (float): The actual duration it took
to speak the ref_text.
low_threshold (float): The minimum duration threshold below which the
estimation will be considered unreliable.
boost_strength (float): Controls the power-curve boost for short durations.
Higher values boost small durations more aggressively.
1 = no boost (linear), 2 = sqrt-like
Returns:
float: The estimated duration for the target_text based
on the ref_text and ref_duration.
"""
if ref_duration <= 0 or not ref_text:
return 0.0
ref_weight = self.calculate_total_weight(ref_text)
if ref_weight == 0:
return 0.0
speed_factor = ref_weight / ref_duration
target_weight = self.calculate_total_weight(target_text)
estimated_duration = target_weight / speed_factor
if low_threshold is not None and estimated_duration < low_threshold:
alpha = 1.0 / boost_strength
return low_threshold * (estimated_duration / low_threshold) ** alpha
else:
return estimated_duration
# ==========================================
# Example Usage
# ==========================================
if __name__ == "__main__":
estimator = RuleDurationEstimator()
ref_txt = "Hello, world."
ref_dur = 1.5
test_cases = [
("Hindi (With complex marks)", "नमस्ते दुनिया"),
("Arabic (With vowels)", "مَرْحَبًا بِالْعَالَم"),
("Vietnamese (Lots of diacritics)", "Chào thế giới"),
("Chinese", "你好,世界!"),
("Mixed Emoji", "Hello 🌍! This is fun 🎉"),
]
print("--- Reference ---")
print(f"Reference Text: '{ref_txt}'")
print(f"Reference Duration: {ref_dur}s")
print("-" * 30)
for lang, txt in test_cases:
est_time = estimator.estimate_duration(txt, ref_txt, ref_dur)
weight = estimator.calculate_total_weight(txt)
print(f"[{lang}]")
print(f"Text: {txt}")
print(f"Total Weight: {weight:.2f}")
print(f"Estimated Duration: {est_time:.2f} s")
print("-" * 30)
@@ -0,0 +1,520 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
OmniVoice model for vLLM-Omni two-stage TTS pipeline.
Stage 0 (Generator): Qwen3 backbone + iterative unmasking → 8-codebook tokens
Stage 1 (Decoder): HiggsAudioV2 decoder → 24kHz waveform
"""
from __future__ import annotations
import os
from collections.abc import Iterable, Mapping, Sequence
import numpy as np
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.logger import init_logger
from vllm.multimodal.inputs import (
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
ProcessorInputs,
PromptIndexTargets,
PromptInsertion,
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.omnivoice.config import OmniVoiceConfig
from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
# ---------------------------------------------------------------------------
# Multimodal processing
# ---------------------------------------------------------------------------
class OmniVoiceMultiModalProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(OmniVoiceConfig)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None}
def get_data_parser(self):
return MultiModalDataParser(
target_sr=self.ctx.get_hf_config().sample_rate,
expected_hidden_size=self._get_expected_hidden_size(),
)
class OmniVoiceMultiModalProcessor(BaseMultiModalProcessor[OmniVoiceMultiModalProcessingInfo]):
"""Processes text + optional reference audio for OmniVoice.
For voice cloning: text + reference audio → tokenized reference
For auto voice: text only
"""
def _ensure_cached_runtime_components(self, model_dir: str, config: OmniVoiceConfig) -> None:
cached_model_dir = getattr(self, "_cached_model_dir", None)
if cached_model_dir == model_dir:
return
from transformers import AutoTokenizer
self.text_tokenizer = AutoTokenizer.from_pretrained(model_dir)
# Audio tokenizer for encoding reference audio
audio_tokenizer_path = os.path.join(model_dir, "audio_tokenizer")
if os.path.isdir(audio_tokenizer_path):
try:
from transformers import (
AutoFeatureExtractor,
HiggsAudioV2TokenizerModel,
)
except ImportError as e:
raise ImportError(
"OmniVoice voice cloning requires transformers with "
"HiggsAudioV2TokenizerModel. Upgrade transformers or "
"use text-only mode (no reference audio)."
) from e
self.audio_tokenizer = HiggsAudioV2TokenizerModel.from_pretrained(audio_tokenizer_path, device_map="cpu")
self.feature_extractor = AutoFeatureExtractor.from_pretrained(audio_tokenizer_path)
self.audio_tokenizer.eval()
else:
self.audio_tokenizer = None
self.feature_extractor = None
logger.warning(
"audio_tokenizer not found at %s, voice cloning disabled",
audio_tokenizer_path,
)
self._cached_model_dir = model_dir
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
config = self.info.ctx.get_hf_config()
model_dir = self.info.ctx.model_config.model
self._ensure_cached_runtime_components(model_dir, config)
audio = mm_data.get("audio", None)
if audio is None:
audio = mm_data.get("audios")
if audio is not None:
audio = audio[0], config.sample_rate
# Build text prompt with control tokens
lang = mm_kwargs.get("lang", None)
instruct = mm_kwargs.get("instruct", None)
denoise = mm_kwargs.get("denoise", True)
ref_text = mm_kwargs.get("ref_text", None)
# Construct the style + text portion
style_text = ""
if denoise:
style_text += "<|denoise|>"
lang_str = lang if lang else "None"
instruct_str = instruct if instruct else "None"
style_text += f"<|lang_start|>{lang_str}<|lang_end|>"
style_text += f"<|instruct_start|>{instruct_str}<|instruct_end|>"
# Combine ref_text and main text
if ref_text:
full_text = f"{ref_text} {prompt}"
else:
full_text = prompt
text_prompt = f"{style_text}<|text_start|>{full_text}<|text_end|>"
text_tokens = self.text_tokenizer(text_prompt, return_tensors="pt").input_ids.squeeze(0) # [N_text]
if audio is None:
# Text-only path (auto voice mode)
return BatchFeature(
{
"input_ids": text_tokens,
"input_len": [len(text_tokens)],
}
)
# Voice cloning: encode reference audio to tokens
audio_signal, sr = audio
if isinstance(audio_signal, np.ndarray):
audio_signal = torch.from_numpy(audio_signal).float()
if audio_signal.dim() == 1:
audio_signal = audio_signal.unsqueeze(0)
# Resample to tokenizer sample rate if needed
if self.feature_extractor is not None:
target_sr = self.feature_extractor.sampling_rate
if sr != target_sr:
import torchaudio
audio_signal = torchaudio.functional.resample(audio_signal, sr, target_sr)
# Encode reference audio to 8-codebook tokens
if self.audio_tokenizer is not None:
with torch.inference_mode():
ref_audio_tokens = self.audio_tokenizer.encode(audio_signal) # [8, T_ref]
if ref_audio_tokens.dim() == 3:
ref_audio_tokens = ref_audio_tokens.squeeze(0) # [8, T_ref]
else:
raise RuntimeError(
"Audio tokenizer not available for voice cloning. Ensure audio_tokenizer/ exists in model directory."
)
ft = BatchFeature(
{
"input_ids": text_tokens,
"ref_audio_tokens": ref_audio_tokens, # [8, T_ref]
"ref_audio_len": [ref_audio_tokens.shape[1]],
}
)
return ft
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return {
"ref_audio_tokens": MultiModalFieldConfig.batched("audio"),
"ref_audio_len": MultiModalFieldConfig.batched("audio"),
}
def _hf_processor_applies_updates(
self,
prompt_text: str,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
) -> bool:
return False
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
def insertion_end(item_idx):
if "audio" in out_mm_kwargs and out_mm_kwargs["audio"]:
ref_len = out_mm_kwargs["audio"][0]["ref_audio_len"].data[0].item()
return [1] * ref_len
return []
return [
PromptInsertion(
modality="audio",
target=PromptIndexTargets.start(),
insertion=insertion_end,
),
]
class OmniVoiceDummyInputsBuilder(BaseDummyInputsBuilder[OmniVoiceMultiModalProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
return "Hello, this is a test of the OmniVoice system."
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio")
max_prompt_seconds = 10
prompt_sample_rate = 24000
target_audio_length = max_prompt_seconds * prompt_sample_rate
audio_overrides = mm_options.get("audio") if mm_options else None
mm_data = {
"audio": (
self._get_dummy_audios(
length=target_audio_length,
num_audios=num_audios,
overrides=audio_overrides,
)[0],
24000,
),
}
return mm_data
def get_dummy_processor_inputs(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> ProcessorInputs:
inputs = super().get_dummy_processor_inputs(seq_len, mm_counts, mm_options)
inputs.hf_processor_mm_kwargs = {"ref_text": "Testing voice cloning."}
return inputs
# ---------------------------------------------------------------------------
# Main model class
# ---------------------------------------------------------------------------
class OmniVoiceModel(
nn.Module,
):
"""OmniVoice model for vLLM-Omni two-stage pipeline.
Routes to generator (Stage 0) or decoder (Stage 1) based on model_stage.
"""
requires_raw_input_tokens = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.have_multimodal_outputs = True
self.model_stage = vllm_config.model_config.model_stage
self.model_dir = vllm_config.model_config.model
if self.model_stage == "omnivoice_generator":
from vllm_omni.model_executor.models.omnivoice.omnivoice_generator import (
OmniVoiceGenerator,
)
self.generator = OmniVoiceGenerator(self.config)
self.model = self.generator
elif self.model_stage == "omnivoice_decoder":
from vllm_omni.model_executor.models.omnivoice.omnivoice_decoder import (
OmniVoiceDecoder,
)
self.decoder = OmniVoiceDecoder(self.config)
self.model = self.decoder
else:
raise ValueError(f"Unsupported model_stage: {self.model_stage}")
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings=None,
is_multimodal=None,
) -> torch.Tensor:
if self.model_stage == "omnivoice_generator":
# Generator handles its own embedding in forward()
hidden = int(self.config.llm_hidden_size)
return torch.zeros((input_ids.shape[0], hidden), device=input_ids.device)
elif self.model_stage == "omnivoice_decoder":
hidden = int(self.config.llm_hidden_size)
return torch.zeros((input_ids.shape[0], hidden), device=input_ids.device)
else:
raise RuntimeError(f"embed_input_ids not valid for {self.model_stage}")
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
additional_information: dict[str, object] | None = None,
**kwargs: object,
) -> OmniOutput:
if self.model_stage == "omnivoice_generator":
return self._forward_generator(input_ids, kwargs)
elif self.model_stage == "omnivoice_decoder":
return self._forward_decoder(input_ids, kwargs)
else:
raise ValueError(f"Unsupported model_stage: {self.model_stage}")
def _forward_generator(self, input_ids: torch.Tensor, kwargs: dict) -> OmniOutput:
"""Run generator stage: text → 8-codebook audio tokens."""
runtime_info = kwargs.get("runtime_additional_information", [])
if not runtime_info:
# Profiling / dummy run — return a plain tensor (not OmniOutput)
# so the v1 model runner's _dummy_run can index into it.
return torch.zeros(
(input_ids.shape[0], self.config.llm_hidden_size),
device=input_ids.device,
dtype=torch.float32,
)
info = runtime_info[0]
device = input_ids.device
num_codebooks = self.config.num_audio_codebook
mask_id = self.config.audio_mask_id
# Extract text tokens from input_ids
text_tokens = input_ids # [N_text]
text_len = text_tokens.shape[0]
# Estimate target length using RuleDurationEstimator
# (same formula as reference OmniVoice: weight * 25 / 14.1)
from vllm_omni.model_executor.models.omnivoice.duration import (
RuleDurationEstimator,
)
if not hasattr(self, "_duration_estimator"):
self._duration_estimator = RuleDurationEstimator()
raw_text = info.get("raw_text", "")
if raw_text:
target_len = self._duration_estimator.estimate_duration(raw_text, "Nice to meet you.", 25)
target_len = max(1, int(target_len))
else:
# Fallback: use character weight formula on text tokens
# approximate ~1.77 frames per text token (25/14.1)
target_len = max(int(text_len * 1.77), 25)
# Get reference audio tokens if available
ref_audio_tokens = info.get("ref_audio_tokens", None)
# Build input_ids tensor: [2*B, 8, S]
# B=1, conditional + unconditional
# Replicate text tokens across 8 codebooks
text_ids = text_tokens.unsqueeze(0).repeat(num_codebooks, 1) # [8, N_text]
# Target: all MASK
target_ids = torch.full((num_codebooks, target_len), mask_id, dtype=torch.long, device=device)
# Conditional: [text] [ref_audio?] [target_mask]
if ref_audio_tokens is not None:
ref_tokens = ref_audio_tokens.to(device) # [8, T_ref]
cond_ids = torch.cat([text_ids, ref_tokens, target_ids], dim=1)
cond_audio_start = text_ids.shape[1]
else:
cond_ids = torch.cat([text_ids, target_ids], dim=1)
cond_audio_start = text_ids.shape[1]
cond_len = cond_ids.shape[1]
# Unconditional: [target_mask only]
uncond_ids = target_ids.clone()
uncond_len = target_len
# Pad to same length
max_len = max(cond_len, uncond_len)
if cond_len < max_len:
pad = torch.full(
(num_codebooks, max_len - cond_len),
mask_id,
dtype=torch.long,
device=device,
)
cond_ids = torch.cat([cond_ids, pad], dim=1)
if uncond_len < max_len:
pad = torch.full(
(num_codebooks, max_len - uncond_len),
mask_id,
dtype=torch.long,
device=device,
)
uncond_ids = torch.cat([uncond_ids, pad], dim=1)
batch_input_ids = torch.stack([cond_ids, uncond_ids], dim=0) # [2, 8, max_len]
# Audio mask: True for audio positions
batch_audio_mask = torch.zeros((2, max_len), dtype=torch.bool, device=device)
batch_audio_mask[0, cond_audio_start:cond_len] = True
batch_audio_mask[1, :uncond_len] = True
# Attention mask: [2, 1, S, S]
batch_attention_mask = torch.zeros((2, 1, max_len, max_len), dtype=torch.bool, device=device)
batch_attention_mask[0, :, :cond_len, :cond_len] = True
batch_attention_mask[1, :, :uncond_len, :uncond_len] = True
# Run iterative generation
tokens = self.generator(
input_ids=batch_input_ids,
audio_mask=batch_audio_mask,
attention_mask=batch_attention_mask,
target_lens=[target_len],
num_step=self.config.num_step,
guidance_scale=self.config.guidance_scale,
t_shift=self.config.t_shift,
layer_penalty_factor=self.config.layer_penalty_factor,
position_temperature=self.config.position_temperature,
class_temperature=self.config.class_temperature,
) # [1, 8, target_len]
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"audio_tokens": tokens},
)
def _forward_decoder(self, input_ids: torch.Tensor, kwargs: dict) -> OmniOutput:
"""Run decoder stage: 8-codebook tokens → audio waveform."""
runtime_info = kwargs.get("runtime_additional_information", [])
if not runtime_info:
# Profiling / dummy run — return plain tensor for v1 runner compat
return torch.zeros(
(input_ids.shape[0], self.config.llm_hidden_size),
device=input_ids.device,
dtype=torch.float32,
)
info = runtime_info[0]
audio_tokens = info.get("audio_tokens", None)
if audio_tokens is None:
raise RuntimeError("No audio_tokens received from generator stage")
if isinstance(audio_tokens, np.ndarray):
audio_tokens = torch.from_numpy(audio_tokens)
# audio_tokens: [B, 8, T]; buffer may be CPU — move to decoder weights
if audio_tokens.dim() == 2:
audio_tokens = audio_tokens.unsqueeze(0) # Add batch dim
dec_device = next(self.decoder.parameters()).device
audio_tokens = audio_tokens.to(device=dec_device, dtype=torch.long)
tts_speech = self.decoder(audio_tokens)
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"audio": tts_speech,
"sr": self.config.sample_rate,
},
)
def _resolve_model_dir(self) -> str:
"""Resolve model directory to local path (handles HF hub IDs)."""
model_dir = self.model_dir
if os.path.isdir(model_dir):
return model_dir
# HF hub model ID — resolve to local cache
from huggingface_hub import snapshot_download
return snapshot_download(model_dir)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
try:
device = next(self.parameters()).device
except StopIteration:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model_dir = self._resolve_model_dir()
if self.model_stage == "omnivoice_generator":
self.generator.load_weights(model_dir, device)
elif self.model_stage == "omnivoice_decoder":
self.decoder.load_weights(model_dir, device)
else:
raise ValueError(f"{self.model_stage} not supported!")
@@ -0,0 +1,211 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
OmniVoice Decoder (Stage 1) - Audio token to waveform conversion.
Implements the HiggsAudioV2 decode path using transformers' DacModel decoder
and a custom RVQ quantizer, compatible with transformers 4.x.
Decode path:
audio_codes [B, 8, T]
→ RVQ codebook lookup + project_out → sum → [B, 1024, T]
→ fc2 Linear(1024, 256) → [B, 256, T]
→ DAC acoustic decoder (conv transpose upsampling) → [B, 1, T*960]
→ 24kHz waveform (25fps × 960 samples/frame)
"""
from __future__ import annotations
import json
import os
import torch
import torch.nn as nn
from vllm.logger import init_logger
from vllm_omni.model_executor.models.omnivoice.config import OmniVoiceConfig
logger = init_logger(__name__)
class HiggsAudioVQLayer(nn.Module):
"""Single VQ layer: codebook lookup + project_out."""
def __init__(self, codebook_size: int = 1024, codebook_dim: int = 64, hidden_size: int = 1024):
super().__init__()
self.codebook = nn.Embedding(codebook_size, codebook_dim)
self.project_out = nn.Linear(codebook_dim, hidden_size)
def decode(self, indices: torch.Tensor) -> torch.Tensor:
"""indices: [B, T] → [B, hidden_size, T]"""
quantized = self.codebook(indices) # [B, T, codebook_dim]
quantized = self.project_out(quantized) # [B, T, hidden_size]
return quantized.permute(0, 2, 1) # [B, hidden_size, T]
class HiggsAudioRVQ(nn.Module):
"""Residual Vector Quantizer with 8 codebook layers."""
def __init__(
self, num_quantizers: int = 8, codebook_size: int = 1024, codebook_dim: int = 64, hidden_size: int = 1024
):
super().__init__()
self.quantizers = nn.ModuleList(
[HiggsAudioVQLayer(codebook_size, codebook_dim, hidden_size) for _ in range(num_quantizers)]
)
def decode(self, codes: torch.Tensor) -> torch.Tensor:
"""codes: [num_quantizers, B, T] → [B, hidden_size, T]"""
result = torch.zeros(
codes.shape[1],
self.quantizers[0].project_out.out_features,
codes.shape[2],
device=codes.device,
dtype=torch.float32,
)
for i, quantizer in enumerate(self.quantizers):
result = result + quantizer.decode(codes[i])
return result
class OmniVoiceDecoder(nn.Module):
"""OmniVoice Stage 1: Token-to-audio decoder.
Uses DAC acoustic decoder from transformers + custom HiggsAudio RVQ
quantizer to convert 8-codebook tokens into 24kHz waveform.
"""
def __init__(self, config: OmniVoiceConfig):
super().__init__()
self.config = config
self.sample_rate = config.sample_rate
self._loaded = False
# These are populated by load_weights
self.quantizer = None
self.fc2 = None
self.acoustic_decoder = None
@torch.inference_mode()
def forward(self, audio_codes: torch.Tensor) -> torch.Tensor:
"""Decode audio tokens to waveform.
Args:
audio_codes: [B, 8, T] - 8-codebook audio token IDs
Returns:
waveform: [B, 1, audio_samples] at 24kHz
"""
if not self._loaded:
raise RuntimeError("Decoder not loaded. Call load_weights() first.")
device = audio_codes.device
# Transpose: [B, 8, T] → [8, B, T]
codes = audio_codes.transpose(0, 1).long()
# RVQ decode: sum codebook embeddings → [B, 1024, T]
quantized = self.quantizer.decode(codes)
# Project: [B, 1024, T] → fc2 → [B, 256, T]
quantized = self.fc2(quantized.transpose(1, 2)).transpose(1, 2)
# Acoustic decoder: [B, 256, T] → [B, 1, T*960]
audio = self.acoustic_decoder(quantized)
# Ensure [B, 1, samples]
if audio.dim() == 2:
audio = audio.unsqueeze(1)
return audio.to(device)
def _adjust_output_padding(self, decoder: nn.Module):
"""Adjust ConvTranspose1d output_padding (HiggsAudioV2 modification)."""
for module in decoder.modules():
if isinstance(module, nn.ConvTranspose1d):
stride = module.stride[0] if isinstance(module.stride, tuple) else module.stride
module.output_padding = (stride % 2,)
def load_weights(self, model_dir: str, device: torch.device) -> None:
"""Load decoder components from audio_tokenizer/model.safetensors."""
from safetensors.torch import load_file
from transformers import DacConfig, DacModel
audio_tokenizer_path = os.path.join(model_dir, "audio_tokenizer")
config_path = os.path.join(audio_tokenizer_path, "config.json")
weights_path = os.path.join(audio_tokenizer_path, "model.safetensors")
if not os.path.exists(weights_path):
raise FileNotFoundError(f"Audio tokenizer weights not found at {weights_path}")
with open(config_path) as f:
tokenizer_config = json.load(f)
state_dict = load_file(weights_path, device=str(device))
# 1. Build RVQ quantizer
codebook_dim = tokenizer_config.get("codebook_dim", 64)
codebook_size = tokenizer_config.get("codebook_size", 1024)
# Hidden size = quantizer project_out output dim
hidden_size = state_dict["quantizer.quantizers.0.project_out.weight"].shape[0]
num_quantizers = sum(
1 for k in state_dict if k.startswith("quantizer.quantizers.") and k.endswith(".codebook.embed")
)
self.quantizer = HiggsAudioRVQ(
num_quantizers=num_quantizers,
codebook_size=codebook_size,
codebook_dim=codebook_dim,
hidden_size=hidden_size,
).to(device)
# Load quantizer weights
for i in range(num_quantizers):
prefix = f"quantizer.quantizers.{i}"
embed_key = f"{prefix}.codebook.embed"
if embed_key in state_dict:
self.quantizer.quantizers[i].codebook.weight.data.copy_(state_dict[embed_key])
proj_out_w = f"{prefix}.project_out.weight"
proj_out_b = f"{prefix}.project_out.bias"
if proj_out_w in state_dict:
self.quantizer.quantizers[i].project_out.weight.data.copy_(state_dict[proj_out_w])
if proj_out_b in state_dict:
self.quantizer.quantizers[i].project_out.bias.data.copy_(state_dict[proj_out_b])
# 2. Build fc2 projection
fc2_w = state_dict["fc2.weight"]
fc2_b = state_dict["fc2.bias"]
self.fc2 = nn.Linear(fc2_w.shape[1], fc2_w.shape[0]).to(device)
self.fc2.weight.data.copy_(fc2_w)
self.fc2.bias.data.copy_(fc2_b)
# 3. Build DAC acoustic decoder
dac_cfg = DacConfig(**tokenizer_config["acoustic_model_config"])
dac_model = DacModel(dac_cfg)
self.acoustic_decoder = dac_model.decoder.to(device)
# Load acoustic decoder weights
loaded = 0
for name, param in self.acoustic_decoder.named_parameters():
higgs_name = f"acoustic_decoder.{name}"
if higgs_name in state_dict:
param.data.copy_(state_dict[higgs_name])
loaded += 1
# Apply HiggsAudioV2 output padding adjustment
self._adjust_output_padding(self.acoustic_decoder)
# Remove tanh if present (HiggsAudioV2 uses Identity instead)
if hasattr(self.acoustic_decoder, "tanh"):
self.acoustic_decoder.tanh = nn.Identity()
self.acoustic_decoder.eval()
self._loaded = True
logger.info(
"Loaded OmniVoice decoder: %d quantizers, fc2(%d→%d), acoustic decoder (%d weights)",
num_quantizers,
fc2_w.shape[1],
fc2_w.shape[0],
loaded,
)
@@ -0,0 +1,588 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
OmniVoice Generator (Stage 0) - Iterative unmasking with Qwen3 backbone.
Generates 8-codebook audio tokens from text via 32-step non-autoregressive
iterative masked prediction with classifier-free guidance.
Uses vLLM-Omni's DiffusionAttention for optimized full (bidirectional) attention
via FlashAttention/SageAttention/SDPA backends.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.logger import init_logger
from vllm_omni.model_executor.models.omnivoice.config import OmniVoiceConfig
logger = init_logger(__name__)
# ---------------------------------------------------------------------------
# Unmasking schedule helpers
# ---------------------------------------------------------------------------
def _get_time_steps(
t_start: float,
t_end: float,
num_step: int,
t_shift: float,
) -> torch.Tensor:
"""Compute the unmasking schedule with time shift.
Returns cumulative proportions [0, ..., 1] of length num_step.
Formula: r_n = t_shift * (n/N) / (1 + (t_shift - 1) * (n/N))
"""
steps = torch.linspace(t_start, t_end, num_step)
shifted = t_shift * steps / (1.0 + (t_shift - 1.0) * steps)
return shifted
def _gumbel_sample(logits: torch.Tensor, temperature: float) -> torch.Tensor:
"""Add Gumbel noise for stochastic position selection."""
noise = -torch.log(-torch.log(torch.rand_like(logits).clamp(min=1e-8)))
return logits / max(temperature, 1e-8) + noise
# ---------------------------------------------------------------------------
# Qwen3-style transformer blocks using DiffusionAttention
# ---------------------------------------------------------------------------
class OmniVoiceRMSNorm(nn.Module):
def __init__(self, hidden_size: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size))
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
variance = x.to(torch.float32).pow(2).mean(-1, keepdim=True)
x = x * torch.rsqrt(variance + self.eps)
return self.weight * x.to(self.weight.dtype)
class OmniVoiceAttention(nn.Module):
"""Qwen3-style GQA attention using DiffusionAttention backend."""
def __init__(self, config: OmniVoiceConfig):
super().__init__()
self.hidden_size = config.llm_hidden_size
self.num_heads = config.llm_num_attention_heads
self.num_kv_heads = config.llm_num_key_value_heads
self.head_dim = config.llm_head_dim
self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
self.k_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
self.v_proj = nn.Linear(self.hidden_size, self.num_kv_heads * self.head_dim, bias=False)
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
# Qwen3 uses per-head QK norm
self.q_norm = OmniVoiceRMSNorm(self.head_dim)
self.k_norm = OmniVoiceRMSNorm(self.head_dim)
self.scale = 1.0 / math.sqrt(self.head_dim)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
cos: torch.Tensor | None = None,
sin: torch.Tensor | None = None,
) -> torch.Tensor:
batch_size, seq_len, _ = hidden_states.shape
q = self.q_proj(hidden_states)
k = self.k_proj(hidden_states)
v = self.v_proj(hidden_states)
q = q.view(batch_size, seq_len, self.num_heads, self.head_dim)
k = k.view(batch_size, seq_len, self.num_kv_heads, self.head_dim)
v = v.view(batch_size, seq_len, self.num_kv_heads, self.head_dim)
# Per-head QK norm (Qwen3)
q = self.q_norm(q)
k = self.k_norm(k)
# Apply RoPE
if cos is not None and sin is not None:
q = _apply_rotary_pos_emb(q, cos, sin)
k = _apply_rotary_pos_emb(k, cos, sin)
# Expand KV heads for GQA (8 KV heads → 16 Q heads)
if self.num_kv_heads != self.num_heads:
repeat_factor = self.num_heads // self.num_kv_heads
k = k.repeat_interleave(repeat_factor, dim=2)
v = v.repeat_interleave(repeat_factor, dim=2)
# Full bidirectional attention via SDPA with proper mask support
# Permute to (batch, heads, seq, head_dim) for SDPA
q = q.permute(0, 2, 1, 3)
k = k.permute(0, 2, 1, 3)
v = v.permute(0, 2, 1, 3)
# Convert [B, 1, S, S] bool mask to float mask for SDPA
sdpa_mask = None
if attention_mask is not None:
sdpa_mask = attention_mask.to(dtype=q.dtype)
sdpa_mask = sdpa_mask.masked_fill(~attention_mask, float("-inf"))
sdpa_mask = sdpa_mask.masked_fill(attention_mask, 0.0)
out = F.scaled_dot_product_attention(
q,
k,
v,
attn_mask=sdpa_mask,
scale=1.0 / math.sqrt(self.head_dim),
)
# Back to (batch, seq, heads * head_dim)
out = out.permute(0, 2, 1, 3).contiguous()
out = out.view(batch_size, seq_len, self.num_heads * self.head_dim)
return self.o_proj(out)
class OmniVoiceMLP(nn.Module):
"""Qwen3-style MLP with SwiGLU."""
def __init__(self, config: OmniVoiceConfig):
super().__init__()
self.gate_proj = nn.Linear(config.llm_hidden_size, config.llm_intermediate_size, bias=False)
self.up_proj = nn.Linear(config.llm_hidden_size, config.llm_intermediate_size, bias=False)
self.down_proj = nn.Linear(config.llm_intermediate_size, config.llm_hidden_size, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
class OmniVoiceTransformerBlock(nn.Module):
"""Single Qwen3 transformer block with DiffusionAttention."""
def __init__(self, config: OmniVoiceConfig):
super().__init__()
self.input_layernorm = OmniVoiceRMSNorm(config.llm_hidden_size, eps=config.llm_rms_norm_eps)
self.self_attn = OmniVoiceAttention(config)
self.post_attention_layernorm = OmniVoiceRMSNorm(config.llm_hidden_size, eps=config.llm_rms_norm_eps)
self.mlp = OmniVoiceMLP(config)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
cos: torch.Tensor | None = None,
sin: torch.Tensor | None = None,
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(hidden_states, attention_mask=attention_mask, cos=cos, sin=sin)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
# ---------------------------------------------------------------------------
# RoPE
# ---------------------------------------------------------------------------
def _precompute_rope(
head_dim: int,
max_seq_len: int,
theta: float = 1000000.0,
device: torch.device | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Precompute RoPE cos/sin tensors."""
inv_freq = 1.0 / (theta ** (torch.arange(0, head_dim, 2, device=device, dtype=torch.float32) / head_dim))
t = torch.arange(max_seq_len, device=device, dtype=torch.float32)
freqs = torch.outer(t, inv_freq)
cos = freqs.cos()
sin = freqs.sin()
return cos, sin
def _apply_rotary_pos_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
"""Apply rotary position embedding. x shape: (B, S, H, D)."""
seq_len = x.shape[1]
cos = cos[:seq_len].unsqueeze(0).unsqueeze(2) # (1, S, 1, D/2)
sin = sin[:seq_len].unsqueeze(0).unsqueeze(2)
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
rotated = torch.cat([-x2, x1], dim=-1)
return x * torch.cat([cos, cos], dim=-1) + rotated * torch.cat([sin, sin], dim=-1)
# ---------------------------------------------------------------------------
# Generator model
# ---------------------------------------------------------------------------
class OmniVoiceGenerator(nn.Module):
"""OmniVoice Stage 0: Iterative unmasking generator.
Architecture:
- Text embedding (from Qwen3 vocab) + Audio embedding (8*1025 entries)
- 28-layer Qwen3 transformer with full bidirectional attention
- 8-codebook prediction head (single linear: hidden → 8*1025)
- 32-step iterative unmasking with classifier-free guidance
Optimizations:
- DiffusionAttention (FlashAttn/SageAttn/SDPA auto-selected)
- TeaCache / Cache-DiT compatible (hook-based, non-intrusive)
- regionally_compile() compatible for torch.compile on repeated blocks
- Sequence parallelism via SP hooks for multi-GPU
"""
# For regionally_compile() support
_repeated_blocks = ["layers"]
def __init__(self, config: OmniVoiceConfig):
super().__init__()
self.config = config
# Text embedding (shared with LLM)
self.text_embedding = nn.Embedding(config.llm_vocab_size, config.llm_hidden_size)
# Audio embedding: 8 codebooks * 1025 tokens
self.audio_embeddings = nn.Embedding(
config.num_audio_codebook * config.audio_vocab_size,
config.llm_hidden_size,
)
self.register_buffer(
"codebook_layer_offsets",
torch.arange(config.num_audio_codebook) * config.audio_vocab_size,
)
# Transformer layers
self.layers = nn.ModuleList([OmniVoiceTransformerBlock(config) for _ in range(config.llm_num_hidden_layers)])
self.norm = OmniVoiceRMSNorm(config.llm_hidden_size, eps=config.llm_rms_norm_eps)
# Prediction head: hidden → 8 * 1025
self.audio_heads = nn.Linear(
config.llm_hidden_size,
config.num_audio_codebook * config.audio_vocab_size,
bias=False,
)
# Precompute RoPE
self._rope_cos = None
self._rope_sin = None
def _ensure_rope(self, seq_len: int, device: torch.device) -> None:
"""Lazily compute RoPE cos/sin if needed."""
if self._rope_cos is None or self._rope_cos.shape[0] < seq_len:
max_len = max(seq_len, 4096)
self._rope_cos, self._rope_sin = _precompute_rope(
self.config.llm_head_dim,
max_len,
theta=self.config.llm_rope_theta,
device=device,
)
def _prepare_embeddings(
self,
input_ids: torch.Tensor,
audio_mask: torch.Tensor,
) -> torch.Tensor:
"""Prepare mixed text+audio embeddings.
Args:
input_ids: [B, 8, S] - text tokens replicated across codebooks,
audio positions have per-codebook token IDs
audio_mask: [B, S] - True for audio positions, False for text
Returns:
embeddings: [B, S, hidden_size]
"""
# Text embeddings from first codebook row (all rows identical for text)
text_embeds = self.text_embedding(input_ids[:, 0, :])
# Audio embeddings: offset per codebook, then sum across codebooks
shifted_ids = (input_ids * audio_mask.unsqueeze(1)) + self.codebook_layer_offsets.view(1, -1, 1)
audio_embeds = self.audio_embeddings(shifted_ids).sum(dim=1)
# Merge: audio where audio_mask=True, text elsewhere
return torch.where(audio_mask.unsqueeze(-1), audio_embeds, text_embeds)
def _transformer_forward(
self,
inputs_embeds: torch.Tensor,
attention_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run through transformer layers.
Args:
inputs_embeds: [B, S, hidden_size]
attention_mask: [B, 1, S, S] or None
Returns:
hidden_states: [B, S, hidden_size]
"""
device = inputs_embeds.device
seq_len = inputs_embeds.shape[1]
self._ensure_rope(seq_len, device)
hidden_states = inputs_embeds
cos = self._rope_cos.to(device=device, dtype=hidden_states.dtype)
sin = self._rope_sin.to(device=device, dtype=hidden_states.dtype)
for layer in self.layers:
hidden_states = layer(
hidden_states,
attention_mask=attention_mask,
cos=cos,
sin=sin,
)
return self.norm(hidden_states)
def _get_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Project hidden states to per-codebook logits.
Args:
hidden_states: [B, S, hidden_size]
Returns:
logits: [B, 8, S, 1025]
"""
batch_size, seq_len, _ = hidden_states.shape
logits_flat = self.audio_heads(hidden_states) # [B, S, 8*1025]
return logits_flat.view(
batch_size,
seq_len,
self.config.num_audio_codebook,
self.config.audio_vocab_size,
).permute(0, 2, 1, 3) # [B, 8, S, 1025]
@torch.inference_mode()
def forward(
self,
input_ids: torch.Tensor,
audio_mask: torch.Tensor,
attention_mask: torch.Tensor,
target_lens: list[int],
num_step: int = 32,
guidance_scale: float = 2.0,
t_shift: float = 0.1,
layer_penalty_factor: float = 5.0,
position_temperature: float = 5.0,
class_temperature: float = 0.0,
) -> torch.Tensor:
"""Run the full 32-step iterative unmasking generation.
Args:
input_ids: [2*B, 8, S] - conditional (0:B) + unconditional (B:2B)
audio_mask: [2*B, S] - True for audio positions
attention_mask: [2*B, 1, S, S] - attention mask
target_lens: List of target audio lengths per batch item
num_step: Number of unmasking steps
guidance_scale: CFG scale
t_shift: Time shift for schedule
layer_penalty_factor: Penalty for later codebooks
position_temperature: Gumbel temperature for position selection
class_temperature: Temperature for token prediction (0=greedy)
Returns:
tokens: [B, 8, max_target_len] - generated audio tokens
"""
B = len(target_lens)
device = input_ids.device
max_target_len = max(target_lens)
mask_id = self.config.audio_mask_id
num_codebooks = self.config.num_audio_codebook
# Initialize all target tokens as [MASK]
tokens = torch.full(
(B, num_codebooks, max_target_len),
mask_id,
dtype=torch.long,
device=device,
)
# Compute unmasking schedule
timesteps = _get_time_steps(0.0, 1.0, num_step + 1, t_shift).tolist()
schedules = []
for t_len in target_lens:
total_mask = t_len * num_codebooks
rem = total_mask
sched = []
for step in range(num_step):
num = (
rem
if step == num_step - 1
else min(
math.ceil(total_mask * (timesteps[step + 1] - timesteps[step])),
rem,
)
)
sched.append(int(num))
rem -= int(num)
schedules.append(sched)
layer_ids = torch.arange(num_codebooks, device=device).view(1, -1, 1)
# Compute c_lens for extracting target region from full sequence
c_lens = []
for i in range(B):
# Conditional sequence length = number of non-padding positions
c_len = attention_mask[i, 0, 0].sum().item()
c_lens.append(int(c_len))
# Main iterative loop
for step in range(num_step):
# Prepare embeddings and run transformer
inputs_embeds = self._prepare_embeddings(input_ids, audio_mask)
hidden_states = self._transformer_forward(inputs_embeds, attention_mask)
batch_logits = self._get_logits(hidden_states).to(torch.float32)
# batch_logits: [2*B, 8, S, 1025]
for i in range(B):
k = schedules[i][step]
if k <= 0:
continue
c_len = c_lens[i]
t_len = target_lens[i]
# Extract logits for target region
c_logits = batch_logits[i : i + 1, :, c_len - t_len : c_len, :] # [1, 8, T, 1025]
u_logits = batch_logits[B + i : B + i + 1, :, :t_len, :] # [1, 8, T, 1025]
# Classifier-free guidance
if guidance_scale != 0:
c_log_probs = F.log_softmax(c_logits, dim=-1)
u_log_probs = F.log_softmax(u_logits, dim=-1)
log_probs = torch.log_softmax(
c_log_probs + guidance_scale * (c_log_probs - u_log_probs),
dim=-1,
)
else:
log_probs = F.log_softmax(c_logits, dim=-1)
# Prevent predicting [MASK]
log_probs[..., mask_id] = -float("inf")
# Token prediction
if class_temperature > 0.0:
pred_tokens = _gumbel_sample(log_probs, class_temperature).argmax(dim=-1)
else:
pred_tokens = log_probs.argmax(dim=-1) # [1, 8, T]
# Confidence scores
scores = log_probs.max(dim=-1)[0] # [1, 8, T]
# Layer penalty (earlier codebooks get higher priority)
scores = scores - (layer_ids * layer_penalty_factor)
# Gumbel noise for position selection
if position_temperature > 0.0:
scores = _gumbel_sample(scores, position_temperature)
# Mask out already unmasked positions
sample_tokens = tokens[i : i + 1, :, :t_len]
scores.masked_fill_(sample_tokens != mask_id, -float("inf"))
# Select top-k positions to unmask
_, topk_idx = torch.topk(scores.flatten(), k)
flat_tokens = sample_tokens.flatten().clone()
flat_tokens[topk_idx] = pred_tokens.flatten()[topk_idx]
sample_tokens.copy_(flat_tokens.view_as(sample_tokens))
# Update tokens and batch inputs for next iteration
tokens[i : i + 1, :, :t_len] = sample_tokens
input_ids = input_ids.clone()
input_ids[i, :, c_len - t_len : c_len] = sample_tokens.squeeze(0)
input_ids[B + i, :, :t_len] = sample_tokens.squeeze(0)
return tokens
def load_weights(self, model_dir: str, device: torch.device) -> None:
"""Load weights from HuggingFace OmniVoice model.safetensors.
The HF checkpoint contains:
- llm.* -> Qwen3 transformer weights
- audio_embeddings.* -> audio embedding table
- audio_heads.* -> prediction head
"""
import os
from safetensors.torch import load_file
weights_path = os.path.join(model_dir, "model.safetensors")
if not os.path.exists(weights_path):
raise FileNotFoundError(f"Model weights not found at {weights_path}")
state_dict = load_file(weights_path, device=str(device))
# Map HF weight names to our module names
loaded_keys = set()
# 1. Text embedding: llm.embed_tokens.weight -> text_embedding.weight
text_emb_key = "llm.embed_tokens.weight"
if text_emb_key in state_dict:
self.text_embedding.weight.data.copy_(state_dict[text_emb_key])
loaded_keys.add(text_emb_key)
# 2. Audio embeddings
for key in ["audio_embeddings.weight"]:
if key in state_dict:
self.audio_embeddings.weight.data.copy_(state_dict[key])
loaded_keys.add(key)
# 3. Audio heads
for key in ["audio_heads.weight"]:
if key in state_dict:
self.audio_heads.weight.data.copy_(state_dict[key])
loaded_keys.add(key)
# 4. Transformer layers: llm.layers.N.* -> layers.N.*
for key, value in state_dict.items():
if key.startswith("llm.layers."):
# llm.layers.0.self_attn.q_proj.weight -> layers.0.self_attn.q_proj.weight
our_key = key.replace("llm.layers.", "layers.")
parts = our_key.split(".")
module = self
try:
for part in parts[:-1]:
if part.isdigit():
module = module[int(part)]
else:
module = getattr(module, part)
param_name = parts[-1]
param = getattr(module, param_name)
if isinstance(param, nn.Parameter):
param.data.copy_(value)
elif isinstance(param, torch.Tensor):
param.copy_(value)
loaded_keys.add(key)
except (AttributeError, IndexError, KeyError) as e:
logger.warning("Failed to load weight %s: %s", key, e)
# 5. Final norm: llm.norm.weight -> norm.weight
norm_key = "llm.norm.weight"
if norm_key in state_dict:
self.norm.weight.data.copy_(state_dict[norm_key])
loaded_keys.add(norm_key)
unloaded = set(state_dict.keys()) - loaded_keys
# Filter out audio_tokenizer weights (loaded in decoder stage)
unloaded = {k for k in unloaded if not k.startswith("audio_tokenizer.")}
if unloaded:
logger.info(
"Generator: %d/%d weights loaded, %d skipped (decoder weights)",
len(loaded_keys),
len(state_dict),
len(unloaded),
)
else:
logger.info("Generator: all %d weights loaded", len(loaded_keys))
@@ -57,6 +57,11 @@ _OMNI_MODELS = {
"cosyvoice3",
"CosyVoice3Model",
),
"OmniVoiceModel": (
"omnivoice",
"omnivoice",
"OmniVoiceModel",
),
"MammothModa2Qwen2ForCausalLM": (
"mammoth_moda2",
"mammoth_moda2",