Merge branch 'main' into dev/vllm-align

This commit is contained in:
tzhouam committed 2026-03-13 06:51:22 +00:00
commit d3db476f2d
144 files changed
+17247 -1024

No files matched your search

Whitespace-only changes.
@@ -0,0 +1,154 @@
"""Configuration classes for Fish Speech S2 Pro (fish_qwen3_omni).
The HuggingFace config uses field names from the original Fish Speech codebase
(dim, n_head, n_layer, etc.). This module re-exports them under the standard
Transformers/Qwen3 attribute names so that vLLM's ``Qwen3Model`` can consume
the Slow AR config transparently.
"""
from __future__ import annotations
from transformers import PretrainedConfig
class FishSpeechSlowARConfig(PretrainedConfig):
"""Slow AR (text_model) config -- Qwen3-based transformer.
Maps Fish Speech field names to Qwen3-compatible attribute names so
``vllm.model_executor.models.qwen3.Qwen3Model`` works out of the box.
"""
model_type = "fish_qwen3"
def __init__(
self,
vocab_size: int = 155776,
dim: int = 2560,
n_head: int = 32,
n_local_heads: int = 8,
head_dim: int = 128,
n_layer: int = 36,
intermediate_size: int = 9728,
attention_qk_norm: bool = True,
rope_base: float = 1_000_000.0,
max_seq_len: int = 32768,
tie_word_embeddings: bool = True,
codebook_size: int = 4096,
num_codebooks: int = 10,
semantic_begin_id: int = 0,
semantic_end_id: int = 0,
scale_codebook_embeddings: bool = False,
rms_norm_eps: float = 1e-6,
**kwargs,
):
# Fish Speech field names → standard Transformers/Qwen3 names.
self.hidden_size = dim
self.num_attention_heads = n_head
self.num_key_value_heads = n_local_heads
self.head_dim = head_dim
self.num_hidden_layers = n_layer
self.intermediate_size = intermediate_size
self.max_position_embeddings = max_seq_len
self.rope_theta = rope_base
self.rms_norm_eps = rms_norm_eps
self.hidden_act = "silu"
self.attention_bias = False
self.attention_qk_norm = attention_qk_norm
# Fish Speech codec / codebook fields.
self.codebook_size = codebook_size
self.num_codebooks = num_codebooks
self.semantic_begin_id = semantic_begin_id
self.semantic_end_id = semantic_end_id
self.scale_codebook_embeddings = scale_codebook_embeddings
super().__init__(
vocab_size=vocab_size,
tie_word_embeddings=tie_word_embeddings,
**kwargs,
)
class FishSpeechFastARConfig(PretrainedConfig):
"""Fast AR (audio_decoder) config -- 4-layer residual codebook predictor."""
model_type = "fish_qwen3_audio_decoder"
def __init__(
self,
vocab_size: int = 4096,
num_codebooks: int = 10,
dim: int = 2560,
n_head: int = 32,
n_local_heads: int = 8,
head_dim: int = 128,
n_layer: int = 4,
intermediate_size: int = 9728,
max_seq_len: int = 11,
text_dim: int = 2560,
audio_hidden_dim: int = 5120,
attention_qk_norm: bool = False,
rope_base: float = 1_000_000.0,
rms_norm_eps: float = 1e-6,
**kwargs,
):
self.hidden_size = dim
self.num_attention_heads = n_head
self.num_key_value_heads = n_local_heads
self.head_dim = head_dim
self.num_hidden_layers = n_layer
self.intermediate_size = intermediate_size
self.max_position_embeddings = max_seq_len
self.rope_theta = rope_base
self.rms_norm_eps = rms_norm_eps
self.hidden_act = "silu"
self.attention_bias = False
self.attention_qk_norm = attention_qk_norm
self.num_codebooks = num_codebooks
self.text_dim = text_dim
self.audio_hidden_dim = audio_hidden_dim
super().__init__(vocab_size=vocab_size, **kwargs)
class FishSpeechConfig(PretrainedConfig):
"""Top-level config for Fish Speech S2 Pro (fish_qwen3_omni).
Wraps ``text_config`` (Slow AR) and ``audio_decoder_config`` (Fast AR).
"""
model_type = "fish_qwen3_omni"
sub_configs = {
"text_config": FishSpeechSlowARConfig,
"audio_decoder_config": FishSpeechFastARConfig,
}
def __init__(
self,
text_config: dict | FishSpeechSlowARConfig | None = None,
audio_decoder_config: dict | FishSpeechFastARConfig | None = None,
semantic_start_token_id: int = 151678,
semantic_end_token_id: int = 155773,
audio_pad_token_id: int = 151677,
**kwargs,
):
if isinstance(text_config, dict):
text_config = FishSpeechSlowARConfig(**text_config)
if isinstance(audio_decoder_config, dict):
audio_decoder_config = FishSpeechFastARConfig(**audio_decoder_config)
self.text_config = text_config or FishSpeechSlowARConfig()
self.audio_decoder_config = audio_decoder_config or FishSpeechFastARConfig()
self.semantic_start_token_id = semantic_start_token_id
self.semantic_end_token_id = semantic_end_token_id
self.audio_pad_token_id = audio_pad_token_id
# Propagate semantic IDs into text_config for convenience.
self.text_config.semantic_begin_id = semantic_start_token_id
self.text_config.semantic_end_id = semantic_end_token_id
super().__init__(**kwargs)
def get_text_config(self, **kwargs) -> FishSpeechSlowARConfig:
return self.text_config
@@ -0,0 +1,111 @@
"""DAC codec encoder for Fish Speech S2 Pro voice cloning.
Encodes reference audio into VQ codes for use as prompt conditioning.
Runs on CPU in the API server process -- loaded lazily on first use.
"""
from __future__ import annotations
import os
import numpy as np
import torch
import torch.nn as nn
from vllm.logger import init_logger
from vllm_omni.model_executor.models.fish_speech.dac_utils import (
DAC_SAMPLE_RATE,
build_dac_codec,
)
logger = init_logger(__name__)
_codec_cache: dict[str, nn.Module] = {}
def _load_dac_codec(model_path: str) -> nn.Module:
"""Load the DAC codec model from codec.pth (cached per model_path)."""
if model_path in _codec_cache:
return _codec_cache[model_path]
codec_path = os.path.join(model_path, "codec.pth")
if not os.path.exists(codec_path):
from transformers.utils.hub import cached_file
cached = cached_file(model_path, "codec.pth")
if cached is not None:
codec_path = cached
if not os.path.exists(codec_path):
raise FileNotFoundError(
f"codec.pth not found for {model_path}. Required for voice cloning with Fish Speech S2 Pro."
)
codec = build_dac_codec()
state_dict = torch.load(codec_path, map_location="cpu", weights_only=True)
if "generator" in state_dict:
state_dict = state_dict["generator"]
codec.load_state_dict(state_dict, strict=False)
codec.eval()
_codec_cache[model_path] = codec
logger.info("Loaded DAC codec encoder from %s (CPU)", codec_path)
return codec
def _resample(wav: np.ndarray, sr: int, target_sr: int) -> np.ndarray:
"""Resample audio using torchaudio's polyphase resampling."""
if sr == target_sr:
return wav
import torchaudio
wav_t = torch.from_numpy(wav).unsqueeze(0).float()
wav_t = torchaudio.functional.resample(wav_t, sr, target_sr)
return wav_t.squeeze(0).numpy()
@torch.no_grad()
def encode_reference_audio(
model_path: str,
wav_samples: list[float] | np.ndarray,
sample_rate: int,
) -> list[int]:
"""Encode reference audio into semantic token IDs for prompt conditioning.
Args:
model_path: HuggingFace model path (for locating codec.pth).
wav_samples: Audio waveform samples (mono, float).
sample_rate: Sample rate of the input audio.
Returns:
List of semantic token IDs (151678 + code_value for each frame).
"""
codec = _load_dac_codec(model_path)
wav = np.asarray(wav_samples, dtype=np.float32)
if wav.ndim > 1:
wav = np.mean(wav, axis=-1)
# Resample to DAC sample rate (44100).
wav = _resample(wav, sample_rate, DAC_SAMPLE_RATE)
# Encode: [1, 1, T] -> codes [1, num_codebooks, num_frames]
wav_tensor = torch.from_numpy(wav).unsqueeze(0).unsqueeze(0).float()
feature_lengths = torch.tensor([wav_tensor.shape[-1]])
codes, feature_lengths_out = codec.encode(wav_tensor, feature_lengths)
# Extract semantic codebook (index 0) - shape [num_frames].
semantic_codes = codes[0, 0, :].tolist()
# Convert to semantic token IDs: <|semantic:{i}|> = 151678 + i
SEMANTIC_TOKEN_OFFSET = 151678
semantic_token_ids = [SEMANTIC_TOKEN_OFFSET + int(c) for c in semantic_codes]
logger.info(
"Encoded reference audio: %d samples @ %dHz -> %d semantic tokens",
len(wav_samples),
sample_rate,
len(semantic_token_ids),
)
return semantic_token_ids
@@ -0,0 +1,92 @@
"""Shared DAC codec construction for Fish Speech S2 Pro.
Used by both the encoder (voice cloning, CPU) and the decoder (Stage 1, GPU).
"""
from __future__ import annotations
import torch.nn as nn
# Default DAC codec config matching s2-pro's modded_dac_vq.yaml.
DAC_SAMPLE_RATE = 44100
DAC_HOP_LENGTH = 2048 # 512 (decoder upsample) * 4 (quantizer upsample)
DAC_NUM_CODEBOOKS = 10 # 1 semantic + 9 residual
def build_dac_codec() -> nn.Module:
"""Construct a DAC codec model (uninitialized weights).
Returns the model on CPU in eval mode -- caller is responsible for
loading weights and moving to the target device.
"""
from fish_speech.models.dac.modded_dac import (
DAC,
ModelArgs,
WindowLimitedTransformer,
)
from fish_speech.models.dac.rvq import DownsampleResidualVectorQuantize
base_transformer_kwargs = dict(
block_size=16384,
n_local_heads=-1,
head_dim=64,
rope_base=10000,
norm_eps=1e-5,
dropout_rate=0.0,
attn_dropout_rate=0.0,
channels_first=True,
)
def _make_transformer_config(**kw):
return ModelArgs(**{**base_transformer_kwargs, **kw})
quantizer_transformer_config = ModelArgs(
block_size=4096,
n_layer=8,
n_head=16,
dim=1024,
intermediate_size=3072,
n_local_heads=-1,
head_dim=64,
rope_base=10000,
norm_eps=1e-5,
dropout_rate=0.0,
attn_dropout_rate=0.0,
channels_first=True,
)
post_module = WindowLimitedTransformer(
causal=True,
window_size=128,
input_dim=1024,
config=quantizer_transformer_config,
)
pre_module = WindowLimitedTransformer(
causal=True,
window_size=128,
input_dim=1024,
config=quantizer_transformer_config,
)
quantizer = DownsampleResidualVectorQuantize(
input_dim=1024,
n_codebooks=9,
codebook_size=1024,
codebook_dim=8,
quantizer_dropout=0.0,
downsample_factor=[2, 2],
post_module=post_module,
pre_module=pre_module,
semantic_codebook_size=4096,
)
codec = DAC(
encoder_dim=64,
encoder_rates=[2, 4, 8, 8],
decoder_dim=1536,
decoder_rates=[8, 8, 4, 2],
quantizer=quantizer,
sample_rate=DAC_SAMPLE_RATE,
causal=True,
encoder_transformer_layers=[0, 0, 0, 4],
decoder_transformer_layers=[4, 0, 0, 0],
transformer_general_config=_make_transformer_config,
)
return codec
@@ -0,0 +1,272 @@
"""Fish Speech S2 Pro -- DAC Decoder (Stage 1).
Loads the DAC codec from ``codec.pth`` and decodes codebook indices
[num_codebooks, T] → audio waveform at 44.1 kHz.
Analogous to ``Qwen3TTSCode2Wav`` in qwen3_tts.
Requires the ``fish-speech`` package for the DAC model architecture.
Install with: ``pip install fish-speech``
"""
from __future__ import annotations
import os
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm_omni.model_executor.models.fish_speech.dac_utils import (
DAC_HOP_LENGTH,
DAC_NUM_CODEBOOKS,
DAC_SAMPLE_RATE,
build_dac_codec,
)
from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
class FishSpeechDACDecoder(nn.Module):
"""Stage-1 DAC decoder for Fish Speech S2 Pro (GenerationModelRunner).
Consumes frame-aligned codec tokens from input_ids and decodes waveform
via the DAC codec decoder.
"""
input_modalities = "audio"
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.enable_update_additional_information = True
self.requires_raw_input_tokens = True
self._codec: nn.Module | None = None
self._num_codebooks: int = DAC_NUM_CODEBOOKS
self._output_sample_rate: int = DAC_SAMPLE_RATE
self._hop_length: int = DAC_HOP_LENGTH
self._logged_codec_stats = False
def _ensure_codec_loaded(self) -> None:
if self._codec is not None:
return
codec_path = os.path.join(self.model_path, "codec.pth")
if not os.path.exists(codec_path):
# Try HuggingFace cache.
try:
from transformers.utils.hub import cached_file
cached = cached_file(self.model_path, "codec.pth")
if cached is not None:
codec_path = cached
except Exception:
pass
if not os.path.exists(codec_path):
raise FileNotFoundError(
f"codec.pth not found at {codec_path}. Make sure the Fish Speech S2 Pro model includes codec.pth."
)
codec = build_dac_codec()
# Load weights.
state_dict = torch.load(codec_path, map_location="cpu", weights_only=True)
# Some checkpoints wrap under "generator" key.
if "generator" in state_dict:
state_dict = state_dict["generator"]
codec.load_state_dict(state_dict, strict=False)
device = self.vllm_config.device_config.device
codec = codec.to(device=device, dtype=torch.float32)
codec.eval()
self._codec = codec
logger.info(
"Fish Speech DAC codec loaded from %s (device=%s, sample_rate=%d)",
codec_path,
device,
self._output_sample_rate,
)
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
if input_ids.numel() == 0:
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
return None
def _split_request_ids(
self,
ids: torch.Tensor,
seq_token_counts: list[int] | None = None,
) -> list[torch.Tensor]:
if seq_token_counts is not None and len(seq_token_counts) > 1:
boundaries = [0]
for count in seq_token_counts:
boundaries.append(boundaries[-1] + count)
n = ids.numel()
return [ids[boundaries[i] : min(boundaries[i + 1], n)] for i in range(len(seq_token_counts))]
if is_forward_context_available():
slices = get_forward_context().ubatch_slices
if slices is not None and len(slices) > 1 and not any(hasattr(s, "token_slice") for s in slices):
boundaries = [0]
for s in slices:
boundaries.append(boundaries[-1] + s)
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
return [ids]
@torch.no_grad()
def forward(
self,
input_ids: torch.Tensor | None = None,
positions: torch.Tensor | None = None,
intermediate_tensors: Any = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict[str, Any]] | None = None,
**kwargs: Any,
) -> OmniOutput:
"""Decode codec codes into audio waveform.
input_ids layout per request: flat codes [num_codebooks * num_frames].
Codes are codebook-major: [cb0_f0, cb0_f1, ..., cb0_fN, cb1_f0, ...].
"""
self._ensure_codec_loaded()
assert self._codec is not None
q = self._num_codebooks
sr_val = self._output_sample_rate
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
empty = torch.zeros((0,), dtype=torch.float32)
if input_ids is None or input_ids.numel() == 0:
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
)
ids = input_ids.reshape(-1).to(dtype=torch.long)
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
parsed: list[tuple[int, int]] = []
valid_codes_qf: list[torch.Tensor] = []
valid_indices: list[int] = []
left_context_size = [0] * len(request_ids_list)
if runtime_additional_information is not None:
for i, info in enumerate(runtime_additional_information):
if i >= len(left_context_size):
break
if "left_context_size" in info:
left_context_size[i] = info["left_context_size"]
for i, req_ids in enumerate(request_ids_list):
if req_ids.numel() < 1:
parsed.append((0, 0))
continue
ctx_frames = left_context_size[i]
flat = req_ids
n = flat.numel()
if n == 0 or n % q != 0:
if n > 0:
logger.warning(
"DAC decoder input_ids length %d not divisible by num_codebooks %d; returning empty audio.",
n,
q,
)
parsed.append((0, 0))
continue
frames = n // q
codes_qf = flat.reshape(q, frames)
parsed.append((ctx_frames, frames))
valid_codes_qf.append(codes_qf)
valid_indices.append(i)
num_req = len(request_ids_list)
if not valid_codes_qf:
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={
"model_outputs": [empty] * num_req,
"sr": [sr_tensor] * num_req,
},
)
if not self._logged_codec_stats:
self._logged_codec_stats = True
try:
c = valid_codes_qf[0]
logger.info(
"DAC decoder: frames=%d q=%d uniq=%d range=[%d,%d] batch=%d",
c.shape[1],
q,
int(torch.unique(c).numel()),
int(c.min().item()),
int(c.max().item()),
len(valid_codes_qf),
)
except Exception:
pass
# Decode each request individually.
wav_tensors: list[torch.Tensor] = []
for codes_qf in valid_codes_qf:
codes_bqf = codes_qf.unsqueeze(0) # [1, num_codebooks, num_frames]
num_frames = codes_qf.shape[1]
feature_lengths = torch.tensor([num_frames], device=codes_bqf.device)
with torch.cuda.amp.autocast(enabled=False):
wav, audio_lengths = self._codec.decode(codes_bqf, feature_lengths)
# wav shape: [1, 1, wav_len]
wav_tensors.append(wav.squeeze(0).squeeze(0)) # [wav_len]
audios: list[torch.Tensor] = [empty] * num_req
srs = [sr_tensor] * num_req
for j, idx in enumerate(valid_indices):
ctx_frames, actual_frames = parsed[idx]
wav = wav_tensors[j]
# Trim context frames (left overlap for streaming).
if ctx_frames > 0:
cut = ctx_frames * self._hop_length
if cut < wav.shape[0]:
wav = wav[cut:]
else:
logger.warning(
"Context trim %d >= decoded length %d; returning empty audio.",
cut,
wav.shape[0],
)
continue
if wav.shape[0] > 0:
audios[idx] = wav.to(dtype=torch.float32).reshape(-1)
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audios, "sr": srs},
)
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
if not (isinstance(model_outputs, tuple) and len(model_outputs) == 2):
raise TypeError(f"FishSpeechDACDecoder expected (audio_tensor, sr), got {type(model_outputs)}")
audio_tensor, sr = model_outputs
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"model_outputs": audio_tensor, "sr": sr},
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# DAC codec weights are loaded lazily from codec.pth, not from the main checkpoint.
return set()
@@ -0,0 +1,470 @@
"""Fast AR (audio_decoder) for Fish Speech S2 Pro.
4-layer transformer that predicts residual codebook codes (1..num_codebooks-1)
autoregressively after each Slow AR step. Analogous to Qwen3 TTS's
CodePredictor but with its own embedding table, RoPE, and output head.
Uses re-prefill (no KV cache): each AR step forwards the full growing
sequence through the 4-layer transformer. This trades ~O(T^2) attention
FLOPs (negligible for T=10, 4 layers) for zero KV cache management.
Optimisations:
- torch.compile on model forward (kernel fusion)
- Pre-allocated projection / embedding buffer
- Pre-allocated position_ids
- Inline sampling
"""
from __future__ import annotations
from collections.abc import Iterable
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import is_pp_missing_parameter
from .configuration_fish_speech import FishSpeechFastARConfig, FishSpeechSlowARConfig
logger = init_logger(__name__)
# ===================================================================
# Standalone Fast AR Layers (no vLLM paged attention)
# ===================================================================
class _FastARAttention(nn.Module):
"""Multi-head attention using F.scaled_dot_product_attention (SDPA).
Supports fused QKV, RoPE, optional q/k normalization, and native GQA.
Input: [B, seq_len, hidden_size].
"""
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
super().__init__()
self.hidden_size = config.hidden_size
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.head_dim = config.head_dim
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self._use_gqa = self.num_kv_heads != self.num_heads
self.qkv_proj = QKVParallelLinear(
hidden_size=self.hidden_size,
head_size=self.head_dim,
total_num_heads=self.num_heads,
total_num_kv_heads=self.num_kv_heads,
bias=False,
prefix=f"{prefix}.qkv_proj",
disable_tp=True,
)
self.o_proj = RowParallelLinear(
input_size=self.num_heads * self.head_dim,
output_size=self.hidden_size,
bias=False,
prefix=f"{prefix}.o_proj",
disable_tp=True,
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=config.max_position_embeddings,
is_neox_style=False, # Fish Speech uses interleaved (GPT-J) RoPE
rope_parameters={"rope_theta": config.rope_theta, "rope_type": "default"},
)
# Fish Speech Fast AR typically has attention_qk_norm=false.
if config.attention_qk_norm:
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
else:
self.q_norm = None
self.k_norm = None
def forward(self, hidden_states: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
qkv, _ = self.qkv_proj(hidden_states.reshape(bsz * seq_len, -1))
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
if self.q_norm is not None:
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
q, k = self.rotary_emb(position_ids, q, k)
q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
k = k.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
v = v.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
attn_out = F.scaled_dot_product_attention(
q,
k,
v,
scale=self.scaling,
is_causal=True,
enable_gqa=self._use_gqa,
)
attn_out = attn_out.transpose(1, 2).reshape(bsz * seq_len, -1)
output, _ = self.o_proj(attn_out)
return output.view(bsz, seq_len, -1)
class _FastARMLP(nn.Module):
"""SiLU-gated MLP, matching Qwen3/LLaMA MLP structure."""
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
input_size=config.hidden_size,
output_sizes=[config.intermediate_size] * 2,
bias=False,
prefix=f"{prefix}.gate_up_proj",
disable_tp=True,
)
self.down_proj = RowParallelLinear(
input_size=config.intermediate_size,
output_size=config.hidden_size,
bias=False,
prefix=f"{prefix}.down_proj",
disable_tp=True,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
gate, up = gate_up.chunk(2, dim=-1)
x = F.silu(gate) * up
x, _ = self.down_proj(x)
return x
class _FastARDecoderLayer(nn.Module):
"""Transformer decoder layer for Fast AR (SDPA, no KV cache)."""
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
super().__init__()
self.self_attn = _FastARAttention(config, prefix=f"{prefix}.self_attn")
self.mlp = _FastARMLP(config, prefix=f"{prefix}.mlp")
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(self, hidden_states: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(hidden_states, position_ids)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
# ===================================================================
# Fast AR Transformer Model
# ===================================================================
class FishSpeechFastARModel(nn.Module):
"""4-layer transformer for residual codebook prediction (re-prefill)."""
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
super().__init__()
self.config = config
self.layers = nn.ModuleList(
[_FastARDecoderLayer(config, prefix=f"{prefix}.layers.{i}") for i in range(config.num_hidden_layers)]
)
# NOTE: final norm is handled by FishSpeechFastAR.fast_norm (one norm weight
# in checkpoint: audio_decoder.norm.weight → fast_ar.fast_norm.weight).
def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
hidden_states = inputs_embeds
for layer in self.layers:
hidden_states = layer(hidden_states, position_ids)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped = name.replace(weight_name, param_name)
if mapped.endswith(".bias") and mapped not in params_dict:
continue
if is_pp_missing_parameter(mapped, self):
continue
if mapped.endswith("scale"):
mapped = maybe_remap_kv_scale_name(mapped, params_dict)
if mapped is None:
continue
param = params_dict.get(mapped)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(mapped)
break
else:
mapped = maybe_remap_kv_scale_name(name, params_dict)
if mapped is None:
continue
if name.endswith(".bias") and mapped not in params_dict:
continue
if is_pp_missing_parameter(mapped, self):
continue
param = params_dict.get(mapped)
if param is None:
continue
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(mapped)
return loaded_params
# ===================================================================
# Fast AR Wrapper (optimised re-prefill + torch.compile)
# ===================================================================
class FishSpeechFastAR(nn.Module):
"""vLLM-native Fast AR for Fish Speech (residual codebooks).
Re-prefill approach: each AR step forwards the full growing sequence
through the 4-layer transformer. No KV cache needed.
Optimisations over baseline:
1. torch.compile on model forward -- kernel fusion.
2. Pre-allocated embedding buffer [B, max_seq, H].
3. Pre-allocated position_ids.
4. Inline sampling.
"""
def __init__(
self,
*,
vllm_config: VllmConfig,
config: FishSpeechFastARConfig,
slow_ar_config: FishSpeechSlowARConfig,
prefix: str = "fast_ar",
) -> None:
super().__init__()
self._vllm_config = vllm_config
self.config = config
self.slow_ar_config = slow_ar_config
self.model = FishSpeechFastARModel(config, prefix=f"{prefix}.model")
# Codebook embeddings for Fast AR (separate from Slow AR's codebook_embeddings).
self.fast_embeddings = nn.Embedding(config.vocab_size, config.hidden_size)
# Output projection for residual codes.
self.fast_output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
# Layer norm for Fast AR output.
self.fast_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
# Projection from Slow AR hidden dim → Fast AR hidden dim.
# s2-pro has same dim for both (2560) and no fast_project_in in checkpoint,
# so use identity when dims match.
if slow_ar_config.hidden_size != config.hidden_size:
self.fast_project_in = nn.Linear(slow_ar_config.hidden_size, config.hidden_size, bias=True)
else:
self.fast_project_in = nn.Identity()
self._num_codebooks = config.num_codebooks
self._fast_dim = config.hidden_size
# Pre-allocated buffers (lazily initialised on first forward).
self._embed_buf: torch.Tensor | None = None
self._pos_ids: torch.Tensor | None = None
self._compiled_model_fwd: object | None = None
def _ensure_buffers(self, bsz: int, device: torch.device, dtype: torch.dtype) -> None:
max_seq = self._num_codebooks + 1 # hidden_state + num_codebooks codes
if (
self._embed_buf is not None
and self._embed_buf.shape[0] >= bsz
and self._embed_buf.device == device
and self._embed_buf.dtype == dtype
):
return
self._embed_buf = torch.zeros(bsz, max_seq, self._fast_dim, dtype=dtype, device=device)
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device)
def _setup_compile(self) -> None:
if self._compiled_model_fwd is not None:
return
# TODO: Enable torch.compile for performance. Eager for now to avoid
# potential graph-break issues during initial bring-up.
self._compiled_model_fwd = self.model.forward
@torch.inference_mode()
def forward(
self,
slow_ar_hidden: torch.Tensor,
semantic_token_id: torch.Tensor,
*,
do_sample: bool = True,
temperature: float = 0.8,
top_k: int = 30,
top_p: float = 0.9,
) -> torch.Tensor:
"""Predict residual codebook codes 0..num_codebooks-1 autoregressively.
Args:
slow_ar_hidden: [B, hidden_size] last hidden state from Slow AR.
semantic_token_id: [B] or [B, 1] sampled semantic token IDs (in vocab space).
Returns:
all_codes: [B, num_codebooks] codes for codebooks 0..num_codebooks-1.
Code 0 is the semantic code (token - semantic_begin_id).
"""
bsz = int(slow_ar_hidden.shape[0])
num_cb = self._num_codebooks
device = slow_ar_hidden.device
dtype = slow_ar_hidden.dtype
semantic_begin = self.slow_ar_config.semantic_begin_id
semantic_end = self.slow_ar_config.semantic_end_id
codebook_size = semantic_end - semantic_begin + 1 # 4096
# Convert vocab-space semantic token to codebook index.
# Clamp to valid range: im_end or other non-semantic tokens map to 0 (pad).
semantic_code = (semantic_token_id.reshape(bsz) - semantic_begin).clamp(min=0, max=codebook_size - 1)
all_codes = torch.empty(bsz, num_cb, dtype=torch.long, device=device)
all_codes[:, 0] = semantic_code
self._ensure_buffers(bsz, device, dtype)
self._setup_compile()
embed_buf = self._embed_buf
pos_ids = self._pos_ids
model_fwd = self._compiled_model_fwd
# Position 0: projected Slow AR hidden state.
projected = self.fast_project_in(slow_ar_hidden.reshape(bsz, -1))
embed_buf[:bsz, 0, :] = projected
# Position 1: embedding of semantic code.
code_embed = self.fast_embeddings(semantic_code.clamp(min=0))
embed_buf[:bsz, 1, :] = code_embed
use_sampling = do_sample and temperature > 0
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
# Residual codebook size (1024) vs semantic codebook size (4096).
# The fast_output head has codebook_size (4096) outputs, but residual
# codebooks only have 1024 entries. Truncate logits for steps > 0.
residual_codebook_size = 1024
for step in range(1, num_cb):
seq_len = step + 1
step_input = embed_buf[:bsz, :seq_len, :]
step_pos_ids = pos_ids[:seq_len] if bsz == 1 else pos_ids[:seq_len].repeat(bsz)
hidden_out = model_fwd(step_input, step_pos_ids)
logits = self.fast_output(self.fast_norm(hidden_out[:, -1, :]))
# Residual codebooks (step >= 1) only have 1024 entries.
if step >= 1:
logits = logits[:, :residual_codebook_size]
if use_sampling:
scaled = logits * inv_temperature
if top_k > 0:
topk_vals, _ = scaled.topk(min(top_k, scaled.shape[-1]), dim=-1)
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
if top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(scaled, descending=True)
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
sorted_indices_to_remove = cumulative_probs - F.softmax(sorted_logits, dim=-1) >= top_p
sorted_logits[sorted_indices_to_remove] = float("-inf")
scaled = sorted_logits.scatter(1, sorted_indices, sorted_logits)
probs = F.softmax(scaled, dim=-1)
next_ids = torch.multinomial(probs, num_samples=1)
else:
next_ids = logits.argmax(dim=-1, keepdim=True)
all_codes[:, step] = next_ids.reshape(bsz)
if step < num_cb - 1:
new_embed = self.fast_embeddings(next_ids.reshape(bsz))
embed_buf[:bsz, step + 1, :] = new_embed
return all_codes
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights -- Fast AR weights have already been remapped by the parent."""
with set_current_vllm_config(self._vllm_config):
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded: set[str] = set()
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
handled = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped = name.replace(weight_name, param_name)
if mapped not in params_dict:
continue
param = params_dict[mapped]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
loaded.add(mapped)
handled = True
break
if handled:
continue
if name in params_dict:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded.add(name)
return loaded
@@ -0,0 +1,669 @@
"""Fish Speech S2 Pro -- Slow AR model (Stage 0).
Uses vLLM's ``Qwen3Model`` as the transformer backbone. Adds:
- Multi-codebook input embedding (text + summed codebook embeddings at
semantic-token positions).
- Semantic logit masking.
- Nested Fast AR for residual codebook prediction (``talker_mtp``).
- ``preprocess`` / ``postprocess`` hooks for vLLM-omni's AR scheduler.
Analogous to ``Qwen3TTSTalkerForConditionalGeneration`` in qwen3_tts.
"""
from __future__ import annotations
import dataclasses
import math
from collections.abc import Iterable
from typing import Any
import torch
import torch.nn as nn
from transformers import AutoTokenizer
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3 import Qwen3Model
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.models.output_templates import OmniOutput
from .configuration_fish_speech import FishSpeechConfig, FishSpeechFastARConfig, FishSpeechSlowARConfig
from .fish_speech_fast_ar import FishSpeechFastAR
logger = init_logger(__name__)
def _remap_fish_speech_weights(
weights: Iterable[tuple[str, torch.Tensor]],
n_head: int,
n_local_heads: int,
head_dim: int,
fast_n_head: int,
fast_n_local_heads: int,
fast_head_dim: int,
) -> Iterable[tuple[str, torch.Tensor]]:
"""Transform Fish Speech HF weight names/values to Qwen3-compatible format.
Key transformations:
- ``wqkv`` → split into ``q_proj``, ``k_proj``, ``v_proj``
- ``wo`` → ``o_proj``
- ``w1`` → ``gate_proj``, ``w3`` → ``up_proj``, ``w2`` → ``down_proj``
- ``attention_norm`` → ``input_layernorm``
- ``ffn_norm`` → ``post_attention_layernorm``
- ``text_model.model.embeddings`` → ``model.embed_tokens``
- ``audio_decoder.*`` → ``fast_ar.*`` with similar transforms
"""
q_size_text = n_head * head_dim
kv_size_text = n_local_heads * head_dim
q_size_fast = fast_n_head * fast_head_dim
kv_size_fast = fast_n_local_heads * fast_head_dim
for name, tensor in weights:
# --- Text model (Slow AR) ---
if name.startswith("text_model.model."):
suffix = name[len("text_model.model.") :]
# Embeddings
if suffix == "embeddings.weight":
yield "model.embed_tokens.weight", tensor
continue
# Norm
if suffix == "norm.weight":
yield "model.norm.weight", tensor
continue
# Layer weights
if suffix.startswith("layers."):
# layers.{N}.attention.wqkv.weight → split into q/k/v
if ".attention.wqkv.weight" in suffix:
layer_prefix = suffix.split(".attention.wqkv.weight")[0]
q = tensor[:q_size_text, :]
k = tensor[q_size_text : q_size_text + kv_size_text, :]
v = tensor[q_size_text + kv_size_text :, :]
yield f"model.{layer_prefix}.self_attn.q_proj.weight", q
yield f"model.{layer_prefix}.self_attn.k_proj.weight", k
yield f"model.{layer_prefix}.self_attn.v_proj.weight", v
continue
new_suffix = suffix
new_suffix = new_suffix.replace(".attention.wo.", ".self_attn.o_proj.")
new_suffix = new_suffix.replace(".attention.q_norm.", ".self_attn.q_norm.")
new_suffix = new_suffix.replace(".attention.k_norm.", ".self_attn.k_norm.")
new_suffix = new_suffix.replace(".attention_norm.", ".input_layernorm.")
new_suffix = new_suffix.replace(".feed_forward.w1.", ".mlp.gate_proj.")
new_suffix = new_suffix.replace(".feed_forward.w3.", ".mlp.up_proj.")
new_suffix = new_suffix.replace(".feed_forward.w2.", ".mlp.down_proj.")
new_suffix = new_suffix.replace(".ffn_norm.", ".post_attention_layernorm.")
yield f"model.{new_suffix}", tensor
continue
# Fallback for any other text_model.model.* weights
yield f"model.{suffix}", tensor
continue
# --- Audio decoder (Fast AR) ---
if name.startswith("audio_decoder."):
suffix = name[len("audio_decoder.") :]
# Codebook embeddings (belongs to the main model, not Fast AR).
if suffix == "codebook_embeddings.weight":
yield "codebook_embeddings.weight", tensor
continue
# Fast AR embeddings, output, norm.
if suffix == "embeddings.weight":
yield "fast_ar.fast_embeddings.weight", tensor
continue
if suffix == "output.weight":
yield "fast_ar.fast_output.weight", tensor
continue
if suffix == "norm.weight":
yield "fast_ar.fast_norm.weight", tensor
continue
# Fast AR projection in.
if suffix.startswith("fast_project_in."):
yield f"fast_ar.fast_project_in.{suffix[len('fast_project_in.') :]}", tensor
continue
# Fast AR layer weights.
if suffix.startswith("layers."):
if ".attention.wqkv.weight" in suffix:
layer_prefix = suffix.split(".attention.wqkv.weight")[0]
q = tensor[:q_size_fast, :]
k = tensor[q_size_fast : q_size_fast + kv_size_fast, :]
v = tensor[q_size_fast + kv_size_fast :, :]
yield f"fast_ar.model.{layer_prefix}.self_attn.q_proj.weight", q
yield f"fast_ar.model.{layer_prefix}.self_attn.k_proj.weight", k
yield f"fast_ar.model.{layer_prefix}.self_attn.v_proj.weight", v
continue
new_suffix = suffix
new_suffix = new_suffix.replace(".attention.wo.", ".self_attn.o_proj.")
new_suffix = new_suffix.replace(".attention.q_norm.", ".self_attn.q_norm.")
new_suffix = new_suffix.replace(".attention.k_norm.", ".self_attn.k_norm.")
new_suffix = new_suffix.replace(".attention_norm.", ".input_layernorm.")
new_suffix = new_suffix.replace(".feed_forward.w1.", ".mlp.gate_proj.")
new_suffix = new_suffix.replace(".feed_forward.w3.", ".mlp.up_proj.")
new_suffix = new_suffix.replace(".feed_forward.w2.", ".mlp.down_proj.")
new_suffix = new_suffix.replace(".ffn_norm.", ".post_attention_layernorm.")
yield f"fast_ar.model.{new_suffix}", tensor
continue
yield f"fast_ar.{suffix}", tensor
continue
# Pass through any other weights.
yield name, tensor
class FishSpeechSlowARForConditionalGeneration(nn.Module):
"""vLLM-AR Slow AR model for Fish Speech S2 Pro.
Stage 0: text → semantic tokens (+ residual codebook codes via Fast AR).
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.vllm_config = vllm_config
self.model_path = vllm_config.model_config.model
config: FishSpeechConfig = vllm_config.model_config.hf_config # type: ignore[assignment]
self.config = config
self.text_config: FishSpeechSlowARConfig = config.text_config
self.fast_ar_config: FishSpeechFastARConfig = config.audio_decoder_config
self._semantic_begin_id = int(config.semantic_start_token_id)
self._semantic_end_id = int(config.semantic_end_token_id)
self._audio_pad_token_id = int(config.audio_pad_token_id)
self._codebook_size = int(self.text_config.codebook_size)
self._num_codebooks = int(self.text_config.num_codebooks)
self.have_multimodal_outputs = True
self.has_preprocess = True
self.has_postprocess = True
self.mtp_hidden_size = int(self.text_config.hidden_size)
self.talker_mtp_output_key = "audio_codes"
# Qwen3 transformer backbone.
self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
# Fish Speech uses interleaved (GPT-J) RoPE, not NeoX style.
# vLLM's Qwen3Attention defaults to NeoX (is_neox_style=True).
# Replace with interleaved RoPE to match training.
self._fix_rope_style()
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
self.text_config.vocab_size,
self.text_config.hidden_size,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(self.text_config.vocab_size)
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
# Multi-codebook embedding table: codebook_size * num_codebooks entries.
self.codebook_embeddings = nn.Embedding(
self._codebook_size * self._num_codebooks,
self.text_config.hidden_size,
)
# Fast AR (residual codebook predictor).
predictor_compilation = dataclasses.replace(vllm_config.compilation_config)
predictor_compilation.static_forward_context = {}
self._fast_ar_vllm_config = dataclasses.replace(vllm_config, compilation_config=predictor_compilation)
from vllm.config.vllm import set_current_vllm_config as _set_cfg
with _set_cfg(self._fast_ar_vllm_config):
self.fast_ar = FishSpeechFastAR(
vllm_config=self._fast_ar_vllm_config,
config=self.fast_ar_config,
slow_ar_config=self.text_config,
prefix="fast_ar",
)
# Constant logit mask: allow only semantic tokens + im_end.
vocab = int(self.text_config.vocab_size)
semantic_mask = torch.zeros((vocab,), dtype=torch.bool)
lo = self._semantic_begin_id
hi = min(self._semantic_end_id + 1, vocab)
if hi > lo:
semantic_mask[lo:hi] = True
# Also allow <|im_end|> (token 151645 in Qwen3 tokeniser).
im_end_id = 151645
if im_end_id < vocab:
semantic_mask[im_end_id] = True
self.register_buffer("_semantic_allowed_mask", semantic_mask, persistent=False)
# Tokeniser (lazy).
self._tokenizer = None
def _fix_rope_style(self) -> None:
"""Replace NeoX-style RoPE with interleaved (GPT-J) style.
Fish Speech was trained with interleaved RoPE (complex-number pairs),
but vLLM's Qwen3Attention defaults to NeoX style. We rebuild the
rotary embedding with ``is_neox_style=False`` for each attention layer.
"""
from vllm.model_executor.layers.rotary_embedding import get_rope
for layer in self.model.layers:
attn = layer.self_attn
# Extract parameters from the existing RoPE to rebuild it.
head_dim = attn.head_dim
max_position = self.text_config.max_position_embeddings
rope_params = getattr(self.text_config, "rope_scaling", None) or {}
rope_params.setdefault("rope_theta", getattr(self.text_config, "rope_theta", 1000000.0))
attn.rotary_emb = get_rope(
head_size=head_dim,
max_position=max_position,
is_neox_style=False,
rope_parameters=rope_params,
)
logger.info("Fixed RoPE style to interleaved (GPT-J) for %d layers", len(self.model.layers))
# -------------------- vLLM required hooks --------------------
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**_: Any,
) -> torch.Tensor | IntermediateTensors:
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
def compute_logits(
self,
hidden_states: torch.Tensor | OmniOutput,
sampling_metadata: Any = None,
) -> torch.Tensor | None:
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if hidden_states is None:
return None
logits = self.logits_processor(self.lm_head, hidden_states)
if logits is None:
return None
# Mask to semantic tokens + im_end only.
logits = logits.masked_fill(~self._semantic_allowed_mask, float("-inf"))
return logits
# -------------------- Omni multimodal output plumbing --------------------
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
if isinstance(model_outputs, OmniOutput):
return model_outputs
hidden = model_outputs
info_dicts = kwargs.get("model_intermediate_buffer")
if info_dicts is None:
info_dicts = kwargs.get("runtime_additional_information") or []
audio_codes_list: list[torch.Tensor] = []
for info in info_dicts:
if not isinstance(info, dict):
continue
ac = info.get("audio_codes")
if isinstance(ac, torch.Tensor):
audio_codes_list.append(ac)
if not audio_codes_list:
logger.debug("make_omni_output: no audio_codes found in info_dicts (len=%d)", len(info_dicts))
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
audio_codes = torch.cat(audio_codes_list, dim=0)
span_len = int(audio_codes.shape[0])
hidden = hidden[:span_len]
mm: dict[str, torch.Tensor] = {"audio_codes": audio_codes}
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
# -------------------- preprocess / postprocess --------------------
def preprocess(
self,
input_ids: torch.Tensor,
input_embeds: torch.Tensor | None,
**info_dict: Any,
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
additional_information = info_dict.get("additional_information")
if isinstance(additional_information, dict):
merged: dict[str, Any] = {k: v for k, v in info_dict.items() if k != "additional_information"}
for k, v in additional_information.items():
merged.setdefault(k, v)
info_dict = merged
span_len = int(input_ids.shape[0])
if span_len <= 0:
return input_ids, input_embeds if input_embeds is not None else self.embed_input_ids(input_ids), {}
if span_len > 1:
# --- Prefill ---
prompt_embeds_cpu = info_dict.get("slow_ar_prompt_embeds")
is_first_prefill = not isinstance(prompt_embeds_cpu, torch.Tensor) or prompt_embeds_cpu.ndim != 2
dev = input_ids.device
if is_first_prefill:
prompt_embeds = self._build_prefill_embeds(input_ids, info_dict)
prompt_embeds_cpu = prompt_embeds.detach().to("cpu").contiguous()
info_update: dict[str, Any] = {
"slow_ar_prompt_embeds": prompt_embeds_cpu,
"prefill_offset": 0,
}
take = prompt_embeds_cpu[:span_len]
if int(take.shape[0]) < span_len:
pad_n = span_len - int(take.shape[0])
pad_embed = self.embed_input_ids(
torch.tensor([self._audio_pad_token_id], device=dev, dtype=torch.long)
).reshape(1, -1)
take = torch.cat([take, pad_embed.detach().cpu().expand(pad_n, -1)], dim=0)
prompt_embeds = take.to(device=dev, dtype=torch.bfloat16)
info_update["prefill_offset"] = span_len
zeros = torch.zeros(
(prompt_embeds.shape[0], self._num_codebooks),
device=dev,
dtype=torch.long,
)
info_update["audio_codes"] = zeros
input_ids_out = input_ids.clone()
input_ids_out[:] = self._audio_pad_token_id
return input_ids_out, prompt_embeds, info_update
else:
# Subsequent prefill chunk.
offset = int(info_dict.get("prefill_offset", 0) or 0)
s = max(0, min(offset, int(prompt_embeds_cpu.shape[0])))
e = max(0, min(offset + span_len, int(prompt_embeds_cpu.shape[0])))
take = prompt_embeds_cpu[s:e]
if int(take.shape[0]) < span_len:
pad_n = span_len - int(take.shape[0])
pad_embed = self.embed_input_ids(
torch.tensor([self._audio_pad_token_id], device=dev, dtype=torch.long)
).reshape(1, -1)
take = torch.cat([take, pad_embed.detach().cpu().expand(pad_n, -1)], dim=0)
prompt_embeds = take.to(device=dev, dtype=torch.bfloat16)
zeros = torch.zeros((prompt_embeds.shape[0], self._num_codebooks), device=dev, dtype=torch.long)
return (
input_ids.clone().fill_(self._audio_pad_token_id),
prompt_embeds,
{
"prefill_offset": offset + span_len,
"audio_codes": zeros,
},
)
# --- Decode: span_len == 1 ---
dev = input_ids.device
last_hidden_cpu = info_dict.get("last_slow_ar_hidden")
if not isinstance(last_hidden_cpu, torch.Tensor):
# First decode step after prefill -- just embed the token directly.
logger.warning(
"preprocess decode: last_slow_ar_hidden not found (keys=%s), "
"returning plain embed (mtp_inputs will NOT be set)",
list(info_dict.keys()),
)
embeds = self.embed_input_ids(input_ids.reshape(1, 1).to(torch.long)).reshape(1, -1)
return input_ids, embeds.to(dtype=torch.bfloat16), {}
token_embed = self.embed_input_ids(input_ids.reshape(1, 1).to(torch.long)).to(
device=dev, dtype=torch.bfloat16
) # [1, 1, H]
# Codebook embeddings are added by talker_mtp (using the CURRENT step's
# FastAR hidden state) instead of here (which would use the PREVIOUS step's
# codes, one step too old). Preprocess always returns plain text embed.
inputs_embeds_out = token_embed.reshape(1, -1)
info_update = {
"mtp_inputs": (
last_hidden_cpu.to(device=dev, dtype=torch.bfloat16).reshape(1, -1),
torch.zeros(1, self.text_config.hidden_size, device=dev, dtype=torch.bfloat16),
),
}
return input_ids, inputs_embeds_out, info_update
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
if hidden_states.numel() == 0:
logger.debug("postprocess: empty hidden_states")
return {}
last = hidden_states[-1, :].detach().to("cpu").contiguous()
logger.debug("postprocess: saved last_slow_ar_hidden shape=%s", tuple(last.shape))
return {"last_slow_ar_hidden": last}
# -------------------- prompt construction --------------------
def _get_tokenizer(self):
if self._tokenizer is None:
self._tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True)
return self._tokenizer
def _build_prefill_embeds(
self,
input_ids: torch.Tensor,
info_dict: dict[str, Any],
) -> torch.Tensor:
"""Build prefill embeddings, adding codebook embeddings at semantic positions.
For text-only prefill (no reference audio), this is just embed_input_ids.
For voice cloning, reference codes are embedded with codebook offsets.
"""
dev = input_ids.device
# Basic text embeddings.
base_embeds = self.embed_input_ids(input_ids.reshape(1, -1).to(torch.long)) # [1, T, H]
# Check for reference codebook codes (for voice cloning).
ref_codes = info_dict.get("ref_codes")
if not isinstance(ref_codes, torch.Tensor) or ref_codes.numel() == 0:
return base_embeds.squeeze(0).to(dtype=torch.bfloat16)
# ref_codes: [T_ref, num_codebooks] -- codebook codes for reference audio positions.
ref_codes = ref_codes.to(device=dev, dtype=torch.long)
ref_positions = info_dict.get("ref_positions")
if not isinstance(ref_positions, torch.Tensor):
return base_embeds.squeeze(0).to(dtype=torch.bfloat16)
ref_positions = ref_positions.to(device=dev, dtype=torch.long).reshape(-1)
seq_len = int(input_ids.shape[0])
codebook_sum = torch.zeros_like(base_embeds) # [1, T, H]
for pos_idx in range(int(ref_positions.shape[0])):
pos = int(ref_positions[pos_idx].item())
if pos < 0 or pos >= seq_len:
continue
for cb_idx in range(min(int(ref_codes.shape[1]), self._num_codebooks)):
code = ref_codes[pos_idx, cb_idx].clamp(min=0)
code_with_offset = code + cb_idx * self._codebook_size
emb = self.codebook_embeddings(code_with_offset.unsqueeze(0))
codebook_sum[0, pos, :] += emb.squeeze(0).to(dtype=base_embeds.dtype)
result = base_embeds + codebook_sum
return result.squeeze(0).to(dtype=torch.bfloat16)
# -------------------- GPU-side MTP fast-path --------------------
@torch.inference_mode()
def talker_mtp(
self,
input_ids: torch.Tensor,
input_embeds: torch.Tensor,
last_talker_hidden: torch.Tensor,
text_step: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""GPU fast-path: run Fast AR to predict residual codebook codes.
Returns (inputs_embeds, audio_codes).
The embedding is: text_embed(token) + sum(codebook_embed(code_i + i * codebook_size))
where codes come from FastAR(last_talker_hidden).
This matches the reference Fish Speech inference flow:
- At step t, the Slow AR embedding includes codes from FastAR(hidden_{t-1})
- last_talker_hidden IS hidden_{t-1} (from postprocess of the previous step)
- Preprocess provides the plain text embed; we add codebook embeddings here
"""
bsz = int(input_ids.shape[0])
dev = input_embeds.device
input_ids = input_ids.reshape(bsz, 1).to(dtype=torch.long, device=dev)
past_hidden = last_talker_hidden.reshape(bsz, -1).to(dtype=torch.bfloat16, device=dev)
# Run Fast AR to predict all num_codebooks codes.
audio_codes = self.fast_ar(
slow_ar_hidden=past_hidden,
semantic_token_id=input_ids.reshape(bsz),
do_sample=True,
temperature=0.8,
top_k=30,
top_p=0.9,
) # [B, num_codebooks]
# Add codebook embeddings to the input embedding (from preprocess).
# This ensures the Slow AR sees codes from FastAR(hidden_{t-1}).
inputs_embeds_out = input_embeds.reshape(bsz, -1).clone()
for b in range(bsz):
token_id = int(input_ids[b, 0].item())
is_semantic = self._semantic_begin_id <= token_id <= self._semantic_end_id
if is_semantic:
codes = audio_codes[b] # [num_codebooks]
codebook_sum = torch.zeros(self.text_config.hidden_size, device=dev, dtype=torch.bfloat16)
for i in range(self._num_codebooks):
code_with_offset = codes[i].clamp(min=0) + i * self._codebook_size
emb = self.codebook_embeddings(code_with_offset.unsqueeze(0))
codebook_sum += emb.squeeze(0).to(dtype=torch.bfloat16)
# Normalize by sqrt(num_codebooks + 1) as in the reference model
# (scale_codebook_embeddings=True for fish_qwen3_omni).
inputs_embeds_out[b] = (inputs_embeds_out[b] + codebook_sum) / math.sqrt(self._num_codebooks + 1)
return inputs_embeds_out, audio_codes.to(dtype=torch.long)
# -------------------- Prompt length estimation --------------------
@staticmethod
def estimate_prompt_len_from_additional_information(
additional_information: dict[str, Any] | None,
**kwargs: Any,
) -> int:
"""Estimate prompt length for placeholder allocation."""
info = additional_information or {}
text = info.get("text", [""])[0] if isinstance(info.get("text"), list) else info.get("text", "")
# Conservative estimate: tokenize text length + overhead.
return max(2, len(str(text)) // 2 + 64)
# -------------------- Weight loading --------------------
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with Fish Speech → Qwen3 format transformation.
Transforms weight names (wqkv → q/k/v split, w1/w2/w3 → gate/up/down,
etc.) and routes to the correct sub-modules.
"""
n_head = self.text_config.num_attention_heads
n_local_heads = self.text_config.num_key_value_heads
head_dim = self.text_config.head_dim
fast_n_head = self.fast_ar_config.num_attention_heads
fast_n_local_heads = self.fast_ar_config.num_key_value_heads
fast_head_dim = self.fast_ar_config.head_dim
remapped = _remap_fish_speech_weights(
weights,
n_head,
n_local_heads,
head_dim,
fast_n_head,
fast_n_local_heads,
fast_head_dim,
)
# Qwen3Model uses stacked_params_mapping for q/k/v → qkv_proj
# and gate/up → gate_up_proj. Feed the remapped weights through
# the standard Qwen3 loading path.
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in remapped:
if "rotary_emb.inv_freq" in name:
continue
# Handle tied embeddings → lm_head.
if name == "model.embed_tokens.weight" and self.text_config.tie_word_embeddings:
# Also load into lm_head if present.
lm_key = "lm_head.weight"
if lm_key in params_dict:
p = params_dict[lm_key]
wl = getattr(p, "weight_loader", default_weight_loader)
wl(p, loaded_weight)
loaded_params.add(lm_key)
# Try stacked params mapping (q/k/v → qkv_proj, gate/up → gate_up_proj).
handled = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped = name.replace(weight_name, param_name)
if mapped not in params_dict:
continue
param = params_dict[mapped]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(mapped)
handled = True
break
if handled:
continue
# Direct parameter mapping.
if name in params_dict:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
logger.info("Loaded %d weights for FishSpeechSlowARForConditionalGeneration", len(loaded_params))
# Truncate RoPE cos/sin caches to bf16 precision to match training.
# Without this, f32 RoPE values cause logit divergence and premature EOS.
truncated = 0
for module in self.modules():
if hasattr(module, "cos_sin_cache") and isinstance(module.cos_sin_cache, torch.Tensor):
cache = module.cos_sin_cache
module.cos_sin_cache = cache.to(torch.bfloat16).to(cache.dtype)
truncated += 1
if truncated:
logger.info("Truncated %d RoPE cos_sin_cache buffers to bf16 precision", truncated)
return loaded_params
@@ -1054,6 +1054,79 @@ class HunyuanImage3MultiModalProcessor(BaseMultiModalProcessor[HunyuanImage3Proc
]
class HunyuanImage3RotaryEmbedding(nn.Module):
"""Custom interleaved 2D Rotary Embedding for HunyuanImage3.
The original HunyuanImage3 ``build_2d_rope`` interleaves y (height) and
x (width) positions across consecutive frequencies::
theta = inv_freq.reshape(n_elem // 4, 2)
idx_theta[2k] = y_pos * theta[k, 0] # even freq -> y
idx_theta[2k+1] = x_pos * theta[k, 1] # odd freq -> x
vLLM's standard ``MRotaryEmbedding`` instead assigns *contiguous* blocks
of frequencies to each position dimension, producing different encodings
for the same positions. This class re-implements the original interleaved
pattern so that pre-trained weights work correctly.
Reference: https://kexue.fm/archives/10352
"""
def __init__(self, head_dim: int, rope_theta: float = 10000.0) -> None:
super().__init__()
self.head_dim = head_dim
assert head_dim % 4 == 0, f"head_dim must be divisible by 4, got {head_dim}"
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(
self,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
if positions.dim() == 2 and positions.shape[0] == 3:
y_pos = positions[1].float()
x_pos = positions[2].float()
else:
# 1D fallback: both dims get the same position → standard RoPE
y_pos = positions.float()
x_pos = positions.float()
num_tokens = y_pos.shape[0]
dtype = query.dtype
query_shape = query.shape
key_shape = key.shape
inv_freq = self.inv_freq.to(device=y_pos.device, dtype=torch.float32)
inv_freq_y = inv_freq[0::2] # even indices -> y
inv_freq_x = inv_freq[1::2] # odd indices -> x
y_freqs = y_pos.unsqueeze(-1) * inv_freq_y.unsqueeze(0)
x_freqs = x_pos.unsqueeze(-1) * inv_freq_x.unsqueeze(0)
# Interleave: [y*θ₀, x*θ₁, y*θ₂, x*θ₃, ...]
freqs = torch.stack([y_freqs, x_freqs], dim=-1).reshape(num_tokens, -1)
emb = torch.cat([freqs, freqs], dim=-1)
cos = emb.cos().to(dtype).unsqueeze(1)
sin = emb.sin().to(dtype).unsqueeze(1)
query = query.view(num_tokens, -1, self.head_dim)
key = key.view(num_tokens, -1, self.head_dim)
query = query * cos + self._rotate_half(query) * sin
key = key * cos + self._rotate_half(key) * sin
return query.view(query_shape), key.view(key_shape)
def _rotate_half(self, x: torch.Tensor) -> torch.Tensor:
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
@MULTIMODAL_REGISTRY.register_processor(
HunyuanImage3MultiModalProcessor,
info=HunyuanImage3ProcessingInfo,
@@ -1153,6 +1226,38 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
self._mrope_joint_img_sep_token_id = tokenizer.convert_tokens_to_ids("<joint_img_sep>")
self._mrope_max_num_patches = config.vit_processor.get("max_num_patches", 729)
self._replace_rotary_embeddings()
def _replace_rotary_embeddings(self):
"""Replace vLLM's standard MRotaryEmbedding with the custom
interleaved 2D RoPE that matches the original HunyuanImage3 model."""
rope_theta = getattr(self.config, "rope_theta", 10000.0)
head_dim = getattr(
self.config,
"head_dim",
getattr(
self.config,
"attention_head_dim",
self.config.hidden_size // self.config.num_attention_heads,
),
)
custom_rope = HunyuanImage3RotaryEmbedding(
head_dim=head_dim,
rope_theta=rope_theta,
)
replaced = 0
for layer in self.model.layers:
if hasattr(layer, "self_attn") and hasattr(layer.self_attn, "rotary_emb"):
layer.self_attn.rotary_emb = custom_rope
replaced += 1
logger.info(
"Replaced %d rotary embeddings with HunyuanImage3RotaryEmbedding "
"(interleaved 2D RoPE, head_dim=%d, rope_theta=%.1f)",
replaced,
head_dim,
rope_theta,
)
def _parse_and_validate_image_input(
self,
**kwargs: dict[str, Any],
@@ -1414,19 +1519,20 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
"""Compute mRoPE positions for HunyuanImage-3.
Maps the original model's build_2d_rope logic into vLLM's 3-dim
mRoPE position tensor [3, seq_len] where dim-0 is temporal (unused,
kept equal to 1D), dim-1 is height, and dim-2 is width.
mRoPE position tensor ``[3, seq_len]`` where dim-1 is height and
dim-2 is width. dim-0 is unused (temporal) and kept equal to 1D.
For text tokens and auxiliary image tokens (timestep, ViT):
For text tokens and auxiliary image tokens (timestep, separators):
All three dims get the same flat 1D position id.
For VAE image tokens:
dim-0 (T): flat 1D position id at the image start
For VAE / ViT image tokens:
dim-0 (T): flat 1D position id at the region start
dim-1 (H): 2D y-position using build_2d_rope centering
dim-2 (W): 2D x-position using build_2d_rope centering
"""
# Extract per-image VAE grid dims from mm_features
# Extract per-image VAE and ViT grid dims from mm_features
vae_grids: list[tuple[int, int]] = []
vit_grids: list[tuple[int, int]] = []
if mm_features is not None:
for mm_feature in mm_features:
mm_item = mm_feature.data
@@ -1437,20 +1543,22 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
if vae_hw is not None:
grid = vae_hw.tolist()
vae_grids.append((int(grid[0]), int(grid[1])))
vit_hw = mm_input.get("vit_spatial_shapes")
if vit_hw is not None:
grid = vit_hw.tolist()
vit_grids.append((int(grid[0]), int(grid[1])))
# Identify image token ids (cached from __init__)
img_token_id = self._mrope_img_token_id
boi_token_id = self._mrope_boi_token_id
eoi_token_id = self._mrope_eoi_token_id
joint_img_sep_token_id = self._mrope_joint_img_sep_token_id
max_num_patches = self._mrope_max_num_patches
# Build position arrays
t_pos = [] # temporal (same as 1D for this model)
h_pos = [] # height
w_pos = [] # width
t_pos: list[int] = [] # temporal (same as 1D for this model)
h_pos: list[int] = [] # height
w_pos: list[int] = [] # width
pos = 0 # current 1D position counter
pos = 0
image_idx = 0
i = 0
n = len(input_tokens)
@@ -1462,7 +1570,7 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
# Found start of image block.
# Structure: <boi> <size> <ratio> <img>*timestep <img>*vae
# <joint_img_sep> <img>*vit <eoi>
# Assign <boi> a flat position
# <boi> token
t_pos.append(pos)
h_pos.append(pos)
w_pos.append(pos)
@@ -1493,16 +1601,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
pos += 1
i += 1
# VAE tokens: get grid dims
# VAE tokens: 2D grid positions
if image_idx < len(vae_grids):
vae_h, vae_w = vae_grids[image_idx]
else:
vae_h, vae_w = 0, 0
image_idx += 1
# Assign 2D positions to VAE tokens using build_2d_rope
# centering logic
L = pos # position at start of VAE region
L = pos
wh = vae_w * vae_h
beta_y = L + (wh - vae_h) / 2
beta_x = L + (wh - vae_w) / 2
@@ -1510,12 +1614,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
for row in range(vae_h):
for col in range(vae_w):
if i < n and input_tokens[i] == img_token_id:
t_pos.append(L) # temporal stays flat
t_pos.append(L)
h_pos.append(int(beta_y + row))
w_pos.append(int(beta_x + col))
i += 1
pos = L + wh # advance past VAE region
pos = L + wh
# <joint_img_sep> token
if i < n and input_tokens[i] == joint_img_sep_token_id:
@@ -1525,15 +1629,34 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
pos += 1
i += 1
# ViT tokens (max_num_patches <img> tokens) — flat 1D
vit_consumed = 0
while i < n and input_tokens[i] == img_token_id and vit_consumed < max_num_patches:
# ViT tokens: 2D grid positions
if image_idx < len(vit_grids):
vit_h, vit_w = vit_grids[image_idx]
else:
vit_h, vit_w = 0, 0
L = pos
wh = vit_w * vit_h
beta_y = L + (wh - vit_h) / 2
beta_x = L + (wh - vit_w) / 2
for row in range(vit_h):
for col in range(vit_w):
if i < n and input_tokens[i] == img_token_id:
t_pos.append(L)
h_pos.append(int(beta_y + row))
w_pos.append(int(beta_x + col))
i += 1
pos = L + wh
# Remaining ViT padding tokens (when max_num_patches >
# vit_h * vit_w) get flat 1D positions.
while i < n and input_tokens[i] == img_token_id:
t_pos.append(pos)
h_pos.append(pos)
w_pos.append(pos)
pos += 1
i += 1
vit_consumed += 1
# <eoi> token
if i < n and input_tokens[i] == eoi_token_id:
@@ -1543,6 +1666,7 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
pos += 1
i += 1
image_idx += 1
else:
# Regular text token — flat 1D position
t_pos.append(pos)
@@ -129,6 +129,17 @@ _OMNI_MODELS = {
"hunyuan_image3",
"HunyuanImage3ForConditionalGeneration",
),
## fish_speech (Fish Speech S2 Pro)
"FishSpeechSlowARForConditionalGeneration": (
"fish_speech",
"fish_speech_slow_ar",
"FishSpeechSlowARForConditionalGeneration",
),
"FishSpeechDACDecoder": (
"fish_speech",
"fish_speech_dac_decoder",
"FishSpeechDACDecoder",
),
}