Merge branch 'main' into dev/vllm-align
This commit is contained in:
commit
d3db476f2d
144 files changed
+17247
-1024
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,154 @@
|
||||
"""Configuration classes for Fish Speech S2 Pro (fish_qwen3_omni).
|
||||
|
||||
The HuggingFace config uses field names from the original Fish Speech codebase
|
||||
(dim, n_head, n_layer, etc.). This module re-exports them under the standard
|
||||
Transformers/Qwen3 attribute names so that vLLM's ``Qwen3Model`` can consume
|
||||
the Slow AR config transparently.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
|
||||
class FishSpeechSlowARConfig(PretrainedConfig):
|
||||
"""Slow AR (text_model) config -- Qwen3-based transformer.
|
||||
|
||||
Maps Fish Speech field names to Qwen3-compatible attribute names so
|
||||
``vllm.model_executor.models.qwen3.Qwen3Model`` works out of the box.
|
||||
"""
|
||||
|
||||
model_type = "fish_qwen3"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 155776,
|
||||
dim: int = 2560,
|
||||
n_head: int = 32,
|
||||
n_local_heads: int = 8,
|
||||
head_dim: int = 128,
|
||||
n_layer: int = 36,
|
||||
intermediate_size: int = 9728,
|
||||
attention_qk_norm: bool = True,
|
||||
rope_base: float = 1_000_000.0,
|
||||
max_seq_len: int = 32768,
|
||||
tie_word_embeddings: bool = True,
|
||||
codebook_size: int = 4096,
|
||||
num_codebooks: int = 10,
|
||||
semantic_begin_id: int = 0,
|
||||
semantic_end_id: int = 0,
|
||||
scale_codebook_embeddings: bool = False,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
**kwargs,
|
||||
):
|
||||
# Fish Speech field names → standard Transformers/Qwen3 names.
|
||||
self.hidden_size = dim
|
||||
self.num_attention_heads = n_head
|
||||
self.num_key_value_heads = n_local_heads
|
||||
self.head_dim = head_dim
|
||||
self.num_hidden_layers = n_layer
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_position_embeddings = max_seq_len
|
||||
self.rope_theta = rope_base
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.hidden_act = "silu"
|
||||
self.attention_bias = False
|
||||
self.attention_qk_norm = attention_qk_norm
|
||||
|
||||
# Fish Speech codec / codebook fields.
|
||||
self.codebook_size = codebook_size
|
||||
self.num_codebooks = num_codebooks
|
||||
self.semantic_begin_id = semantic_begin_id
|
||||
self.semantic_end_id = semantic_end_id
|
||||
self.scale_codebook_embeddings = scale_codebook_embeddings
|
||||
|
||||
super().__init__(
|
||||
vocab_size=vocab_size,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class FishSpeechFastARConfig(PretrainedConfig):
|
||||
"""Fast AR (audio_decoder) config -- 4-layer residual codebook predictor."""
|
||||
|
||||
model_type = "fish_qwen3_audio_decoder"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size: int = 4096,
|
||||
num_codebooks: int = 10,
|
||||
dim: int = 2560,
|
||||
n_head: int = 32,
|
||||
n_local_heads: int = 8,
|
||||
head_dim: int = 128,
|
||||
n_layer: int = 4,
|
||||
intermediate_size: int = 9728,
|
||||
max_seq_len: int = 11,
|
||||
text_dim: int = 2560,
|
||||
audio_hidden_dim: int = 5120,
|
||||
attention_qk_norm: bool = False,
|
||||
rope_base: float = 1_000_000.0,
|
||||
rms_norm_eps: float = 1e-6,
|
||||
**kwargs,
|
||||
):
|
||||
self.hidden_size = dim
|
||||
self.num_attention_heads = n_head
|
||||
self.num_key_value_heads = n_local_heads
|
||||
self.head_dim = head_dim
|
||||
self.num_hidden_layers = n_layer
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_position_embeddings = max_seq_len
|
||||
self.rope_theta = rope_base
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.hidden_act = "silu"
|
||||
self.attention_bias = False
|
||||
self.attention_qk_norm = attention_qk_norm
|
||||
self.num_codebooks = num_codebooks
|
||||
self.text_dim = text_dim
|
||||
self.audio_hidden_dim = audio_hidden_dim
|
||||
|
||||
super().__init__(vocab_size=vocab_size, **kwargs)
|
||||
|
||||
|
||||
class FishSpeechConfig(PretrainedConfig):
|
||||
"""Top-level config for Fish Speech S2 Pro (fish_qwen3_omni).
|
||||
|
||||
Wraps ``text_config`` (Slow AR) and ``audio_decoder_config`` (Fast AR).
|
||||
"""
|
||||
|
||||
model_type = "fish_qwen3_omni"
|
||||
sub_configs = {
|
||||
"text_config": FishSpeechSlowARConfig,
|
||||
"audio_decoder_config": FishSpeechFastARConfig,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | FishSpeechSlowARConfig | None = None,
|
||||
audio_decoder_config: dict | FishSpeechFastARConfig | None = None,
|
||||
semantic_start_token_id: int = 151678,
|
||||
semantic_end_token_id: int = 155773,
|
||||
audio_pad_token_id: int = 151677,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(text_config, dict):
|
||||
text_config = FishSpeechSlowARConfig(**text_config)
|
||||
if isinstance(audio_decoder_config, dict):
|
||||
audio_decoder_config = FishSpeechFastARConfig(**audio_decoder_config)
|
||||
|
||||
self.text_config = text_config or FishSpeechSlowARConfig()
|
||||
self.audio_decoder_config = audio_decoder_config or FishSpeechFastARConfig()
|
||||
|
||||
self.semantic_start_token_id = semantic_start_token_id
|
||||
self.semantic_end_token_id = semantic_end_token_id
|
||||
self.audio_pad_token_id = audio_pad_token_id
|
||||
|
||||
# Propagate semantic IDs into text_config for convenience.
|
||||
self.text_config.semantic_begin_id = semantic_start_token_id
|
||||
self.text_config.semantic_end_id = semantic_end_token_id
|
||||
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_text_config(self, **kwargs) -> FishSpeechSlowARConfig:
|
||||
return self.text_config
|
||||
@@ -0,0 +1,111 @@
|
||||
"""DAC codec encoder for Fish Speech S2 Pro voice cloning.
|
||||
|
||||
Encodes reference audio into VQ codes for use as prompt conditioning.
|
||||
Runs on CPU in the API server process -- loaded lazily on first use.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.fish_speech.dac_utils import (
|
||||
DAC_SAMPLE_RATE,
|
||||
build_dac_codec,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_codec_cache: dict[str, nn.Module] = {}
|
||||
|
||||
|
||||
def _load_dac_codec(model_path: str) -> nn.Module:
|
||||
"""Load the DAC codec model from codec.pth (cached per model_path)."""
|
||||
if model_path in _codec_cache:
|
||||
return _codec_cache[model_path]
|
||||
|
||||
codec_path = os.path.join(model_path, "codec.pth")
|
||||
if not os.path.exists(codec_path):
|
||||
from transformers.utils.hub import cached_file
|
||||
|
||||
cached = cached_file(model_path, "codec.pth")
|
||||
if cached is not None:
|
||||
codec_path = cached
|
||||
|
||||
if not os.path.exists(codec_path):
|
||||
raise FileNotFoundError(
|
||||
f"codec.pth not found for {model_path}. Required for voice cloning with Fish Speech S2 Pro."
|
||||
)
|
||||
|
||||
codec = build_dac_codec()
|
||||
|
||||
state_dict = torch.load(codec_path, map_location="cpu", weights_only=True)
|
||||
if "generator" in state_dict:
|
||||
state_dict = state_dict["generator"]
|
||||
codec.load_state_dict(state_dict, strict=False)
|
||||
codec.eval()
|
||||
|
||||
_codec_cache[model_path] = codec
|
||||
logger.info("Loaded DAC codec encoder from %s (CPU)", codec_path)
|
||||
return codec
|
||||
|
||||
|
||||
def _resample(wav: np.ndarray, sr: int, target_sr: int) -> np.ndarray:
|
||||
"""Resample audio using torchaudio's polyphase resampling."""
|
||||
if sr == target_sr:
|
||||
return wav
|
||||
import torchaudio
|
||||
|
||||
wav_t = torch.from_numpy(wav).unsqueeze(0).float()
|
||||
wav_t = torchaudio.functional.resample(wav_t, sr, target_sr)
|
||||
return wav_t.squeeze(0).numpy()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def encode_reference_audio(
|
||||
model_path: str,
|
||||
wav_samples: list[float] | np.ndarray,
|
||||
sample_rate: int,
|
||||
) -> list[int]:
|
||||
"""Encode reference audio into semantic token IDs for prompt conditioning.
|
||||
|
||||
Args:
|
||||
model_path: HuggingFace model path (for locating codec.pth).
|
||||
wav_samples: Audio waveform samples (mono, float).
|
||||
sample_rate: Sample rate of the input audio.
|
||||
|
||||
Returns:
|
||||
List of semantic token IDs (151678 + code_value for each frame).
|
||||
"""
|
||||
codec = _load_dac_codec(model_path)
|
||||
|
||||
wav = np.asarray(wav_samples, dtype=np.float32)
|
||||
if wav.ndim > 1:
|
||||
wav = np.mean(wav, axis=-1)
|
||||
|
||||
# Resample to DAC sample rate (44100).
|
||||
wav = _resample(wav, sample_rate, DAC_SAMPLE_RATE)
|
||||
|
||||
# Encode: [1, 1, T] -> codes [1, num_codebooks, num_frames]
|
||||
wav_tensor = torch.from_numpy(wav).unsqueeze(0).unsqueeze(0).float()
|
||||
feature_lengths = torch.tensor([wav_tensor.shape[-1]])
|
||||
codes, feature_lengths_out = codec.encode(wav_tensor, feature_lengths)
|
||||
|
||||
# Extract semantic codebook (index 0) - shape [num_frames].
|
||||
semantic_codes = codes[0, 0, :].tolist()
|
||||
|
||||
# Convert to semantic token IDs: <|semantic:{i}|> = 151678 + i
|
||||
SEMANTIC_TOKEN_OFFSET = 151678
|
||||
semantic_token_ids = [SEMANTIC_TOKEN_OFFSET + int(c) for c in semantic_codes]
|
||||
|
||||
logger.info(
|
||||
"Encoded reference audio: %d samples @ %dHz -> %d semantic tokens",
|
||||
len(wav_samples),
|
||||
sample_rate,
|
||||
len(semantic_token_ids),
|
||||
)
|
||||
return semantic_token_ids
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Shared DAC codec construction for Fish Speech S2 Pro.
|
||||
|
||||
Used by both the encoder (voice cloning, CPU) and the decoder (Stage 1, GPU).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
# Default DAC codec config matching s2-pro's modded_dac_vq.yaml.
|
||||
DAC_SAMPLE_RATE = 44100
|
||||
DAC_HOP_LENGTH = 2048 # 512 (decoder upsample) * 4 (quantizer upsample)
|
||||
DAC_NUM_CODEBOOKS = 10 # 1 semantic + 9 residual
|
||||
|
||||
|
||||
def build_dac_codec() -> nn.Module:
|
||||
"""Construct a DAC codec model (uninitialized weights).
|
||||
|
||||
Returns the model on CPU in eval mode -- caller is responsible for
|
||||
loading weights and moving to the target device.
|
||||
"""
|
||||
from fish_speech.models.dac.modded_dac import (
|
||||
DAC,
|
||||
ModelArgs,
|
||||
WindowLimitedTransformer,
|
||||
)
|
||||
from fish_speech.models.dac.rvq import DownsampleResidualVectorQuantize
|
||||
|
||||
base_transformer_kwargs = dict(
|
||||
block_size=16384,
|
||||
n_local_heads=-1,
|
||||
head_dim=64,
|
||||
rope_base=10000,
|
||||
norm_eps=1e-5,
|
||||
dropout_rate=0.0,
|
||||
attn_dropout_rate=0.0,
|
||||
channels_first=True,
|
||||
)
|
||||
|
||||
def _make_transformer_config(**kw):
|
||||
return ModelArgs(**{**base_transformer_kwargs, **kw})
|
||||
|
||||
quantizer_transformer_config = ModelArgs(
|
||||
block_size=4096,
|
||||
n_layer=8,
|
||||
n_head=16,
|
||||
dim=1024,
|
||||
intermediate_size=3072,
|
||||
n_local_heads=-1,
|
||||
head_dim=64,
|
||||
rope_base=10000,
|
||||
norm_eps=1e-5,
|
||||
dropout_rate=0.0,
|
||||
attn_dropout_rate=0.0,
|
||||
channels_first=True,
|
||||
)
|
||||
post_module = WindowLimitedTransformer(
|
||||
causal=True,
|
||||
window_size=128,
|
||||
input_dim=1024,
|
||||
config=quantizer_transformer_config,
|
||||
)
|
||||
pre_module = WindowLimitedTransformer(
|
||||
causal=True,
|
||||
window_size=128,
|
||||
input_dim=1024,
|
||||
config=quantizer_transformer_config,
|
||||
)
|
||||
quantizer = DownsampleResidualVectorQuantize(
|
||||
input_dim=1024,
|
||||
n_codebooks=9,
|
||||
codebook_size=1024,
|
||||
codebook_dim=8,
|
||||
quantizer_dropout=0.0,
|
||||
downsample_factor=[2, 2],
|
||||
post_module=post_module,
|
||||
pre_module=pre_module,
|
||||
semantic_codebook_size=4096,
|
||||
)
|
||||
codec = DAC(
|
||||
encoder_dim=64,
|
||||
encoder_rates=[2, 4, 8, 8],
|
||||
decoder_dim=1536,
|
||||
decoder_rates=[8, 8, 4, 2],
|
||||
quantizer=quantizer,
|
||||
sample_rate=DAC_SAMPLE_RATE,
|
||||
causal=True,
|
||||
encoder_transformer_layers=[0, 0, 0, 4],
|
||||
decoder_transformer_layers=[4, 0, 0, 0],
|
||||
transformer_general_config=_make_transformer_config,
|
||||
)
|
||||
return codec
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Fish Speech S2 Pro -- DAC Decoder (Stage 1).
|
||||
|
||||
Loads the DAC codec from ``codec.pth`` and decodes codebook indices
|
||||
[num_codebooks, T] → audio waveform at 44.1 kHz.
|
||||
|
||||
Analogous to ``Qwen3TTSCode2Wav`` in qwen3_tts.
|
||||
|
||||
Requires the ``fish-speech`` package for the DAC model architecture.
|
||||
Install with: ``pip install fish-speech``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.models.fish_speech.dac_utils import (
|
||||
DAC_HOP_LENGTH,
|
||||
DAC_NUM_CODEBOOKS,
|
||||
DAC_SAMPLE_RATE,
|
||||
build_dac_codec,
|
||||
)
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class FishSpeechDACDecoder(nn.Module):
|
||||
"""Stage-1 DAC decoder for Fish Speech S2 Pro (GenerationModelRunner).
|
||||
|
||||
Consumes frame-aligned codec tokens from input_ids and decodes waveform
|
||||
via the DAC codec decoder.
|
||||
"""
|
||||
|
||||
input_modalities = "audio"
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
self.enable_update_additional_information = True
|
||||
self.requires_raw_input_tokens = True
|
||||
|
||||
self._codec: nn.Module | None = None
|
||||
self._num_codebooks: int = DAC_NUM_CODEBOOKS
|
||||
self._output_sample_rate: int = DAC_SAMPLE_RATE
|
||||
self._hop_length: int = DAC_HOP_LENGTH
|
||||
self._logged_codec_stats = False
|
||||
|
||||
def _ensure_codec_loaded(self) -> None:
|
||||
if self._codec is not None:
|
||||
return
|
||||
|
||||
codec_path = os.path.join(self.model_path, "codec.pth")
|
||||
if not os.path.exists(codec_path):
|
||||
# Try HuggingFace cache.
|
||||
try:
|
||||
from transformers.utils.hub import cached_file
|
||||
|
||||
cached = cached_file(self.model_path, "codec.pth")
|
||||
if cached is not None:
|
||||
codec_path = cached
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not os.path.exists(codec_path):
|
||||
raise FileNotFoundError(
|
||||
f"codec.pth not found at {codec_path}. Make sure the Fish Speech S2 Pro model includes codec.pth."
|
||||
)
|
||||
|
||||
codec = build_dac_codec()
|
||||
|
||||
# Load weights.
|
||||
state_dict = torch.load(codec_path, map_location="cpu", weights_only=True)
|
||||
# Some checkpoints wrap under "generator" key.
|
||||
if "generator" in state_dict:
|
||||
state_dict = state_dict["generator"]
|
||||
codec.load_state_dict(state_dict, strict=False)
|
||||
|
||||
device = self.vllm_config.device_config.device
|
||||
codec = codec.to(device=device, dtype=torch.float32)
|
||||
codec.eval()
|
||||
self._codec = codec
|
||||
|
||||
logger.info(
|
||||
"Fish Speech DAC codec loaded from %s (device=%s, sample_rate=%d)",
|
||||
codec_path,
|
||||
device,
|
||||
self._output_sample_rate,
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
if input_ids.numel() == 0:
|
||||
return torch.empty((0, 1), device=input_ids.device, dtype=torch.float32)
|
||||
return torch.zeros((input_ids.shape[0], 1), device=input_ids.device, dtype=torch.float32)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput, sampling_metadata: Any = None) -> None:
|
||||
return None
|
||||
|
||||
def _split_request_ids(
|
||||
self,
|
||||
ids: torch.Tensor,
|
||||
seq_token_counts: list[int] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
if seq_token_counts is not None and len(seq_token_counts) > 1:
|
||||
boundaries = [0]
|
||||
for count in seq_token_counts:
|
||||
boundaries.append(boundaries[-1] + count)
|
||||
n = ids.numel()
|
||||
return [ids[boundaries[i] : min(boundaries[i + 1], n)] for i in range(len(seq_token_counts))]
|
||||
if is_forward_context_available():
|
||||
slices = get_forward_context().ubatch_slices
|
||||
if slices is not None and len(slices) > 1 and not any(hasattr(s, "token_slice") for s in slices):
|
||||
boundaries = [0]
|
||||
for s in slices:
|
||||
boundaries.append(boundaries[-1] + s)
|
||||
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
|
||||
return [ids]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
positions: torch.Tensor | None = None,
|
||||
intermediate_tensors: Any = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict[str, Any]] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> OmniOutput:
|
||||
"""Decode codec codes into audio waveform.
|
||||
|
||||
input_ids layout per request: flat codes [num_codebooks * num_frames].
|
||||
Codes are codebook-major: [cb0_f0, cb0_f1, ..., cb0_fN, cb1_f0, ...].
|
||||
"""
|
||||
self._ensure_codec_loaded()
|
||||
assert self._codec is not None
|
||||
|
||||
q = self._num_codebooks
|
||||
sr_val = self._output_sample_rate
|
||||
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
|
||||
empty = torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
if input_ids is None or input_ids.numel() == 0:
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": [empty], "sr": [sr_tensor]},
|
||||
)
|
||||
|
||||
ids = input_ids.reshape(-1).to(dtype=torch.long)
|
||||
request_ids_list = self._split_request_ids(ids, kwargs.get("seq_token_counts"))
|
||||
|
||||
parsed: list[tuple[int, int]] = []
|
||||
valid_codes_qf: list[torch.Tensor] = []
|
||||
valid_indices: list[int] = []
|
||||
left_context_size = [0] * len(request_ids_list)
|
||||
if runtime_additional_information is not None:
|
||||
for i, info in enumerate(runtime_additional_information):
|
||||
if i >= len(left_context_size):
|
||||
break
|
||||
if "left_context_size" in info:
|
||||
left_context_size[i] = info["left_context_size"]
|
||||
|
||||
for i, req_ids in enumerate(request_ids_list):
|
||||
if req_ids.numel() < 1:
|
||||
parsed.append((0, 0))
|
||||
continue
|
||||
ctx_frames = left_context_size[i]
|
||||
flat = req_ids
|
||||
n = flat.numel()
|
||||
if n == 0 or n % q != 0:
|
||||
if n > 0:
|
||||
logger.warning(
|
||||
"DAC decoder input_ids length %d not divisible by num_codebooks %d; returning empty audio.",
|
||||
n,
|
||||
q,
|
||||
)
|
||||
parsed.append((0, 0))
|
||||
continue
|
||||
frames = n // q
|
||||
codes_qf = flat.reshape(q, frames)
|
||||
parsed.append((ctx_frames, frames))
|
||||
valid_codes_qf.append(codes_qf)
|
||||
valid_indices.append(i)
|
||||
|
||||
num_req = len(request_ids_list)
|
||||
if not valid_codes_qf:
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={
|
||||
"model_outputs": [empty] * num_req,
|
||||
"sr": [sr_tensor] * num_req,
|
||||
},
|
||||
)
|
||||
|
||||
if not self._logged_codec_stats:
|
||||
self._logged_codec_stats = True
|
||||
try:
|
||||
c = valid_codes_qf[0]
|
||||
logger.info(
|
||||
"DAC decoder: frames=%d q=%d uniq=%d range=[%d,%d] batch=%d",
|
||||
c.shape[1],
|
||||
q,
|
||||
int(torch.unique(c).numel()),
|
||||
int(c.min().item()),
|
||||
int(c.max().item()),
|
||||
len(valid_codes_qf),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Decode each request individually.
|
||||
wav_tensors: list[torch.Tensor] = []
|
||||
for codes_qf in valid_codes_qf:
|
||||
codes_bqf = codes_qf.unsqueeze(0) # [1, num_codebooks, num_frames]
|
||||
num_frames = codes_qf.shape[1]
|
||||
feature_lengths = torch.tensor([num_frames], device=codes_bqf.device)
|
||||
with torch.cuda.amp.autocast(enabled=False):
|
||||
wav, audio_lengths = self._codec.decode(codes_bqf, feature_lengths)
|
||||
# wav shape: [1, 1, wav_len]
|
||||
wav_tensors.append(wav.squeeze(0).squeeze(0)) # [wav_len]
|
||||
|
||||
audios: list[torch.Tensor] = [empty] * num_req
|
||||
srs = [sr_tensor] * num_req
|
||||
|
||||
for j, idx in enumerate(valid_indices):
|
||||
ctx_frames, actual_frames = parsed[idx]
|
||||
wav = wav_tensors[j]
|
||||
# Trim context frames (left overlap for streaming).
|
||||
if ctx_frames > 0:
|
||||
cut = ctx_frames * self._hop_length
|
||||
if cut < wav.shape[0]:
|
||||
wav = wav[cut:]
|
||||
else:
|
||||
logger.warning(
|
||||
"Context trim %d >= decoded length %d; returning empty audio.",
|
||||
cut,
|
||||
wav.shape[0],
|
||||
)
|
||||
continue
|
||||
if wav.shape[0] > 0:
|
||||
audios[idx] = wav.to(dtype=torch.float32).reshape(-1)
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": audios, "sr": srs},
|
||||
)
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
if not (isinstance(model_outputs, tuple) and len(model_outputs) == 2):
|
||||
raise TypeError(f"FishSpeechDACDecoder expected (audio_tensor, sr), got {type(model_outputs)}")
|
||||
audio_tensor, sr = model_outputs
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"model_outputs": audio_tensor, "sr": sr},
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# DAC codec weights are loaded lazily from codec.pth, not from the main checkpoint.
|
||||
return set()
|
||||
@@ -0,0 +1,470 @@
|
||||
"""Fast AR (audio_decoder) for Fish Speech S2 Pro.
|
||||
|
||||
4-layer transformer that predicts residual codebook codes (1..num_codebooks-1)
|
||||
autoregressively after each Slow AR step. Analogous to Qwen3 TTS's
|
||||
CodePredictor but with its own embedding table, RoPE, and output head.
|
||||
|
||||
Uses re-prefill (no KV cache): each AR step forwards the full growing
|
||||
sequence through the 4-layer transformer. This trades ~O(T^2) attention
|
||||
FLOPs (negligible for T=10, 4 layers) for zero KV cache management.
|
||||
|
||||
Optimisations:
|
||||
- torch.compile on model forward (kernel fusion)
|
||||
- Pre-allocated projection / embedding buffer
|
||||
- Pre-allocated position_ids
|
||||
- Inline sampling
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import is_pp_missing_parameter
|
||||
|
||||
from .configuration_fish_speech import FishSpeechFastARConfig, FishSpeechSlowARConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Standalone Fast AR Layers (no vLLM paged attention)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class _FastARAttention(nn.Module):
|
||||
"""Multi-head attention using F.scaled_dot_product_attention (SDPA).
|
||||
|
||||
Supports fused QKV, RoPE, optional q/k normalization, and native GQA.
|
||||
Input: [B, seq_len, hidden_size].
|
||||
"""
|
||||
|
||||
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_kv_heads = config.num_key_value_heads
|
||||
self.head_dim = config.head_dim
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self._use_gqa = self.num_kv_heads != self.num_heads
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=self.hidden_size,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.num_heads,
|
||||
total_num_kv_heads=self.num_kv_heads,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
input_size=self.num_heads * self.head_dim,
|
||||
output_size=self.hidden_size,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=config.max_position_embeddings,
|
||||
is_neox_style=False, # Fish Speech uses interleaved (GPT-J) RoPE
|
||||
rope_parameters={"rope_theta": config.rope_theta, "rope_type": "default"},
|
||||
)
|
||||
# Fish Speech Fast AR typically has attention_qk_norm=false.
|
||||
if config.attention_qk_norm:
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.q_norm = None
|
||||
self.k_norm = None
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states.reshape(bsz * seq_len, -1))
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
if self.q_norm is not None:
|
||||
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
|
||||
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
|
||||
|
||||
q, k = self.rotary_emb(position_ids, q, k)
|
||||
|
||||
q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
|
||||
k = k.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
|
||||
v = v.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
|
||||
|
||||
attn_out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
scale=self.scaling,
|
||||
is_causal=True,
|
||||
enable_gqa=self._use_gqa,
|
||||
)
|
||||
attn_out = attn_out.transpose(1, 2).reshape(bsz * seq_len, -1)
|
||||
output, _ = self.o_proj(attn_out)
|
||||
return output.view(bsz, seq_len, -1)
|
||||
|
||||
|
||||
class _FastARMLP(nn.Module):
|
||||
"""SiLU-gated MLP, matching Qwen3/LLaMA MLP structure."""
|
||||
|
||||
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_sizes=[config.intermediate_size] * 2,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
input_size=config.intermediate_size,
|
||||
output_size=config.hidden_size,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
gate, up = gate_up.chunk(2, dim=-1)
|
||||
x = F.silu(gate) * up
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class _FastARDecoderLayer(nn.Module):
|
||||
"""Transformer decoder layer for Fast AR (SDPA, no KV cache)."""
|
||||
|
||||
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.self_attn = _FastARAttention(config, prefix=f"{prefix}.self_attn")
|
||||
self.mlp = _FastARMLP(config, prefix=f"{prefix}.mlp")
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(hidden_states, position_ids)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Fast AR Transformer Model
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class FishSpeechFastARModel(nn.Module):
|
||||
"""4-layer transformer for residual codebook prediction (re-prefill)."""
|
||||
|
||||
def __init__(self, config: FishSpeechFastARConfig, *, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layers = nn.ModuleList(
|
||||
[_FastARDecoderLayer(config, prefix=f"{prefix}.layers.{i}") for i in range(config.num_hidden_layers)]
|
||||
)
|
||||
# NOTE: final norm is handled by FishSpeechFastAR.fast_norm (one norm weight
|
||||
# in checkpoint: audio_decoder.norm.weight → fast_ar.fast_norm.weight).
|
||||
|
||||
def forward(self, inputs_embeds: torch.Tensor, position_ids: torch.Tensor) -> torch.Tensor:
|
||||
hidden_states = inputs_embeds
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(hidden_states, position_ids)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped = name.replace(weight_name, param_name)
|
||||
if mapped.endswith(".bias") and mapped not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(mapped, self):
|
||||
continue
|
||||
if mapped.endswith("scale"):
|
||||
mapped = maybe_remap_kv_scale_name(mapped, params_dict)
|
||||
if mapped is None:
|
||||
continue
|
||||
param = params_dict.get(mapped)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(mapped)
|
||||
break
|
||||
else:
|
||||
mapped = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if mapped is None:
|
||||
continue
|
||||
if name.endswith(".bias") and mapped not in params_dict:
|
||||
continue
|
||||
if is_pp_missing_parameter(mapped, self):
|
||||
continue
|
||||
param = params_dict.get(mapped)
|
||||
if param is None:
|
||||
continue
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(mapped)
|
||||
return loaded_params
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Fast AR Wrapper (optimised re-prefill + torch.compile)
|
||||
# ===================================================================
|
||||
|
||||
|
||||
class FishSpeechFastAR(nn.Module):
|
||||
"""vLLM-native Fast AR for Fish Speech (residual codebooks).
|
||||
|
||||
Re-prefill approach: each AR step forwards the full growing sequence
|
||||
through the 4-layer transformer. No KV cache needed.
|
||||
|
||||
Optimisations over baseline:
|
||||
1. torch.compile on model forward -- kernel fusion.
|
||||
2. Pre-allocated embedding buffer [B, max_seq, H].
|
||||
3. Pre-allocated position_ids.
|
||||
4. Inline sampling.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
config: FishSpeechFastARConfig,
|
||||
slow_ar_config: FishSpeechSlowARConfig,
|
||||
prefix: str = "fast_ar",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._vllm_config = vllm_config
|
||||
self.config = config
|
||||
self.slow_ar_config = slow_ar_config
|
||||
|
||||
self.model = FishSpeechFastARModel(config, prefix=f"{prefix}.model")
|
||||
|
||||
# Codebook embeddings for Fast AR (separate from Slow AR's codebook_embeddings).
|
||||
self.fast_embeddings = nn.Embedding(config.vocab_size, config.hidden_size)
|
||||
|
||||
# Output projection for residual codes.
|
||||
self.fast_output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
||||
|
||||
# Layer norm for Fast AR output.
|
||||
self.fast_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
# Projection from Slow AR hidden dim → Fast AR hidden dim.
|
||||
# s2-pro has same dim for both (2560) and no fast_project_in in checkpoint,
|
||||
# so use identity when dims match.
|
||||
if slow_ar_config.hidden_size != config.hidden_size:
|
||||
self.fast_project_in = nn.Linear(slow_ar_config.hidden_size, config.hidden_size, bias=True)
|
||||
else:
|
||||
self.fast_project_in = nn.Identity()
|
||||
|
||||
self._num_codebooks = config.num_codebooks
|
||||
self._fast_dim = config.hidden_size
|
||||
|
||||
# Pre-allocated buffers (lazily initialised on first forward).
|
||||
self._embed_buf: torch.Tensor | None = None
|
||||
self._pos_ids: torch.Tensor | None = None
|
||||
self._compiled_model_fwd: object | None = None
|
||||
|
||||
def _ensure_buffers(self, bsz: int, device: torch.device, dtype: torch.dtype) -> None:
|
||||
max_seq = self._num_codebooks + 1 # hidden_state + num_codebooks codes
|
||||
if (
|
||||
self._embed_buf is not None
|
||||
and self._embed_buf.shape[0] >= bsz
|
||||
and self._embed_buf.device == device
|
||||
and self._embed_buf.dtype == dtype
|
||||
):
|
||||
return
|
||||
self._embed_buf = torch.zeros(bsz, max_seq, self._fast_dim, dtype=dtype, device=device)
|
||||
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device)
|
||||
|
||||
def _setup_compile(self) -> None:
|
||||
if self._compiled_model_fwd is not None:
|
||||
return
|
||||
# TODO: Enable torch.compile for performance. Eager for now to avoid
|
||||
# potential graph-break issues during initial bring-up.
|
||||
self._compiled_model_fwd = self.model.forward
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
self,
|
||||
slow_ar_hidden: torch.Tensor,
|
||||
semantic_token_id: torch.Tensor,
|
||||
*,
|
||||
do_sample: bool = True,
|
||||
temperature: float = 0.8,
|
||||
top_k: int = 30,
|
||||
top_p: float = 0.9,
|
||||
) -> torch.Tensor:
|
||||
"""Predict residual codebook codes 0..num_codebooks-1 autoregressively.
|
||||
|
||||
Args:
|
||||
slow_ar_hidden: [B, hidden_size] last hidden state from Slow AR.
|
||||
semantic_token_id: [B] or [B, 1] sampled semantic token IDs (in vocab space).
|
||||
|
||||
Returns:
|
||||
all_codes: [B, num_codebooks] codes for codebooks 0..num_codebooks-1.
|
||||
Code 0 is the semantic code (token - semantic_begin_id).
|
||||
"""
|
||||
bsz = int(slow_ar_hidden.shape[0])
|
||||
num_cb = self._num_codebooks
|
||||
device = slow_ar_hidden.device
|
||||
dtype = slow_ar_hidden.dtype
|
||||
|
||||
semantic_begin = self.slow_ar_config.semantic_begin_id
|
||||
semantic_end = self.slow_ar_config.semantic_end_id
|
||||
codebook_size = semantic_end - semantic_begin + 1 # 4096
|
||||
# Convert vocab-space semantic token to codebook index.
|
||||
# Clamp to valid range: im_end or other non-semantic tokens map to 0 (pad).
|
||||
semantic_code = (semantic_token_id.reshape(bsz) - semantic_begin).clamp(min=0, max=codebook_size - 1)
|
||||
|
||||
all_codes = torch.empty(bsz, num_cb, dtype=torch.long, device=device)
|
||||
all_codes[:, 0] = semantic_code
|
||||
|
||||
self._ensure_buffers(bsz, device, dtype)
|
||||
self._setup_compile()
|
||||
|
||||
embed_buf = self._embed_buf
|
||||
pos_ids = self._pos_ids
|
||||
model_fwd = self._compiled_model_fwd
|
||||
|
||||
# Position 0: projected Slow AR hidden state.
|
||||
projected = self.fast_project_in(slow_ar_hidden.reshape(bsz, -1))
|
||||
embed_buf[:bsz, 0, :] = projected
|
||||
|
||||
# Position 1: embedding of semantic code.
|
||||
code_embed = self.fast_embeddings(semantic_code.clamp(min=0))
|
||||
embed_buf[:bsz, 1, :] = code_embed
|
||||
|
||||
use_sampling = do_sample and temperature > 0
|
||||
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
|
||||
|
||||
# Residual codebook size (1024) vs semantic codebook size (4096).
|
||||
# The fast_output head has codebook_size (4096) outputs, but residual
|
||||
# codebooks only have 1024 entries. Truncate logits for steps > 0.
|
||||
residual_codebook_size = 1024
|
||||
|
||||
for step in range(1, num_cb):
|
||||
seq_len = step + 1
|
||||
step_input = embed_buf[:bsz, :seq_len, :]
|
||||
step_pos_ids = pos_ids[:seq_len] if bsz == 1 else pos_ids[:seq_len].repeat(bsz)
|
||||
|
||||
hidden_out = model_fwd(step_input, step_pos_ids)
|
||||
logits = self.fast_output(self.fast_norm(hidden_out[:, -1, :]))
|
||||
|
||||
# Residual codebooks (step >= 1) only have 1024 entries.
|
||||
if step >= 1:
|
||||
logits = logits[:, :residual_codebook_size]
|
||||
|
||||
if use_sampling:
|
||||
scaled = logits * inv_temperature
|
||||
if top_k > 0:
|
||||
topk_vals, _ = scaled.topk(min(top_k, scaled.shape[-1]), dim=-1)
|
||||
scaled = scaled.masked_fill(scaled < topk_vals[:, -1:], float("-inf"))
|
||||
if top_p < 1.0:
|
||||
sorted_logits, sorted_indices = torch.sort(scaled, descending=True)
|
||||
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
|
||||
sorted_indices_to_remove = cumulative_probs - F.softmax(sorted_logits, dim=-1) >= top_p
|
||||
sorted_logits[sorted_indices_to_remove] = float("-inf")
|
||||
scaled = sorted_logits.scatter(1, sorted_indices, sorted_logits)
|
||||
probs = F.softmax(scaled, dim=-1)
|
||||
next_ids = torch.multinomial(probs, num_samples=1)
|
||||
else:
|
||||
next_ids = logits.argmax(dim=-1, keepdim=True)
|
||||
|
||||
all_codes[:, step] = next_ids.reshape(bsz)
|
||||
|
||||
if step < num_cb - 1:
|
||||
new_embed = self.fast_embeddings(next_ids.reshape(bsz))
|
||||
embed_buf[:bsz, step + 1, :] = new_embed
|
||||
|
||||
return all_codes
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights -- Fast AR weights have already been remapped by the parent."""
|
||||
with set_current_vllm_config(self._vllm_config):
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded: set[str] = set()
|
||||
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
handled = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped = name.replace(weight_name, param_name)
|
||||
if mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[mapped]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded.add(mapped)
|
||||
handled = True
|
||||
break
|
||||
|
||||
if handled:
|
||||
continue
|
||||
|
||||
if name in params_dict:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded.add(name)
|
||||
|
||||
return loaded
|
||||
@@ -0,0 +1,669 @@
|
||||
"""Fish Speech S2 Pro -- Slow AR model (Stage 0).
|
||||
|
||||
Uses vLLM's ``Qwen3Model`` as the transformer backbone. Adds:
|
||||
- Multi-codebook input embedding (text + summed codebook embeddings at
|
||||
semantic-token positions).
|
||||
- Semantic logit masking.
|
||||
- Nested Fast AR for residual codebook prediction (``talker_mtp``).
|
||||
- ``preprocess`` / ``postprocess`` hooks for vLLM-omni's AR scheduler.
|
||||
|
||||
Analogous to ``Qwen3TTSTalkerForConditionalGeneration`` in qwen3_tts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoTokenizer
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.qwen3 import Qwen3Model
|
||||
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
from .configuration_fish_speech import FishSpeechConfig, FishSpeechFastARConfig, FishSpeechSlowARConfig
|
||||
from .fish_speech_fast_ar import FishSpeechFastAR
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _remap_fish_speech_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
n_head: int,
|
||||
n_local_heads: int,
|
||||
head_dim: int,
|
||||
fast_n_head: int,
|
||||
fast_n_local_heads: int,
|
||||
fast_head_dim: int,
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Transform Fish Speech HF weight names/values to Qwen3-compatible format.
|
||||
|
||||
Key transformations:
|
||||
- ``wqkv`` → split into ``q_proj``, ``k_proj``, ``v_proj``
|
||||
- ``wo`` → ``o_proj``
|
||||
- ``w1`` → ``gate_proj``, ``w3`` → ``up_proj``, ``w2`` → ``down_proj``
|
||||
- ``attention_norm`` → ``input_layernorm``
|
||||
- ``ffn_norm`` → ``post_attention_layernorm``
|
||||
- ``text_model.model.embeddings`` → ``model.embed_tokens``
|
||||
- ``audio_decoder.*`` → ``fast_ar.*`` with similar transforms
|
||||
"""
|
||||
q_size_text = n_head * head_dim
|
||||
kv_size_text = n_local_heads * head_dim
|
||||
q_size_fast = fast_n_head * fast_head_dim
|
||||
kv_size_fast = fast_n_local_heads * fast_head_dim
|
||||
|
||||
for name, tensor in weights:
|
||||
# --- Text model (Slow AR) ---
|
||||
if name.startswith("text_model.model."):
|
||||
suffix = name[len("text_model.model.") :]
|
||||
|
||||
# Embeddings
|
||||
if suffix == "embeddings.weight":
|
||||
yield "model.embed_tokens.weight", tensor
|
||||
continue
|
||||
|
||||
# Norm
|
||||
if suffix == "norm.weight":
|
||||
yield "model.norm.weight", tensor
|
||||
continue
|
||||
|
||||
# Layer weights
|
||||
if suffix.startswith("layers."):
|
||||
# layers.{N}.attention.wqkv.weight → split into q/k/v
|
||||
if ".attention.wqkv.weight" in suffix:
|
||||
layer_prefix = suffix.split(".attention.wqkv.weight")[0]
|
||||
q = tensor[:q_size_text, :]
|
||||
k = tensor[q_size_text : q_size_text + kv_size_text, :]
|
||||
v = tensor[q_size_text + kv_size_text :, :]
|
||||
yield f"model.{layer_prefix}.self_attn.q_proj.weight", q
|
||||
yield f"model.{layer_prefix}.self_attn.k_proj.weight", k
|
||||
yield f"model.{layer_prefix}.self_attn.v_proj.weight", v
|
||||
continue
|
||||
|
||||
new_suffix = suffix
|
||||
new_suffix = new_suffix.replace(".attention.wo.", ".self_attn.o_proj.")
|
||||
new_suffix = new_suffix.replace(".attention.q_norm.", ".self_attn.q_norm.")
|
||||
new_suffix = new_suffix.replace(".attention.k_norm.", ".self_attn.k_norm.")
|
||||
new_suffix = new_suffix.replace(".attention_norm.", ".input_layernorm.")
|
||||
new_suffix = new_suffix.replace(".feed_forward.w1.", ".mlp.gate_proj.")
|
||||
new_suffix = new_suffix.replace(".feed_forward.w3.", ".mlp.up_proj.")
|
||||
new_suffix = new_suffix.replace(".feed_forward.w2.", ".mlp.down_proj.")
|
||||
new_suffix = new_suffix.replace(".ffn_norm.", ".post_attention_layernorm.")
|
||||
yield f"model.{new_suffix}", tensor
|
||||
continue
|
||||
|
||||
# Fallback for any other text_model.model.* weights
|
||||
yield f"model.{suffix}", tensor
|
||||
continue
|
||||
|
||||
# --- Audio decoder (Fast AR) ---
|
||||
if name.startswith("audio_decoder."):
|
||||
suffix = name[len("audio_decoder.") :]
|
||||
|
||||
# Codebook embeddings (belongs to the main model, not Fast AR).
|
||||
if suffix == "codebook_embeddings.weight":
|
||||
yield "codebook_embeddings.weight", tensor
|
||||
continue
|
||||
|
||||
# Fast AR embeddings, output, norm.
|
||||
if suffix == "embeddings.weight":
|
||||
yield "fast_ar.fast_embeddings.weight", tensor
|
||||
continue
|
||||
if suffix == "output.weight":
|
||||
yield "fast_ar.fast_output.weight", tensor
|
||||
continue
|
||||
if suffix == "norm.weight":
|
||||
yield "fast_ar.fast_norm.weight", tensor
|
||||
continue
|
||||
|
||||
# Fast AR projection in.
|
||||
if suffix.startswith("fast_project_in."):
|
||||
yield f"fast_ar.fast_project_in.{suffix[len('fast_project_in.') :]}", tensor
|
||||
continue
|
||||
|
||||
# Fast AR layer weights.
|
||||
if suffix.startswith("layers."):
|
||||
if ".attention.wqkv.weight" in suffix:
|
||||
layer_prefix = suffix.split(".attention.wqkv.weight")[0]
|
||||
q = tensor[:q_size_fast, :]
|
||||
k = tensor[q_size_fast : q_size_fast + kv_size_fast, :]
|
||||
v = tensor[q_size_fast + kv_size_fast :, :]
|
||||
yield f"fast_ar.model.{layer_prefix}.self_attn.q_proj.weight", q
|
||||
yield f"fast_ar.model.{layer_prefix}.self_attn.k_proj.weight", k
|
||||
yield f"fast_ar.model.{layer_prefix}.self_attn.v_proj.weight", v
|
||||
continue
|
||||
|
||||
new_suffix = suffix
|
||||
new_suffix = new_suffix.replace(".attention.wo.", ".self_attn.o_proj.")
|
||||
new_suffix = new_suffix.replace(".attention.q_norm.", ".self_attn.q_norm.")
|
||||
new_suffix = new_suffix.replace(".attention.k_norm.", ".self_attn.k_norm.")
|
||||
new_suffix = new_suffix.replace(".attention_norm.", ".input_layernorm.")
|
||||
new_suffix = new_suffix.replace(".feed_forward.w1.", ".mlp.gate_proj.")
|
||||
new_suffix = new_suffix.replace(".feed_forward.w3.", ".mlp.up_proj.")
|
||||
new_suffix = new_suffix.replace(".feed_forward.w2.", ".mlp.down_proj.")
|
||||
new_suffix = new_suffix.replace(".ffn_norm.", ".post_attention_layernorm.")
|
||||
yield f"fast_ar.model.{new_suffix}", tensor
|
||||
continue
|
||||
|
||||
yield f"fast_ar.{suffix}", tensor
|
||||
continue
|
||||
|
||||
# Pass through any other weights.
|
||||
yield name, tensor
|
||||
|
||||
|
||||
class FishSpeechSlowARForConditionalGeneration(nn.Module):
|
||||
"""vLLM-AR Slow AR model for Fish Speech S2 Pro.
|
||||
|
||||
Stage 0: text → semantic tokens (+ residual codebook codes via Fast AR).
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.model_path = vllm_config.model_config.model
|
||||
|
||||
config: FishSpeechConfig = vllm_config.model_config.hf_config # type: ignore[assignment]
|
||||
self.config = config
|
||||
self.text_config: FishSpeechSlowARConfig = config.text_config
|
||||
self.fast_ar_config: FishSpeechFastARConfig = config.audio_decoder_config
|
||||
|
||||
self._semantic_begin_id = int(config.semantic_start_token_id)
|
||||
self._semantic_end_id = int(config.semantic_end_token_id)
|
||||
self._audio_pad_token_id = int(config.audio_pad_token_id)
|
||||
self._codebook_size = int(self.text_config.codebook_size)
|
||||
self._num_codebooks = int(self.text_config.num_codebooks)
|
||||
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = True
|
||||
self.has_postprocess = True
|
||||
self.mtp_hidden_size = int(self.text_config.hidden_size)
|
||||
self.talker_mtp_output_key = "audio_codes"
|
||||
|
||||
# Qwen3 transformer backbone.
|
||||
self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
|
||||
|
||||
# Fish Speech uses interleaved (GPT-J) RoPE, not NeoX style.
|
||||
# vLLM's Qwen3Attention defaults to NeoX (is_neox_style=True).
|
||||
# Replace with interleaved RoPE to match training.
|
||||
self._fix_rope_style()
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.text_config.vocab_size,
|
||||
self.text_config.hidden_size,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.logits_processor = LogitsProcessor(self.text_config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = self.model.make_empty_intermediate_tensors
|
||||
|
||||
# Multi-codebook embedding table: codebook_size * num_codebooks entries.
|
||||
self.codebook_embeddings = nn.Embedding(
|
||||
self._codebook_size * self._num_codebooks,
|
||||
self.text_config.hidden_size,
|
||||
)
|
||||
|
||||
# Fast AR (residual codebook predictor).
|
||||
predictor_compilation = dataclasses.replace(vllm_config.compilation_config)
|
||||
predictor_compilation.static_forward_context = {}
|
||||
self._fast_ar_vllm_config = dataclasses.replace(vllm_config, compilation_config=predictor_compilation)
|
||||
from vllm.config.vllm import set_current_vllm_config as _set_cfg
|
||||
|
||||
with _set_cfg(self._fast_ar_vllm_config):
|
||||
self.fast_ar = FishSpeechFastAR(
|
||||
vllm_config=self._fast_ar_vllm_config,
|
||||
config=self.fast_ar_config,
|
||||
slow_ar_config=self.text_config,
|
||||
prefix="fast_ar",
|
||||
)
|
||||
|
||||
# Constant logit mask: allow only semantic tokens + im_end.
|
||||
vocab = int(self.text_config.vocab_size)
|
||||
semantic_mask = torch.zeros((vocab,), dtype=torch.bool)
|
||||
lo = self._semantic_begin_id
|
||||
hi = min(self._semantic_end_id + 1, vocab)
|
||||
if hi > lo:
|
||||
semantic_mask[lo:hi] = True
|
||||
# Also allow <|im_end|> (token 151645 in Qwen3 tokeniser).
|
||||
im_end_id = 151645
|
||||
if im_end_id < vocab:
|
||||
semantic_mask[im_end_id] = True
|
||||
self.register_buffer("_semantic_allowed_mask", semantic_mask, persistent=False)
|
||||
|
||||
# Tokeniser (lazy).
|
||||
self._tokenizer = None
|
||||
|
||||
def _fix_rope_style(self) -> None:
|
||||
"""Replace NeoX-style RoPE with interleaved (GPT-J) style.
|
||||
|
||||
Fish Speech was trained with interleaved RoPE (complex-number pairs),
|
||||
but vLLM's Qwen3Attention defaults to NeoX style. We rebuild the
|
||||
rotary embedding with ``is_neox_style=False`` for each attention layer.
|
||||
"""
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
for layer in self.model.layers:
|
||||
attn = layer.self_attn
|
||||
# Extract parameters from the existing RoPE to rebuild it.
|
||||
head_dim = attn.head_dim
|
||||
max_position = self.text_config.max_position_embeddings
|
||||
rope_params = getattr(self.text_config, "rope_scaling", None) or {}
|
||||
rope_params.setdefault("rope_theta", getattr(self.text_config, "rope_theta", 1000000.0))
|
||||
attn.rotary_emb = get_rope(
|
||||
head_size=head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=False,
|
||||
rope_parameters=rope_params,
|
||||
)
|
||||
logger.info("Fixed RoPE style to interleaved (GPT-J) for %d layers", len(self.model.layers))
|
||||
|
||||
# -------------------- vLLM required hooks --------------------
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**_: Any,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor | OmniOutput,
|
||||
sampling_metadata: Any = None,
|
||||
) -> torch.Tensor | None:
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if hidden_states is None:
|
||||
return None
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
if logits is None:
|
||||
return None
|
||||
|
||||
# Mask to semantic tokens + im_end only.
|
||||
logits = logits.masked_fill(~self._semantic_allowed_mask, float("-inf"))
|
||||
return logits
|
||||
|
||||
# -------------------- Omni multimodal output plumbing --------------------
|
||||
|
||||
def make_omni_output(self, model_outputs: torch.Tensor | OmniOutput, **kwargs: Any) -> OmniOutput:
|
||||
if isinstance(model_outputs, OmniOutput):
|
||||
return model_outputs
|
||||
|
||||
hidden = model_outputs
|
||||
info_dicts = kwargs.get("model_intermediate_buffer")
|
||||
if info_dicts is None:
|
||||
info_dicts = kwargs.get("runtime_additional_information") or []
|
||||
|
||||
audio_codes_list: list[torch.Tensor] = []
|
||||
for info in info_dicts:
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
ac = info.get("audio_codes")
|
||||
if isinstance(ac, torch.Tensor):
|
||||
audio_codes_list.append(ac)
|
||||
|
||||
if not audio_codes_list:
|
||||
logger.debug("make_omni_output: no audio_codes found in info_dicts (len=%d)", len(info_dicts))
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs={})
|
||||
|
||||
audio_codes = torch.cat(audio_codes_list, dim=0)
|
||||
span_len = int(audio_codes.shape[0])
|
||||
hidden = hidden[:span_len]
|
||||
mm: dict[str, torch.Tensor] = {"audio_codes": audio_codes}
|
||||
return OmniOutput(text_hidden_states=hidden, multimodal_outputs=mm)
|
||||
|
||||
# -------------------- preprocess / postprocess --------------------
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
input_embeds: torch.Tensor | None,
|
||||
**info_dict: Any,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]:
|
||||
additional_information = info_dict.get("additional_information")
|
||||
if isinstance(additional_information, dict):
|
||||
merged: dict[str, Any] = {k: v for k, v in info_dict.items() if k != "additional_information"}
|
||||
for k, v in additional_information.items():
|
||||
merged.setdefault(k, v)
|
||||
info_dict = merged
|
||||
|
||||
span_len = int(input_ids.shape[0])
|
||||
if span_len <= 0:
|
||||
return input_ids, input_embeds if input_embeds is not None else self.embed_input_ids(input_ids), {}
|
||||
|
||||
if span_len > 1:
|
||||
# --- Prefill ---
|
||||
prompt_embeds_cpu = info_dict.get("slow_ar_prompt_embeds")
|
||||
is_first_prefill = not isinstance(prompt_embeds_cpu, torch.Tensor) or prompt_embeds_cpu.ndim != 2
|
||||
dev = input_ids.device
|
||||
|
||||
if is_first_prefill:
|
||||
prompt_embeds = self._build_prefill_embeds(input_ids, info_dict)
|
||||
prompt_embeds_cpu = prompt_embeds.detach().to("cpu").contiguous()
|
||||
|
||||
info_update: dict[str, Any] = {
|
||||
"slow_ar_prompt_embeds": prompt_embeds_cpu,
|
||||
"prefill_offset": 0,
|
||||
}
|
||||
|
||||
take = prompt_embeds_cpu[:span_len]
|
||||
if int(take.shape[0]) < span_len:
|
||||
pad_n = span_len - int(take.shape[0])
|
||||
pad_embed = self.embed_input_ids(
|
||||
torch.tensor([self._audio_pad_token_id], device=dev, dtype=torch.long)
|
||||
).reshape(1, -1)
|
||||
take = torch.cat([take, pad_embed.detach().cpu().expand(pad_n, -1)], dim=0)
|
||||
prompt_embeds = take.to(device=dev, dtype=torch.bfloat16)
|
||||
info_update["prefill_offset"] = span_len
|
||||
|
||||
zeros = torch.zeros(
|
||||
(prompt_embeds.shape[0], self._num_codebooks),
|
||||
device=dev,
|
||||
dtype=torch.long,
|
||||
)
|
||||
info_update["audio_codes"] = zeros
|
||||
|
||||
input_ids_out = input_ids.clone()
|
||||
input_ids_out[:] = self._audio_pad_token_id
|
||||
return input_ids_out, prompt_embeds, info_update
|
||||
|
||||
else:
|
||||
# Subsequent prefill chunk.
|
||||
offset = int(info_dict.get("prefill_offset", 0) or 0)
|
||||
s = max(0, min(offset, int(prompt_embeds_cpu.shape[0])))
|
||||
e = max(0, min(offset + span_len, int(prompt_embeds_cpu.shape[0])))
|
||||
take = prompt_embeds_cpu[s:e]
|
||||
if int(take.shape[0]) < span_len:
|
||||
pad_n = span_len - int(take.shape[0])
|
||||
pad_embed = self.embed_input_ids(
|
||||
torch.tensor([self._audio_pad_token_id], device=dev, dtype=torch.long)
|
||||
).reshape(1, -1)
|
||||
take = torch.cat([take, pad_embed.detach().cpu().expand(pad_n, -1)], dim=0)
|
||||
prompt_embeds = take.to(device=dev, dtype=torch.bfloat16)
|
||||
|
||||
zeros = torch.zeros((prompt_embeds.shape[0], self._num_codebooks), device=dev, dtype=torch.long)
|
||||
return (
|
||||
input_ids.clone().fill_(self._audio_pad_token_id),
|
||||
prompt_embeds,
|
||||
{
|
||||
"prefill_offset": offset + span_len,
|
||||
"audio_codes": zeros,
|
||||
},
|
||||
)
|
||||
|
||||
# --- Decode: span_len == 1 ---
|
||||
dev = input_ids.device
|
||||
|
||||
last_hidden_cpu = info_dict.get("last_slow_ar_hidden")
|
||||
if not isinstance(last_hidden_cpu, torch.Tensor):
|
||||
# First decode step after prefill -- just embed the token directly.
|
||||
logger.warning(
|
||||
"preprocess decode: last_slow_ar_hidden not found (keys=%s), "
|
||||
"returning plain embed (mtp_inputs will NOT be set)",
|
||||
list(info_dict.keys()),
|
||||
)
|
||||
embeds = self.embed_input_ids(input_ids.reshape(1, 1).to(torch.long)).reshape(1, -1)
|
||||
return input_ids, embeds.to(dtype=torch.bfloat16), {}
|
||||
|
||||
token_embed = self.embed_input_ids(input_ids.reshape(1, 1).to(torch.long)).to(
|
||||
device=dev, dtype=torch.bfloat16
|
||||
) # [1, 1, H]
|
||||
|
||||
# Codebook embeddings are added by talker_mtp (using the CURRENT step's
|
||||
# FastAR hidden state) instead of here (which would use the PREVIOUS step's
|
||||
# codes, one step too old). Preprocess always returns plain text embed.
|
||||
inputs_embeds_out = token_embed.reshape(1, -1)
|
||||
|
||||
info_update = {
|
||||
"mtp_inputs": (
|
||||
last_hidden_cpu.to(device=dev, dtype=torch.bfloat16).reshape(1, -1),
|
||||
torch.zeros(1, self.text_config.hidden_size, device=dev, dtype=torch.bfloat16),
|
||||
),
|
||||
}
|
||||
return input_ids, inputs_embeds_out, info_update
|
||||
|
||||
def postprocess(self, hidden_states: torch.Tensor, **_: Any) -> dict[str, Any]:
|
||||
if hidden_states.numel() == 0:
|
||||
logger.debug("postprocess: empty hidden_states")
|
||||
return {}
|
||||
last = hidden_states[-1, :].detach().to("cpu").contiguous()
|
||||
logger.debug("postprocess: saved last_slow_ar_hidden shape=%s", tuple(last.shape))
|
||||
return {"last_slow_ar_hidden": last}
|
||||
|
||||
# -------------------- prompt construction --------------------
|
||||
|
||||
def _get_tokenizer(self):
|
||||
if self._tokenizer is None:
|
||||
self._tokenizer = AutoTokenizer.from_pretrained(self.model_path, trust_remote_code=True)
|
||||
return self._tokenizer
|
||||
|
||||
def _build_prefill_embeds(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
info_dict: dict[str, Any],
|
||||
) -> torch.Tensor:
|
||||
"""Build prefill embeddings, adding codebook embeddings at semantic positions.
|
||||
|
||||
For text-only prefill (no reference audio), this is just embed_input_ids.
|
||||
For voice cloning, reference codes are embedded with codebook offsets.
|
||||
"""
|
||||
dev = input_ids.device
|
||||
# Basic text embeddings.
|
||||
base_embeds = self.embed_input_ids(input_ids.reshape(1, -1).to(torch.long)) # [1, T, H]
|
||||
|
||||
# Check for reference codebook codes (for voice cloning).
|
||||
ref_codes = info_dict.get("ref_codes")
|
||||
if not isinstance(ref_codes, torch.Tensor) or ref_codes.numel() == 0:
|
||||
return base_embeds.squeeze(0).to(dtype=torch.bfloat16)
|
||||
|
||||
# ref_codes: [T_ref, num_codebooks] -- codebook codes for reference audio positions.
|
||||
ref_codes = ref_codes.to(device=dev, dtype=torch.long)
|
||||
ref_positions = info_dict.get("ref_positions")
|
||||
if not isinstance(ref_positions, torch.Tensor):
|
||||
return base_embeds.squeeze(0).to(dtype=torch.bfloat16)
|
||||
|
||||
ref_positions = ref_positions.to(device=dev, dtype=torch.long).reshape(-1)
|
||||
seq_len = int(input_ids.shape[0])
|
||||
codebook_sum = torch.zeros_like(base_embeds) # [1, T, H]
|
||||
|
||||
for pos_idx in range(int(ref_positions.shape[0])):
|
||||
pos = int(ref_positions[pos_idx].item())
|
||||
if pos < 0 or pos >= seq_len:
|
||||
continue
|
||||
for cb_idx in range(min(int(ref_codes.shape[1]), self._num_codebooks)):
|
||||
code = ref_codes[pos_idx, cb_idx].clamp(min=0)
|
||||
code_with_offset = code + cb_idx * self._codebook_size
|
||||
emb = self.codebook_embeddings(code_with_offset.unsqueeze(0))
|
||||
codebook_sum[0, pos, :] += emb.squeeze(0).to(dtype=base_embeds.dtype)
|
||||
|
||||
result = base_embeds + codebook_sum
|
||||
return result.squeeze(0).to(dtype=torch.bfloat16)
|
||||
|
||||
# -------------------- GPU-side MTP fast-path --------------------
|
||||
|
||||
@torch.inference_mode()
|
||||
def talker_mtp(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
input_embeds: torch.Tensor,
|
||||
last_talker_hidden: torch.Tensor,
|
||||
text_step: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GPU fast-path: run Fast AR to predict residual codebook codes.
|
||||
|
||||
Returns (inputs_embeds, audio_codes).
|
||||
|
||||
The embedding is: text_embed(token) + sum(codebook_embed(code_i + i * codebook_size))
|
||||
where codes come from FastAR(last_talker_hidden).
|
||||
|
||||
This matches the reference Fish Speech inference flow:
|
||||
- At step t, the Slow AR embedding includes codes from FastAR(hidden_{t-1})
|
||||
- last_talker_hidden IS hidden_{t-1} (from postprocess of the previous step)
|
||||
- Preprocess provides the plain text embed; we add codebook embeddings here
|
||||
"""
|
||||
bsz = int(input_ids.shape[0])
|
||||
dev = input_embeds.device
|
||||
|
||||
input_ids = input_ids.reshape(bsz, 1).to(dtype=torch.long, device=dev)
|
||||
past_hidden = last_talker_hidden.reshape(bsz, -1).to(dtype=torch.bfloat16, device=dev)
|
||||
|
||||
# Run Fast AR to predict all num_codebooks codes.
|
||||
audio_codes = self.fast_ar(
|
||||
slow_ar_hidden=past_hidden,
|
||||
semantic_token_id=input_ids.reshape(bsz),
|
||||
do_sample=True,
|
||||
temperature=0.8,
|
||||
top_k=30,
|
||||
top_p=0.9,
|
||||
) # [B, num_codebooks]
|
||||
|
||||
# Add codebook embeddings to the input embedding (from preprocess).
|
||||
# This ensures the Slow AR sees codes from FastAR(hidden_{t-1}).
|
||||
inputs_embeds_out = input_embeds.reshape(bsz, -1).clone()
|
||||
|
||||
for b in range(bsz):
|
||||
token_id = int(input_ids[b, 0].item())
|
||||
is_semantic = self._semantic_begin_id <= token_id <= self._semantic_end_id
|
||||
if is_semantic:
|
||||
codes = audio_codes[b] # [num_codebooks]
|
||||
codebook_sum = torch.zeros(self.text_config.hidden_size, device=dev, dtype=torch.bfloat16)
|
||||
for i in range(self._num_codebooks):
|
||||
code_with_offset = codes[i].clamp(min=0) + i * self._codebook_size
|
||||
emb = self.codebook_embeddings(code_with_offset.unsqueeze(0))
|
||||
codebook_sum += emb.squeeze(0).to(dtype=torch.bfloat16)
|
||||
|
||||
# Normalize by sqrt(num_codebooks + 1) as in the reference model
|
||||
# (scale_codebook_embeddings=True for fish_qwen3_omni).
|
||||
inputs_embeds_out[b] = (inputs_embeds_out[b] + codebook_sum) / math.sqrt(self._num_codebooks + 1)
|
||||
|
||||
return inputs_embeds_out, audio_codes.to(dtype=torch.long)
|
||||
|
||||
# -------------------- Prompt length estimation --------------------
|
||||
|
||||
@staticmethod
|
||||
def estimate_prompt_len_from_additional_information(
|
||||
additional_information: dict[str, Any] | None,
|
||||
**kwargs: Any,
|
||||
) -> int:
|
||||
"""Estimate prompt length for placeholder allocation."""
|
||||
info = additional_information or {}
|
||||
text = info.get("text", [""])[0] if isinstance(info.get("text"), list) else info.get("text", "")
|
||||
# Conservative estimate: tokenize text length + overhead.
|
||||
return max(2, len(str(text)) // 2 + 64)
|
||||
|
||||
# -------------------- Weight loading --------------------
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights with Fish Speech → Qwen3 format transformation.
|
||||
|
||||
Transforms weight names (wqkv → q/k/v split, w1/w2/w3 → gate/up/down,
|
||||
etc.) and routes to the correct sub-modules.
|
||||
"""
|
||||
n_head = self.text_config.num_attention_heads
|
||||
n_local_heads = self.text_config.num_key_value_heads
|
||||
head_dim = self.text_config.head_dim
|
||||
fast_n_head = self.fast_ar_config.num_attention_heads
|
||||
fast_n_local_heads = self.fast_ar_config.num_key_value_heads
|
||||
fast_head_dim = self.fast_ar_config.head_dim
|
||||
|
||||
remapped = _remap_fish_speech_weights(
|
||||
weights,
|
||||
n_head,
|
||||
n_local_heads,
|
||||
head_dim,
|
||||
fast_n_head,
|
||||
fast_n_local_heads,
|
||||
fast_head_dim,
|
||||
)
|
||||
|
||||
# Qwen3Model uses stacked_params_mapping for q/k/v → qkv_proj
|
||||
# and gate/up → gate_up_proj. Feed the remapped weights through
|
||||
# the standard Qwen3 loading path.
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
("qkv_proj", "v_proj", "v"),
|
||||
("gate_up_proj", "gate_proj", 0),
|
||||
("gate_up_proj", "up_proj", 1),
|
||||
]
|
||||
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
for name, loaded_weight in remapped:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
|
||||
# Handle tied embeddings → lm_head.
|
||||
if name == "model.embed_tokens.weight" and self.text_config.tie_word_embeddings:
|
||||
# Also load into lm_head if present.
|
||||
lm_key = "lm_head.weight"
|
||||
if lm_key in params_dict:
|
||||
p = params_dict[lm_key]
|
||||
wl = getattr(p, "weight_loader", default_weight_loader)
|
||||
wl(p, loaded_weight)
|
||||
loaded_params.add(lm_key)
|
||||
|
||||
# Try stacked params mapping (q/k/v → qkv_proj, gate/up → gate_up_proj).
|
||||
handled = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped = name.replace(weight_name, param_name)
|
||||
if mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[mapped]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(mapped)
|
||||
handled = True
|
||||
break
|
||||
|
||||
if handled:
|
||||
continue
|
||||
|
||||
# Direct parameter mapping.
|
||||
if name in params_dict:
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
logger.info("Loaded %d weights for FishSpeechSlowARForConditionalGeneration", len(loaded_params))
|
||||
|
||||
# Truncate RoPE cos/sin caches to bf16 precision to match training.
|
||||
# Without this, f32 RoPE values cause logit divergence and premature EOS.
|
||||
truncated = 0
|
||||
for module in self.modules():
|
||||
if hasattr(module, "cos_sin_cache") and isinstance(module.cos_sin_cache, torch.Tensor):
|
||||
cache = module.cos_sin_cache
|
||||
module.cos_sin_cache = cache.to(torch.bfloat16).to(cache.dtype)
|
||||
truncated += 1
|
||||
if truncated:
|
||||
logger.info("Truncated %d RoPE cos_sin_cache buffers to bf16 precision", truncated)
|
||||
|
||||
return loaded_params
|
||||
@@ -1054,6 +1054,79 @@ class HunyuanImage3MultiModalProcessor(BaseMultiModalProcessor[HunyuanImage3Proc
|
||||
]
|
||||
|
||||
|
||||
class HunyuanImage3RotaryEmbedding(nn.Module):
|
||||
"""Custom interleaved 2D Rotary Embedding for HunyuanImage3.
|
||||
|
||||
The original HunyuanImage3 ``build_2d_rope`` interleaves y (height) and
|
||||
x (width) positions across consecutive frequencies::
|
||||
|
||||
theta = inv_freq.reshape(n_elem // 4, 2)
|
||||
idx_theta[2k] = y_pos * theta[k, 0] # even freq -> y
|
||||
idx_theta[2k+1] = x_pos * theta[k, 1] # odd freq -> x
|
||||
|
||||
vLLM's standard ``MRotaryEmbedding`` instead assigns *contiguous* blocks
|
||||
of frequencies to each position dimension, producing different encodings
|
||||
for the same positions. This class re-implements the original interleaved
|
||||
pattern so that pre-trained weights work correctly.
|
||||
|
||||
Reference: https://kexue.fm/archives/10352
|
||||
"""
|
||||
|
||||
def __init__(self, head_dim: int, rope_theta: float = 10000.0) -> None:
|
||||
super().__init__()
|
||||
self.head_dim = head_dim
|
||||
assert head_dim % 4 == 0, f"head_dim must be divisible by 4, got {head_dim}"
|
||||
inv_freq = 1.0 / (rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim))
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if positions.dim() == 2 and positions.shape[0] == 3:
|
||||
y_pos = positions[1].float()
|
||||
x_pos = positions[2].float()
|
||||
else:
|
||||
# 1D fallback: both dims get the same position → standard RoPE
|
||||
y_pos = positions.float()
|
||||
x_pos = positions.float()
|
||||
|
||||
num_tokens = y_pos.shape[0]
|
||||
dtype = query.dtype
|
||||
query_shape = query.shape
|
||||
key_shape = key.shape
|
||||
|
||||
inv_freq = self.inv_freq.to(device=y_pos.device, dtype=torch.float32)
|
||||
|
||||
inv_freq_y = inv_freq[0::2] # even indices -> y
|
||||
inv_freq_x = inv_freq[1::2] # odd indices -> x
|
||||
|
||||
y_freqs = y_pos.unsqueeze(-1) * inv_freq_y.unsqueeze(0)
|
||||
x_freqs = x_pos.unsqueeze(-1) * inv_freq_x.unsqueeze(0)
|
||||
|
||||
# Interleave: [y*θ₀, x*θ₁, y*θ₂, x*θ₃, ...]
|
||||
freqs = torch.stack([y_freqs, x_freqs], dim=-1).reshape(num_tokens, -1)
|
||||
|
||||
emb = torch.cat([freqs, freqs], dim=-1)
|
||||
cos = emb.cos().to(dtype).unsqueeze(1)
|
||||
sin = emb.sin().to(dtype).unsqueeze(1)
|
||||
|
||||
query = query.view(num_tokens, -1, self.head_dim)
|
||||
key = key.view(num_tokens, -1, self.head_dim)
|
||||
|
||||
query = query * cos + self._rotate_half(query) * sin
|
||||
key = key * cos + self._rotate_half(key) * sin
|
||||
|
||||
return query.view(query_shape), key.view(key_shape)
|
||||
|
||||
def _rotate_half(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x1 = x[..., : x.shape[-1] // 2]
|
||||
x2 = x[..., x.shape[-1] // 2 :]
|
||||
return torch.cat((-x2, x1), dim=-1)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
HunyuanImage3MultiModalProcessor,
|
||||
info=HunyuanImage3ProcessingInfo,
|
||||
@@ -1153,6 +1226,38 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
self._mrope_joint_img_sep_token_id = tokenizer.convert_tokens_to_ids("<joint_img_sep>")
|
||||
self._mrope_max_num_patches = config.vit_processor.get("max_num_patches", 729)
|
||||
|
||||
self._replace_rotary_embeddings()
|
||||
|
||||
def _replace_rotary_embeddings(self):
|
||||
"""Replace vLLM's standard MRotaryEmbedding with the custom
|
||||
interleaved 2D RoPE that matches the original HunyuanImage3 model."""
|
||||
rope_theta = getattr(self.config, "rope_theta", 10000.0)
|
||||
head_dim = getattr(
|
||||
self.config,
|
||||
"head_dim",
|
||||
getattr(
|
||||
self.config,
|
||||
"attention_head_dim",
|
||||
self.config.hidden_size // self.config.num_attention_heads,
|
||||
),
|
||||
)
|
||||
custom_rope = HunyuanImage3RotaryEmbedding(
|
||||
head_dim=head_dim,
|
||||
rope_theta=rope_theta,
|
||||
)
|
||||
replaced = 0
|
||||
for layer in self.model.layers:
|
||||
if hasattr(layer, "self_attn") and hasattr(layer.self_attn, "rotary_emb"):
|
||||
layer.self_attn.rotary_emb = custom_rope
|
||||
replaced += 1
|
||||
logger.info(
|
||||
"Replaced %d rotary embeddings with HunyuanImage3RotaryEmbedding "
|
||||
"(interleaved 2D RoPE, head_dim=%d, rope_theta=%.1f)",
|
||||
replaced,
|
||||
head_dim,
|
||||
rope_theta,
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self,
|
||||
**kwargs: dict[str, Any],
|
||||
@@ -1414,19 +1519,20 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
"""Compute mRoPE positions for HunyuanImage-3.
|
||||
|
||||
Maps the original model's build_2d_rope logic into vLLM's 3-dim
|
||||
mRoPE position tensor [3, seq_len] where dim-0 is temporal (unused,
|
||||
kept equal to 1D), dim-1 is height, and dim-2 is width.
|
||||
mRoPE position tensor ``[3, seq_len]`` where dim-1 is height and
|
||||
dim-2 is width. dim-0 is unused (temporal) and kept equal to 1D.
|
||||
|
||||
For text tokens and auxiliary image tokens (timestep, ViT):
|
||||
For text tokens and auxiliary image tokens (timestep, separators):
|
||||
All three dims get the same flat 1D position id.
|
||||
For VAE image tokens:
|
||||
dim-0 (T): flat 1D position id at the image start
|
||||
For VAE / ViT image tokens:
|
||||
dim-0 (T): flat 1D position id at the region start
|
||||
dim-1 (H): 2D y-position using build_2d_rope centering
|
||||
dim-2 (W): 2D x-position using build_2d_rope centering
|
||||
"""
|
||||
|
||||
# Extract per-image VAE grid dims from mm_features
|
||||
# Extract per-image VAE and ViT grid dims from mm_features
|
||||
vae_grids: list[tuple[int, int]] = []
|
||||
vit_grids: list[tuple[int, int]] = []
|
||||
if mm_features is not None:
|
||||
for mm_feature in mm_features:
|
||||
mm_item = mm_feature.data
|
||||
@@ -1437,20 +1543,22 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
if vae_hw is not None:
|
||||
grid = vae_hw.tolist()
|
||||
vae_grids.append((int(grid[0]), int(grid[1])))
|
||||
vit_hw = mm_input.get("vit_spatial_shapes")
|
||||
if vit_hw is not None:
|
||||
grid = vit_hw.tolist()
|
||||
vit_grids.append((int(grid[0]), int(grid[1])))
|
||||
|
||||
# Identify image token ids (cached from __init__)
|
||||
img_token_id = self._mrope_img_token_id
|
||||
boi_token_id = self._mrope_boi_token_id
|
||||
eoi_token_id = self._mrope_eoi_token_id
|
||||
joint_img_sep_token_id = self._mrope_joint_img_sep_token_id
|
||||
max_num_patches = self._mrope_max_num_patches
|
||||
|
||||
# Build position arrays
|
||||
t_pos = [] # temporal (same as 1D for this model)
|
||||
h_pos = [] # height
|
||||
w_pos = [] # width
|
||||
t_pos: list[int] = [] # temporal (same as 1D for this model)
|
||||
h_pos: list[int] = [] # height
|
||||
w_pos: list[int] = [] # width
|
||||
|
||||
pos = 0 # current 1D position counter
|
||||
pos = 0
|
||||
image_idx = 0
|
||||
i = 0
|
||||
n = len(input_tokens)
|
||||
@@ -1462,7 +1570,7 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
# Found start of image block.
|
||||
# Structure: <boi> <size> <ratio> <img>*timestep <img>*vae
|
||||
# <joint_img_sep> <img>*vit <eoi>
|
||||
# Assign <boi> a flat position
|
||||
# <boi> token
|
||||
t_pos.append(pos)
|
||||
h_pos.append(pos)
|
||||
w_pos.append(pos)
|
||||
@@ -1493,16 +1601,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
pos += 1
|
||||
i += 1
|
||||
|
||||
# VAE tokens: get grid dims
|
||||
# VAE tokens: 2D grid positions
|
||||
if image_idx < len(vae_grids):
|
||||
vae_h, vae_w = vae_grids[image_idx]
|
||||
else:
|
||||
vae_h, vae_w = 0, 0
|
||||
image_idx += 1
|
||||
|
||||
# Assign 2D positions to VAE tokens using build_2d_rope
|
||||
# centering logic
|
||||
L = pos # position at start of VAE region
|
||||
L = pos
|
||||
wh = vae_w * vae_h
|
||||
beta_y = L + (wh - vae_h) / 2
|
||||
beta_x = L + (wh - vae_w) / 2
|
||||
@@ -1510,12 +1614,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
for row in range(vae_h):
|
||||
for col in range(vae_w):
|
||||
if i < n and input_tokens[i] == img_token_id:
|
||||
t_pos.append(L) # temporal stays flat
|
||||
t_pos.append(L)
|
||||
h_pos.append(int(beta_y + row))
|
||||
w_pos.append(int(beta_x + col))
|
||||
i += 1
|
||||
|
||||
pos = L + wh # advance past VAE region
|
||||
pos = L + wh
|
||||
|
||||
# <joint_img_sep> token
|
||||
if i < n and input_tokens[i] == joint_img_sep_token_id:
|
||||
@@ -1525,15 +1629,34 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
pos += 1
|
||||
i += 1
|
||||
|
||||
# ViT tokens (max_num_patches <img> tokens) — flat 1D
|
||||
vit_consumed = 0
|
||||
while i < n and input_tokens[i] == img_token_id and vit_consumed < max_num_patches:
|
||||
# ViT tokens: 2D grid positions
|
||||
if image_idx < len(vit_grids):
|
||||
vit_h, vit_w = vit_grids[image_idx]
|
||||
else:
|
||||
vit_h, vit_w = 0, 0
|
||||
L = pos
|
||||
wh = vit_w * vit_h
|
||||
beta_y = L + (wh - vit_h) / 2
|
||||
beta_x = L + (wh - vit_w) / 2
|
||||
|
||||
for row in range(vit_h):
|
||||
for col in range(vit_w):
|
||||
if i < n and input_tokens[i] == img_token_id:
|
||||
t_pos.append(L)
|
||||
h_pos.append(int(beta_y + row))
|
||||
w_pos.append(int(beta_x + col))
|
||||
i += 1
|
||||
|
||||
pos = L + wh
|
||||
|
||||
# Remaining ViT padding tokens (when max_num_patches >
|
||||
# vit_h * vit_w) get flat 1D positions.
|
||||
while i < n and input_tokens[i] == img_token_id:
|
||||
t_pos.append(pos)
|
||||
h_pos.append(pos)
|
||||
w_pos.append(pos)
|
||||
pos += 1
|
||||
i += 1
|
||||
vit_consumed += 1
|
||||
|
||||
# <eoi> token
|
||||
if i < n and input_tokens[i] == eoi_token_id:
|
||||
@@ -1543,6 +1666,7 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
|
||||
pos += 1
|
||||
i += 1
|
||||
|
||||
image_idx += 1
|
||||
else:
|
||||
# Regular text token — flat 1D position
|
||||
t_pos.append(pos)
|
||||
|
||||
@@ -129,6 +129,17 @@ _OMNI_MODELS = {
|
||||
"hunyuan_image3",
|
||||
"HunyuanImage3ForConditionalGeneration",
|
||||
),
|
||||
## fish_speech (Fish Speech S2 Pro)
|
||||
"FishSpeechSlowARForConditionalGeneration": (
|
||||
"fish_speech",
|
||||
"fish_speech_slow_ar",
|
||||
"FishSpeechSlowARForConditionalGeneration",
|
||||
),
|
||||
"FishSpeechDACDecoder": (
|
||||
"fish_speech",
|
||||
"fish_speech_dac_decoder",
|
||||
"FishSpeechDACDecoder",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user