[TTS][Perf] Optimize Fish Speech S2 Pro high-concurrency serving (#3773)

Signed-off-by: Sy03 <1370724210@qq.com>
Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
Sy03andYueqian Lin authored and GitHub committed 2026-05-27 13:25:49 -05:00
1 parent 37eebff674
commit d37912b071
23 files changed
+3453 -92

No files matched your search

@@ -143,6 +143,26 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
pip install fish-speech
```
### Kvcache attention fast path
Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or
`VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be
installed.
```bash
# Verify fast path availability.
python - <<'PY'
from vllm_omni.attention import fish_kvcache_attn
print(fish_kvcache_attn.is_available())
print(fish_kvcache_attn.load_error())
PY
# Optional: disable the runtime fast path.
export VLLM_OMNI_FISH_KVCACHE_ATTN=0
```
### Launch
```bash
vllm serve fishaudio/s2-pro --omni --port 8091
+20
View File
@@ -50,6 +50,26 @@ fi
pip install fish-speech
```
#### Kvcache attention fast path
Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or
`VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be
installed.
```bash
# Verify fast path availability.
python - <<'PY'
from vllm_omni.attention import fish_kvcache_attn
print(fish_kvcache_attn.is_available())
print(fish_kvcache_attn.load_error())
PY
# Optional: disable the runtime fast path.
export VLLM_OMNI_FISH_KVCACHE_ATTN=0
```
#### Command
```bash
File diff suppressed because it is too large. Load diff
@@ -33,6 +33,20 @@ def test_tensor_serialization():
assert torch.equal(tensor, deserialized)
def test_noncontiguous_tensor_serialization():
"""Test torch.Tensor serialization for strided views."""
import torch
tensor = torch.arange(40, dtype=torch.long).reshape(4, 10).transpose(0, 1)
assert not tensor.is_contiguous()
assert tensor.stride(-1) != 1
serialized = OmniSerializer.serialize({"codes": tensor})
deserialized = OmniSerializer.deserialize(serialized)
assert torch.equal(tensor, deserialized["codes"])
def test_ndarray_serialization():
"""Test numpy.ndarray serialization."""
import numpy as np
@@ -37,6 +37,7 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
status=status,
prompt_token_ids=[],
num_computed_tokens=0,
num_output_placeholders=0,
additional_information=None,
is_finished=lambda: status == RequestStatus.FINISHED_STOPPED,
)
@@ -44,9 +45,16 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
@pytest.fixture
def build_adapter(monkeypatch, mocker: MockerFixture):
def _build(*, stage_id: int = 1, model_mode: str = "ar", max_num_seqs: int = 2):
def _build(
*,
stage_id: int = 1,
model_mode: str = "ar",
max_num_seqs: int = 2,
connector_extra: dict | None = None,
):
connector = mocker.MagicMock()
connector.stage_id = stage_id
connector.config = {"extra": connector_extra or {}}
connector.get.return_value = None
connector.put.return_value = (True, 1, {})
@@ -127,6 +135,61 @@ def test_load_poll(build_adapter):
assert "req-1" not in adapter._pending_load_reqs
def test_load_poll_generation_tensor_codes_use_placeholder_prompt(build_adapter):
adapter, connector = build_adapter(stage_id=1, model_mode="generation")
request = _req("req-tensor", RequestStatus.WAITING, external_req_id="external-tensor")
codes = torch.tensor([[1, 2], [3, 4]], dtype=torch.long)
payload: OmniPayload = {
"codes": {"audio": codes},
"meta": {
"left_context_size": 1,
"finished": torch.tensor(False, dtype=torch.bool),
},
}
connector.get.return_value = (payload, 16)
adapter._poll_single_request(request)
assert request.prompt_token_ids == [0]
assert request.num_computed_tokens == 0
assert torch.equal(request.additional_information["codes"]["audio"], codes)
assert request.additional_information["meta"]["left_context_size"] == 1
assert "finished" not in request.additional_information["meta"]
assert "req-tensor" in adapter._finished_load_reqs
def test_load_poll_generation_empty_nonterminal_chunk_keeps_polling(build_adapter):
adapter, connector = build_adapter(stage_id=1, model_mode="generation")
request = _req("req-empty-tensor", RequestStatus.WAITING, external_req_id="external-empty")
empty_payload: OmniPayload = {
"codes": {"audio": torch.empty((4, 0), dtype=torch.long)},
"meta": {
"left_context_size": 0,
"finished": torch.tensor(False, dtype=torch.bool),
},
}
ready_payload: OmniPayload = {
"codes": {"audio": torch.tensor([[1, 2]], dtype=torch.long)},
"meta": {
"left_context_size": 0,
"finished": torch.tensor(False, dtype=torch.bool),
},
}
connector.get.side_effect = [(empty_payload, 16), (ready_payload, 16)]
assert adapter._poll_single_request(request) is False
assert request.request_id not in adapter._finished_load_reqs
assert request.request_id not in adapter.requests_with_ready_chunks
assert adapter.get_req_chunk[request.request_id] == 1
assert adapter._poll_single_request(request) is True
assert request.request_id in adapter._finished_load_reqs
assert torch.equal(request.additional_information["codes"]["audio"], ready_payload["codes"]["audio"])
assert adapter.get_req_chunk[request.request_id] == 2
def test_save_async(build_adapter):
adapter, _ = build_adapter(stage_id=1)
request = _req("req-1", RequestStatus.WAITING, external_req_id="external-1")
@@ -140,6 +203,18 @@ def test_save_async(build_adapter):
assert task["is_finished"] is False
def test_save_async_uses_confirmed_tokens_for_async_scheduler_watermark(build_adapter):
adapter, _ = build_adapter(stage_id=1)
request = _req("req-async", RequestStatus.WAITING, external_req_id="external-async")
request.num_computed_tokens = 10
request.num_output_placeholders = 2
adapter.save_async(pooling_output=None, request=request)
assert adapter.requests_num_chunks_sent["external-async"] == 8
assert len(adapter._pending_save_reqs) == 1
def test_send_single_request_struct_without_meta_does_not_crash(build_adapter, monkeypatch):
"""Producer may return a struct with ``meta=None`` (e.g. payload that
carries only ``embed`` or ``codes``). The sender's ``meta is not None``
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm_omni.model_executor.models.fish_speech.configuration_fish_speech import (
FishSpeechFastARConfig,
FishSpeechSlowARConfig,
)
from vllm_omni.model_executor.models.fish_speech.fish_speech_fast_ar import FishSpeechFastAR
from vllm_omni.model_executor.models.fish_speech.fish_speech_slow_ar import (
FishSpeechSlowARForConditionalGeneration,
)
def test_fast_ar_reuses_dense_position_id_buffer(monkeypatch):
fast_config = FishSpeechFastARConfig(
vocab_size=16,
num_codebooks=4,
dim=8,
n_head=2,
n_local_heads=1,
head_dim=4,
n_layer=1,
intermediate_size=16,
max_seq_len=5,
)
slow_config = FishSpeechSlowARConfig(
vocab_size=32,
dim=8,
n_head=2,
n_local_heads=1,
head_dim=4,
n_layer=1,
intermediate_size=16,
codebook_size=16,
num_codebooks=4,
semantic_begin_id=4,
semantic_end_id=19,
)
fast_ar = object.__new__(FishSpeechFastAR)
nn.Module.__init__(fast_ar)
fast_ar.config = fast_config
fast_ar.slow_ar_config = slow_config
fast_ar.fast_project_in = nn.Identity()
fast_ar.fast_embeddings = nn.Embedding(fast_config.vocab_size, fast_config.hidden_size)
fast_ar.fast_output = nn.Linear(fast_config.hidden_size, fast_config.vocab_size, bias=False)
fast_ar.fast_norm = nn.Identity()
fast_ar._num_codebooks = fast_config.num_codebooks
fast_ar._fast_dim = fast_config.hidden_size
fast_ar._embed_buf = None
fast_ar._pos_ids = None
fast_ar._k_cache = None
fast_ar._v_cache = None
fast_ar._compiled_model_fwd = None
fast_ar._compile_attempted = True
fast_ar._compile_failed = False
fast_ar._disable_compile_for_graph = False
seen_position_ids: list[torch.Tensor] = []
def fake_run_model_one(step_input, step_pos_ids, cache_pos):
seen_position_ids.append(step_pos_ids)
return torch.zeros(
step_input.shape[0],
fast_config.hidden_size,
dtype=step_input.dtype,
device=step_input.device,
)
monkeypatch.setattr(fast_ar, "_run_model_one", fake_run_model_one)
hidden = torch.zeros(3, slow_config.hidden_size, dtype=torch.float32)
semantic = torch.full((3,), slow_config.semantic_begin_id, dtype=torch.long)
fast_ar(hidden, semantic, do_sample=False)
assert fast_ar._pos_ids is not None
assert fast_ar._pos_ids.shape == (3, fast_config.num_codebooks + 1)
assert len(seen_position_ids) == fast_config.num_codebooks
for step, step_pos_ids in enumerate(seen_position_ids):
assert step_pos_ids.shape == (3,)
assert step_pos_ids.untyped_storage().data_ptr() == fast_ar._pos_ids.untyped_storage().data_ptr()
assert step_pos_ids.stride() == (fast_ar._pos_ids.stride(0),)
expected = torch.full((3,), step)
assert torch.equal(step_pos_ids.cpu(), expected)
def test_talker_mtp_does_not_mutate_input():
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
nn.Module.__init__(model)
model._semantic_begin_id = 4
model._semantic_end_id = 19
model._codebook_size = 16
model._num_codebooks = 4
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
model.fast_ar = lambda **_: torch.tensor(
[
[0, 1, 2, 3],
[15, 14, 13, 12],
],
dtype=torch.long,
)
input_ids = torch.tensor([4, 2], dtype=torch.long)
input_embeds = torch.randn(2, 8)
input_embeds_before = input_embeds.clone()
last_hidden = torch.randn(2, 8, dtype=torch.bfloat16)
text_step = torch.zeros(2, 8, dtype=torch.bfloat16)
out1, _ = FishSpeechSlowARForConditionalGeneration.talker_mtp(
model,
input_ids,
input_embeds,
last_hidden,
text_step,
)
assert torch.equal(input_embeds, input_embeds_before)
assert out1.untyped_storage().data_ptr() != input_embeds.untyped_storage().data_ptr()
def test_talker_mtp_forwards_sampling_params():
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
nn.Module.__init__(model)
model._semantic_begin_id = 4
model._semantic_end_id = 19
model._codebook_size = 16
model._num_codebooks = 4
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
seen = {}
def fake_fast_ar(**kwargs):
seen.update(kwargs)
return torch.tensor([[0, 1, 2, 3]], dtype=torch.long)
model.fast_ar = fake_fast_ar
input_ids = torch.tensor([4], dtype=torch.long)
input_embeds = torch.randn(1, 8)
last_hidden = torch.randn(1, 8, dtype=torch.bfloat16)
text_step = torch.zeros(1, 8, dtype=torch.bfloat16)
FishSpeechSlowARForConditionalGeneration.talker_mtp(
model,
input_ids,
input_embeds,
last_hidden,
text_step,
do_sample=False,
temperature=0.1,
top_k=5,
top_p=0.7,
)
assert seen["do_sample"] is False
assert seen["temperature"] == 0.1
assert seen["top_k"] == 5
assert seen["top_p"] == 0.7
@@ -1,5 +1,6 @@
import functools
import math
from types import SimpleNamespace
import pytest
import torch
@@ -26,6 +27,39 @@ class _FakeCodec:
return wav, audio_lengths
class _RecordingCodec:
def __init__(self):
self.calls = []
def decode(self, codes_bqf: torch.Tensor, feature_lengths: torch.Tensor):
self.calls.append((tuple(codes_bqf.shape), tuple(int(x) for x in feature_lengths.tolist())))
batch = codes_bqf.shape[0]
wav_len = int(codes_bqf.shape[-1]) * 10
wav = torch.arange(batch * wav_len, dtype=torch.float32).view(batch, 1, wav_len)
audio_lengths = feature_lengths.to(torch.long) * 10
return wav, audio_lengths
class _FakeOneArgCodec:
def decode(self, codes_bqf: torch.Tensor):
batch = codes_bqf.shape[0]
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
class _FakeFromIndicesCodec:
def __init__(self):
self.from_indices_calls = 0
def from_indices(self, codes_bqf: torch.Tensor):
self.from_indices_calls += 1
batch = codes_bqf.shape[0]
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
def decode(self, codes_bqf: torch.Tensor):
del codes_bqf
raise AssertionError("decode() should not be used for index tensors when from_indices() exists")
class _FakeTokenizer:
def __init__(self, mapping, unk_token_id=-1):
self._mapping = mapping
@@ -40,6 +74,7 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = _FakeCodec()
decoder._codec_decode_takes_lengths = True
decoder._num_codebooks = 10
decoder._output_sample_rate = 44100
decoder._hop_length = 512
@@ -62,6 +97,132 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
assert audios[1].shape[0] == 50
def test_dac_decoder_uses_tensor_codes_from_runtime_info():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = _FakeCodec()
decoder._codec_decode_takes_lengths = True
decoder._num_codebooks = 2
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._ensure_codec_loaded = lambda: None
out = decoder.forward(
input_ids=torch.tensor([0], dtype=torch.long),
runtime_additional_information=[
{
"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)},
"meta": {"left_context_size": 1},
}
],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 1
# 2 total frames with 1 frame of left context => proportional trim removes half the samples.
assert audios[0].shape[0] == 50
def test_dac_decoder_supports_codec_decode_without_lengths():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = _FakeOneArgCodec()
decoder._codec_decode_takes_lengths = False
decoder._num_codebooks = 10
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._ensure_codec_loaded = lambda: None
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
torch.arange(10, dtype=torch.long),
torch.arange(20, dtype=torch.long),
]
out = decoder.forward(
input_ids=torch.arange(30, dtype=torch.long),
runtime_additional_information=[{}, {}],
)
audios = out.multimodal_outputs["model_outputs"]
assert len(audios) == 2
assert audios[0].shape[0] == 50
assert audios[1].shape[0] == 100
def test_dac_decoder_prefers_from_indices_for_one_arg_codec():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
codec = _FakeFromIndicesCodec()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = codec
decoder._codec_decode_takes_lengths = False
decoder._num_codebooks = 10
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._ensure_codec_loaded = lambda: None
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
torch.arange(10, dtype=torch.long),
torch.arange(20, dtype=torch.long),
]
out = decoder.forward(
input_ids=torch.arange(30, dtype=torch.long),
runtime_additional_information=[{}, {}],
)
audios = out.multimodal_outputs["model_outputs"]
assert codec.from_indices_calls == 1
assert len(audios) == 2
assert audios[0].shape[0] == 50
assert audios[1].shape[0] == 100
def test_dac_decoder_splits_groups_by_padded_frame_budget():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
codec = _RecordingCodec()
decoder = object.__new__(FishSpeechDACDecoder)
torch.nn.Module.__init__(decoder)
decoder._codec = codec
decoder._codec_decode_takes_lengths = True
decoder._num_codebooks = 2
decoder._output_sample_rate = 44100
decoder._hop_length = 512
decoder._logged_codec_stats = False
decoder._decode_batch_max_padded_frames = 100
decoder._decode_batch_max_batch = 0
decoder._ensure_codec_loaded = lambda: None
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
torch.arange(2 * 46, dtype=torch.long),
torch.arange(2 * 45, dtype=torch.long),
torch.arange(2 * 47, dtype=torch.long),
]
decoder.forward(input_ids=torch.arange(2 * (46 + 45 + 47), dtype=torch.long))
assert codec.calls == [
((2, 2, 46), (45, 46)),
((1, 2, 47), (47,)),
]
def test_dac_decoder_dtype_can_be_configured():
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
cfg = SimpleNamespace(
model_config=SimpleNamespace(
model="unused",
stage_connector_config={"extra": {"fish_speech_dac_dtype": "fp16"}},
)
)
decoder = FishSpeechDACDecoder(vllm_config=cfg)
assert decoder._dac_dtype == torch.float16
def test_structured_voice_clone_prefill_adds_full_codebooks_with_decode_scale(monkeypatch):
slow_ar_module, _, FishSpeechSlowARForConditionalGeneration = _fish_speech_regression_modules()
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
@@ -0,0 +1,229 @@
from collections import defaultdict
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.stage_input_processors.fish_speech import (
slow_ar_to_dac_decoder_async_chunk,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_transfer_manager(
*,
tensor_payload: bool = False,
chunk_frames: int = 2,
left_context: int = 1,
initial_chunk_frames: int = 0,
single_initial_chunk: bool = False,
backlog_chunk_frames: int = 0,
backlog_load_threshold: float = 0.75,
max_num_seqs: int = 1,
):
return SimpleNamespace(
code_prompt_token_ids=defaultdict(list),
put_req_chunk=defaultdict(int),
scheduler_max_num_seqs=max_num_seqs,
connector=SimpleNamespace(
config={
"extra": {
"codec_chunk_frames": chunk_frames,
"codec_left_context_frames": left_context,
"initial_codec_chunk_frames": initial_chunk_frames,
"fish_speech_tensor_codes": tensor_payload,
"fish_speech_single_initial_chunk": single_initial_chunk,
"fish_speech_backlog_codec_chunk_frames": backlog_chunk_frames,
"fish_speech_backlog_load_threshold": backlog_load_threshold,
}
}
),
)
def _make_request(req_id="req", *, finished=False):
return SimpleNamespace(
external_req_id=req_id,
additional_information=None,
is_finished=lambda: finished,
)
def _seed_frames(transfer_manager, req_id: str, n_frames: int, n_codebooks: int = 3):
frame = torch.arange(1, n_codebooks + 1, dtype=torch.long)
transfer_manager.code_prompt_token_ids[req_id] = [frame + i * n_codebooks for i in range(n_frames)]
def _call_seeded(transfer_manager, req_id: str = "req", *, n_frames: int, finished: bool = False):
_seed_frames(transfer_manager, req_id, n_frames)
return slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{},
_make_request(req_id, finished=finished),
is_finished=finished,
)
def test_slow_ar_to_dac_async_chunk_can_emit_tensor_codes():
transfer_manager = _make_transfer_manager(tensor_payload=True)
request = _make_request()
assert (
slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{"audio_codes": torch.tensor([[1, 2, 3]])},
request,
)
is None
)
payload = slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{"audio_codes": torch.tensor([[4, 5, 6]])},
request,
)
assert payload is not None
audio_codes = payload.codes.audio
assert isinstance(audio_codes, torch.Tensor)
assert audio_codes.dtype == torch.long
assert torch.equal(audio_codes, torch.tensor([[1, 4], [2, 5], [3, 6]], dtype=torch.long))
assert payload.meta.left_context_size == 0
def test_slow_ar_to_dac_async_chunk_uses_valid_mask_for_zero_codes():
transfer_manager = _make_transfer_manager(tensor_payload=True, chunk_frames=1, left_context=0)
request = _make_request()
assert (
slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{
"audio_codes": torch.tensor([[0, 0, 0]]),
"audio_code_valid": torch.tensor([False]),
},
request,
)
is None
)
payload = slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{
"audio_codes": torch.tensor([[4, 5, 6]]),
"audio_code_valid": torch.tensor([True]),
},
request,
)
assert payload is not None
assert torch.equal(payload.codes.audio, torch.tensor([[4], [5], [6]], dtype=torch.long))
def test_slow_ar_to_dac_async_chunk_keeps_flat_codes_by_default():
transfer_manager = _make_transfer_manager(tensor_payload=False)
request = _make_request()
slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{"audio_codes": torch.tensor([[1, 2]])},
request,
)
payload = slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{"audio_codes": torch.tensor([[3, 4]])},
request,
)
assert payload is not None
assert torch.equal(payload.codes.audio, torch.tensor([1, 3, 2, 4], dtype=torch.long))
def test_qwen3_style_single_initial_chunk_only_emits_first_chunk_early():
transfer_manager = _make_transfer_manager(
chunk_frames=5,
left_context=5,
initial_chunk_frames=2,
single_initial_chunk=True,
)
first = _call_seeded(transfer_manager, n_frames=2)
assert first is not None
assert first.meta.left_context_size == 0
assert first.codes.audio.numel() == 3 * 2
# After the first small chunk, return to normal chunk boundaries:
# initial_coverage=2, so the next emit is at 2 + 5 = 7 frames.
assert _call_seeded(transfer_manager, n_frames=5) is None
second = _call_seeded(transfer_manager, n_frames=7)
assert second is not None
assert second.meta.left_context_size == 2
assert second.codes.audio.numel() == 3 * 7
def test_backlog_chunk_size_uses_larger_post_initial_boundary():
transfer_manager = _make_transfer_manager(
chunk_frames=5,
left_context=5,
initial_chunk_frames=2,
single_initial_chunk=True,
backlog_chunk_frames=8,
backlog_load_threshold=0.5,
max_num_seqs=4,
)
for idx in range(2):
_seed_frames(transfer_manager, f"other-{idx}", 10)
first = _call_seeded(transfer_manager, "req", n_frames=2)
assert first is not None
# Under backlog, steady chunk is 8 instead of 5. The next emit is
# initial_coverage=2 plus 8 frames, so n=7 must still be held.
assert _call_seeded(transfer_manager, "req", n_frames=7) is None
second = _call_seeded(transfer_manager, "req", n_frames=10)
assert second is not None
assert second.meta.left_context_size == 2
assert second.codes.audio.numel() == 3 * 10
def test_backlog_chunk_size_keeps_multi_initial_boundary_aligned():
transfer_manager = _make_transfer_manager(
chunk_frames=25,
left_context=25,
initial_chunk_frames=10,
single_initial_chunk=False,
backlog_chunk_frames=50,
backlog_load_threshold=0.5,
max_num_seqs=4,
)
for idx in range(2):
_seed_frames(transfer_manager, f"other-{idx}", 50)
first = _call_seeded(transfer_manager, "req", n_frames=10)
assert first is not None
transfer_manager.put_req_chunk["req"] += 1
second = _call_seeded(transfer_manager, "req", n_frames=20)
assert second is not None
transfer_manager.put_req_chunk["req"] += 1
assert _call_seeded(transfer_manager, "req", n_frames=50) is None
third = _call_seeded(transfer_manager, "req", n_frames=70)
assert third is not None
assert third.meta.left_context_size == 20
assert third.codes.audio.numel() == 3 * 70
def test_invalid_backlog_config_fails_loudly():
transfer_manager = _make_transfer_manager(
chunk_frames=5,
left_context=5,
backlog_chunk_frames="invalid",
)
request = _make_request()
with pytest.raises(ValueError, match="fish_speech_backlog_codec_chunk_frames"):
slow_ar_to_dac_decoder_async_chunk(
transfer_manager,
{"audio_codes": torch.tensor([[1, 2, 3]])},
request,
)
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+191
View File
@@ -0,0 +1,191 @@
from __future__ import annotations
import os
import threading
from typing import Any
import torch
FISH_KVCACHE_LONG_SPLIT_TOKENS = 1024
FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN = 1024
_FISH_KVCACHE_ATTN_ENV = "VLLM_OMNI_FISH_KVCACHE_ATTN"
_ENABLED_VALUES = frozenset({"1", "true", "yes", "on", "required"})
_DISABLED_VALUES = frozenset({"0", "false", "no", "off", "disabled", "disable"})
_REQUIRED_VALUES = frozenset({"required"})
_WORKSPACE_CACHE: dict[tuple[Any, ...], tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
_WORKSPACE_CACHE_LOCK = threading.Lock()
def _triton_backend() -> Any:
from vllm_omni.attention import fish_kvcache_triton
return fish_kvcache_triton
def is_available() -> bool:
return _triton_backend().is_available()
def load_error() -> Exception | None:
return _triton_backend().load_error()
def is_fish_kvcache_attn_enabled() -> bool:
value = os.environ.get(_FISH_KVCACHE_ATTN_ENV)
if value is None:
return True
value = value.lower()
if value in _DISABLED_VALUES:
return False
return value in _ENABLED_VALUES or value == ""
def is_fish_kvcache_attn_required() -> bool:
return os.environ.get(_FISH_KVCACHE_ATTN_ENV, "").lower() in _REQUIRED_VALUES
def _is_sliding_window_disabled(sliding_window: Any) -> bool:
if sliding_window is None:
return True
if isinstance(sliding_window, (list, tuple)) and len(sliding_window) == 2:
return int(sliding_window[0]) == -1 and int(sliding_window[1]) == -1
return False
def can_use_fish_kvcache_attn(
*,
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
block_table: torch.Tensor | None,
seq_lens: torch.Tensor,
max_query_len: int,
max_seq_len: int,
dcp_world_size: int,
use_cascade: bool,
alibi_slopes: Any,
sliding_window: Any,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> bool:
if not is_fish_kvcache_attn_enabled():
return False
if not is_available():
return False
if max_query_len != 1 or use_cascade or dcp_world_size != 1:
return False
if block_table is None or alibi_slopes is not None or not _is_sliding_window_disabled(sliding_window):
return False
if output_scale is not None or output_block_scale is not None:
return False
if query.dim() != 3 or key_cache.dim() != 4 or value_cache.dim() != 4:
return False
if block_table.dim() != 2 or seq_lens.dim() != 1:
return False
if block_table.shape[0] != query.shape[0] or seq_lens.shape[0] != query.shape[0]:
return False
if query.shape[-1] != 128 or key_cache.shape[-1] != 128:
return False
if key_cache.shape[1] != 16:
return False
if query.dtype not in (torch.float16, torch.bfloat16):
return False
if key_cache.dtype != query.dtype or value_cache.dtype != query.dtype:
return False
if block_table.dtype != torch.int32 or seq_lens.dtype != torch.int32:
return False
if max_seq_len <= 0:
return False
if max_seq_len > block_table.shape[1] * key_cache.shape[1]:
return False
if not (
query.is_contiguous()
and key_cache.is_contiguous()
and value_cache.is_contiguous()
and block_table.is_contiguous()
and seq_lens.is_contiguous()
):
return False
return True
def _is_cuda_graph_capturing() -> bool:
return bool(torch.cuda.is_current_stream_capturing())
def _raise_workspace_capture_miss() -> None:
raise RuntimeError("Fish kvcache attention workspace was not prewarmed before CUDA graph capture")
def _get_decode_workspace(
query: torch.Tensor,
max_seq_len: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
batch_size, num_q_heads, head_dim = query.shape
if int(max_seq_len) <= FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN:
key = (query.device.type, query.device.index, "empty")
partial_shapes = ((0,), (0,), (0,))
else:
num_splits = (int(max_seq_len) + FISH_KVCACHE_LONG_SPLIT_TOKENS - 1) // FISH_KVCACHE_LONG_SPLIT_TOKENS
key = (
query.device.type,
query.device.index,
int(batch_size),
int(num_q_heads),
int(head_dim),
int(num_splits),
)
partial_shapes = (
(num_splits, batch_size, num_q_heads),
(num_splits, batch_size, num_q_heads),
(num_splits, batch_size, num_q_heads, head_dim),
)
with _WORKSPACE_CACHE_LOCK:
workspace = _WORKSPACE_CACHE.get(key)
if workspace is None:
if _is_cuda_graph_capturing():
_raise_workspace_capture_miss()
workspace = (
torch.empty(partial_shapes[0], device=query.device, dtype=torch.float32),
torch.empty(partial_shapes[1], device=query.device, dtype=torch.float32),
torch.empty(partial_shapes[2], device=query.device, dtype=torch.float32),
)
_WORKSPACE_CACHE[key] = workspace
return workspace
def prewarm_fish_kvcache_attn_workspace(
query: torch.Tensor,
max_seq_len: int,
) -> None:
_get_decode_workspace(query, int(max_seq_len))
def fish_decode_kvcache_attn(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
block_table: torch.Tensor,
seq_lens: torch.Tensor,
out: torch.Tensor,
*,
scale: float,
max_seq_len: int,
) -> torch.Tensor:
partial_m, partial_l, partial_acc = _get_decode_workspace(query, int(max_seq_len))
return _triton_backend().fish_decode_kvcache_attn_triton(
query,
key_cache,
value_cache,
block_table,
seq_lens,
out,
scale=float(scale),
max_seq_len=int(max_seq_len),
small_path_max_seq_len=FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN,
long_split_tokens=FISH_KVCACHE_LONG_SPLIT_TOKENS,
partial_m=partial_m,
partial_l=partial_l,
partial_acc=partial_acc,
)
+406
View File
@@ -0,0 +1,406 @@
from __future__ import annotations
import types
from collections import Counter
from collections.abc import Callable
from typing import Any
import torch
from vllm.logger import init_logger
from vllm_omni.attention.fish_kvcache_attn import (
FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN,
can_use_fish_kvcache_attn,
fish_decode_kvcache_attn,
is_available,
is_fish_kvcache_attn_enabled,
is_fish_kvcache_attn_required,
load_error,
prewarm_fish_kvcache_attn_workspace,
)
logger = init_logger(__name__)
_FISH_SLOW_AR_ARCH = "FishSpeechSlowARForConditionalGeneration"
_FIRST_SMALL_HIT_LOGGED = False
_FIRST_LONG_HIT_LOGGED = False
_FIRST_MISS_LOGGED = False
_SMALL_HIT_COUNT = 0
_LONG_HIT_COUNT = 0
_FALLBACK_COUNTS: Counter[str] = Counter()
def _fish_kvcache_enabled() -> bool:
return is_fish_kvcache_attn_enabled()
def is_fish_kvcache_attn_active_for_model(model_config: Any) -> bool:
return getattr(model_config, "model_arch", None) == _FISH_SLOW_AR_ARCH and is_fish_kvcache_attn_enabled()
def reset_fish_kvcache_attn_stats() -> None:
global _LONG_HIT_COUNT, _SMALL_HIT_COUNT
_SMALL_HIT_COUNT = 0
_LONG_HIT_COUNT = 0
_FALLBACK_COUNTS.clear()
def get_fish_kvcache_attn_stats() -> dict[str, Any]:
return {
"small_hit_count": _SMALL_HIT_COUNT,
"long_hit_count": _LONG_HIT_COUNT,
"fallback_count_by_reason": dict(_FALLBACK_COUNTS),
}
def _record_fallback(reason: str) -> None:
_FALLBACK_COUNTS[reason] += 1
def _small_decode_bucket(max_seq_len: int) -> int:
if max_seq_len <= 64:
return 64
if max_seq_len <= 128:
return 128
if max_seq_len <= 256:
return 256
if max_seq_len <= 512:
return 512
return FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN
def prewarm_fish_kvcache_attn_capture_workspaces(
*,
model_config: Any,
device: torch.device,
dtype: torch.dtype,
capture_sizes: list[int] | tuple[int, ...],
) -> int:
"""Preallocate Fish attention workspaces used during CUDA graph capture."""
if not _fish_kvcache_enabled() or not is_available():
return 0
if not is_fish_kvcache_attn_active_for_model(model_config):
return 0
text_config = model_config.hf_config.text_config
num_heads = int(text_config.num_attention_heads)
head_dim = int(text_config.head_dim)
max_seq_len = int(model_config.max_model_len)
sizes = sorted({int(size) for size in capture_sizes if int(size) > 0})
if not sizes:
return 0
# The small path uses an empty workspace keyed only by device. Prewarm it
# once, then prewarm every CUDA graph capture batch size for the long path.
small_query = torch.empty((1, num_heads, head_dim), device=device, dtype=dtype)
prewarm_fish_kvcache_attn_workspace(small_query, 1)
prewarmed = 1
for batch_size in sizes:
query = torch.empty((batch_size, num_heads, head_dim), device=device, dtype=dtype)
prewarm_fish_kvcache_attn_workspace(query, max_seq_len)
prewarmed += 1
logger.info(
"Prewarmed Fish kvcache attention workspaces for capture sizes %s (num_heads=%d head_dim=%d max_seq_len=%d)",
sizes,
num_heads,
head_dim,
max_seq_len,
)
return prewarmed
def _attach_seq_lens_cpu_upper_bound(attn_metadata: Any, seq_lens_cpu_upper_bound: torch.Tensor) -> None:
if attn_metadata is None:
return
if isinstance(attn_metadata, dict):
for value in attn_metadata.values():
_attach_seq_lens_cpu_upper_bound(value, seq_lens_cpu_upper_bound)
return
if isinstance(attn_metadata, (list, tuple)):
for value in attn_metadata:
_attach_seq_lens_cpu_upper_bound(value, seq_lens_cpu_upper_bound)
return
setattr(attn_metadata, "seq_lens_cpu_upper_bound", seq_lens_cpu_upper_bound)
def _graph_capture_seq_len_upper_bound(
*,
model_config: Any,
max_query_len: int,
for_cudagraph_capture: bool,
) -> int | None:
if not for_cudagraph_capture or int(max_query_len) != 1:
return None
max_model_len = int(getattr(model_config, "max_model_len", 0))
return max(max_model_len, FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN)
def maybe_attach_fish_kvcache_seq_lens_upper_bound(
*,
model_config: Any,
attn_metadata: Any,
input_batch: Any,
optimistic_seq_lens_cpu: torch.Tensor,
num_reqs: int,
num_reqs_padded: int,
max_query_len: int,
pad_attn: bool,
for_cudagraph_capture: bool = False,
num_scheduled_tokens_np: Any = None,
) -> None:
"""Attach CPU seq-len upper bounds for the Fish-only Triton decode fastpath."""
if not is_fish_kvcache_attn_active_for_model(model_config) or int(max_query_len) != 1:
return
upper_bound_rows = int(num_reqs_padded if pad_attn else num_reqs)
capture_upper_bound = _graph_capture_seq_len_upper_bound(
model_config=model_config,
max_query_len=max_query_len,
for_cudagraph_capture=for_cudagraph_capture,
)
if capture_upper_bound is not None:
optimistic_seq_lens_cpu[:upper_bound_rows].fill_(capture_upper_bound)
else:
if num_scheduled_tokens_np is None:
return
seq_lens_cpu_upper_bound = torch.as_tensor(
input_batch.num_computed_tokens_cpu[:num_reqs] + num_scheduled_tokens_np,
dtype=torch.int32,
device="cpu",
)
optimistic_seq_lens_cpu[:num_reqs] = seq_lens_cpu_upper_bound
if upper_bound_rows > num_reqs:
optimistic_seq_lens_cpu[num_reqs:upper_bound_rows].fill_(1)
_attach_seq_lens_cpu_upper_bound(
attn_metadata,
optimistic_seq_lens_cpu[:upper_bound_rows],
)
def _dispatch_max_seq_len(attn_metadata: Any, active_batch: int, seq_lens: Any) -> int | None:
max_seq_len = int(attn_metadata.max_seq_len)
if int(attn_metadata.max_query_len) != 1:
return max_seq_len
upper_bound = getattr(attn_metadata, "seq_lens_cpu_upper_bound", None)
if upper_bound is None:
if is_fish_kvcache_attn_required():
raise RuntimeError("Fish kvcache attention requires seq_lens_cpu_upper_bound for decode dispatch")
return None
if not isinstance(upper_bound, torch.Tensor) or upper_bound.device.type != "cpu":
if is_fish_kvcache_attn_required():
raise RuntimeError("Fish kvcache attention seq_lens_cpu_upper_bound must be a CPU tensor")
return None
if int(upper_bound.shape[0]) < active_batch:
if is_fish_kvcache_attn_required():
raise RuntimeError(
"Fish kvcache attention seq_lens_cpu_upper_bound does not cover the active decode batch: "
f"upper_bound_rows={int(upper_bound.shape[0])}, active_batch={active_batch}"
)
return None
upper_bound = upper_bound[:active_batch]
real_upper_bound = max(1, int(upper_bound.max().item()))
if is_fish_kvcache_attn_required():
real_max_seq_len = int(seq_lens.max().item())
if real_upper_bound < real_max_seq_len:
raise RuntimeError(
"Fish kvcache attention seq_lens_cpu_upper_bound "
f"underestimates real seq_lens: upper_bound={real_upper_bound}, "
f"real={real_max_seq_len}"
)
if real_upper_bound <= FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN:
return _small_decode_bucket(real_upper_bound)
return real_upper_bound
def _forward_with_fish_kvcache(
impl: Any,
original_forward: Callable[..., Any],
layer: Any,
query: Any,
key: Any,
value: Any,
kv_cache: Any,
attn_metadata: Any,
output: Any,
output_scale: Any = None,
output_block_scale: Any = None,
) -> Any:
global _FIRST_LONG_HIT_LOGGED, _FIRST_MISS_LOGGED, _FIRST_SMALL_HIT_LOGGED
global _LONG_HIT_COUNT, _SMALL_HIT_COUNT
if attn_metadata is not None and not attn_metadata.use_cascade:
num_actual_tokens = attn_metadata.num_actual_tokens
key_cache, value_cache = kv_cache.unbind(0)
q = query[:num_actual_tokens]
out = output[:num_actual_tokens]
active_batch = int(q.shape[0])
block_table = attn_metadata.block_table[:active_batch] if attn_metadata.block_table is not None else None
seq_lens = attn_metadata.seq_lens[:active_batch]
dispatch_max_seq_len = _dispatch_max_seq_len(attn_metadata, active_batch, seq_lens)
can_use = dispatch_max_seq_len is not None and can_use_fish_kvcache_attn(
query=q,
key_cache=key_cache,
value_cache=value_cache,
block_table=block_table,
seq_lens=seq_lens,
max_query_len=attn_metadata.max_query_len,
max_seq_len=dispatch_max_seq_len,
dcp_world_size=impl.dcp_world_size,
use_cascade=attn_metadata.use_cascade,
alibi_slopes=impl.alibi_slopes,
sliding_window=impl.sliding_window,
output_scale=output_scale,
output_block_scale=output_block_scale,
)
if can_use:
is_long_path = dispatch_max_seq_len > FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN
if is_long_path:
_LONG_HIT_COUNT += 1
else:
_SMALL_HIT_COUNT += 1
should_log_hit = (is_long_path and not _FIRST_LONG_HIT_LOGGED) or (
not is_long_path and not _FIRST_SMALL_HIT_LOGGED
)
if should_log_hit:
if is_long_path:
_FIRST_LONG_HIT_LOGGED = True
path_name = "long"
else:
_FIRST_SMALL_HIT_LOGGED = True
path_name = "small"
logger.info(
"Fish decode-only kvcache attention fast path hit: "
"query_shape=%s key_cache_shape=%s block_table_shape=%s "
"seq_lens_shape=%s max_seq_len=%s path=%s",
tuple(q.shape),
tuple(key_cache.shape),
tuple(attn_metadata.block_table.shape),
tuple(attn_metadata.seq_lens.shape),
dispatch_max_seq_len,
path_name,
)
fish_decode_kvcache_attn(
q,
key_cache,
value_cache,
block_table,
seq_lens,
out,
scale=float(impl.scale),
max_seq_len=dispatch_max_seq_len,
)
return output
is_decode_request = int(attn_metadata.max_query_len) == 1
if is_decode_request and dispatch_max_seq_len is None:
_record_fallback("missing_cpu_upper_bound")
elif is_decode_request:
_record_fallback("guard_miss")
else:
_record_fallback("non_decode")
if is_decode_request and is_fish_kvcache_attn_required():
raise RuntimeError(
"Fish kvcache attention is required but guard rejected the request: "
f"query_shape={tuple(q.shape)} query_dtype={q.dtype} "
f"max_query_len={attn_metadata.max_query_len} max_seq_len={dispatch_max_seq_len} "
f"dcp_world_size={impl.dcp_world_size} use_cascade={attn_metadata.use_cascade}"
)
if not _FIRST_MISS_LOGGED:
_FIRST_MISS_LOGGED = True
logger.info(
"Fish decode-only kvcache attention fast path miss: "
"query_shape=%s query_dtype=%s key_cache_shape=%s key_dtype=%s "
"block_table_shape=%s block_dtype=%s seq_lens_shape=%s seq_dtype=%s "
"max_query_len=%s max_seq_len=%s dcp_world_size=%s use_cascade=%s",
tuple(q.shape),
q.dtype,
tuple(key_cache.shape),
key_cache.dtype,
tuple(block_table.shape) if block_table is not None else None,
getattr(block_table, "dtype", None),
tuple(seq_lens.shape),
seq_lens.dtype,
attn_metadata.max_query_len,
dispatch_max_seq_len,
impl.dcp_world_size,
attn_metadata.use_cascade,
)
return original_forward(
layer,
query,
key,
value,
kv_cache,
attn_metadata,
output,
output_scale=output_scale,
output_block_scale=output_block_scale,
)
def install_fish_kvcache_attn_backend(model: Any) -> int:
"""Install the Fish kvcache fast path on this Fish SlowAR model only."""
if not _fish_kvcache_enabled():
return 0
if not is_available():
_record_fallback("implementation_unavailable")
if is_fish_kvcache_attn_required():
raise RuntimeError(
f"VLLM_OMNI_FISH_KVCACHE_ATTN=required but Fish kvcache attention is unavailable: {load_error()!r}"
)
logger.warning(
"Fish kvcache attention is enabled but unavailable: %r",
load_error(),
)
return 0
installed = 0
for layer in model.layers:
impl = layer.self_attn.attn.impl
if getattr(impl, "_fish_kvcache_attn_installed", False):
continue
original_forward = impl.forward
def fish_forward(
this_impl: Any,
layer: Any,
query: Any,
key: Any,
value: Any,
kv_cache: Any,
attn_metadata: Any,
output: Any,
output_scale: Any = None,
output_block_scale: Any = None,
*,
_original_forward: Callable[..., Any] = original_forward,
) -> Any:
return _forward_with_fish_kvcache(
this_impl,
_original_forward,
layer,
query,
key,
value,
kv_cache,
attn_metadata,
output,
output_scale=output_scale,
output_block_scale=output_block_scale,
)
impl.forward = types.MethodType(fish_forward, impl)
impl._fish_kvcache_attn_installed = True
installed += 1
if installed:
logger.info("Installed Fish decode-only kvcache attention backend on %d attention layers", installed)
elif is_fish_kvcache_attn_required():
raise RuntimeError("Fish kvcache attention is required but installed 0 attention layers")
return installed
+359
View File
@@ -0,0 +1,359 @@
# ruff: noqa: N803
from __future__ import annotations
import torch
try:
import triton
import triton.language as tl
except Exception as exc: # pragma: no cover - depends on optional Triton install
triton = None
tl = None
_LOAD_ERROR: Exception | None = exc
else:
_LOAD_ERROR = None
_BLOCK_SIZE = 16
_HEAD_DIM = 128
_BLOCK_N = 128
def is_available() -> bool:
return _LOAD_ERROR is None and triton is not None and tl is not None
def load_error() -> Exception | None:
return _LOAD_ERROR
if is_available():
@triton.jit
def _small_decode_kernel(
Q,
K,
V,
BLOCK_TABLE,
SEQ_LENS,
OUT,
SCALE: tl.constexpr,
MAX_SEQ_LEN: tl.constexpr,
MAX_BLOCKS_PER_SEQ: tl.constexpr,
NUM_Q_HEADS: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
KV_GROUP: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_N: tl.constexpr,
GUARD_MAX_SEQ_LEN: tl.constexpr,
):
batch_id = tl.program_id(0)
kv_head = tl.program_id(1)
offs_h = tl.arange(0, BLOCK_H)
offs_d = tl.arange(0, BLOCK_D)
offs_n = tl.arange(0, BLOCK_N)
q_heads = kv_head * KV_GROUP + offs_h
mask_h = offs_h < KV_GROUP
q_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
q = tl.load(Q + q_offsets, mask=mask_h[:, None], other=0.0)
seq_len = tl.load(SEQ_LENS + batch_id)
if GUARD_MAX_SEQ_LEN and seq_len > GUARD_MAX_SEQ_LEN:
return
m_i = tl.full((BLOCK_H,), -float("inf"), tl.float32)
l_i = tl.zeros((BLOCK_H,), tl.float32)
acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32)
start_n = 0
while start_n < seq_len:
cur_n = start_n + offs_n
mask_n = cur_n < seq_len
logical_block = cur_n // 16
block_offset = cur_n - logical_block * 16
physical_block = tl.load(
BLOCK_TABLE + batch_id * MAX_BLOCKS_PER_SEQ + logical_block,
mask=mask_n,
other=0,
)
kv_offsets = (
(physical_block[:, None] * 16 + block_offset[:, None]) * NUM_KV_HEADS + kv_head
) * BLOCK_D + offs_d[None, :]
k = tl.load(K + kv_offsets, mask=mask_n[:, None], other=0.0)
qk = tl.dot(q, tl.trans(k)) * SCALE
qk = tl.where(mask_h[:, None] & mask_n[None, :], qk, -float("inf"))
v = tl.load(V + kv_offsets, mask=mask_n[:, None], other=0.0)
m_new = tl.maximum(m_i, tl.max(qk, axis=1))
p = tl.exp(qk - m_new[:, None])
alpha = tl.exp(m_i - m_new)
acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v)
l_i = l_i * alpha + tl.sum(p, axis=1)
m_i = m_new
start_n += BLOCK_N
denom = tl.where(l_i > 0.0, l_i, 1.0)
result = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0)
out_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
tl.store(OUT + out_offsets, result, mask=mask_h[:, None])
@triton.jit
def _split_partial_decode_kernel(
Q,
K,
V,
BLOCK_TABLE,
SEQ_LENS,
PARTIAL_M,
PARTIAL_L,
PARTIAL_ACC,
SCALE: tl.constexpr,
MAX_BLOCKS_PER_SEQ: tl.constexpr,
NUM_Q_HEADS: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
KV_GROUP: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_N: tl.constexpr,
SPLIT_TOKENS: tl.constexpr,
BATCH_SIZE: tl.constexpr,
MIN_SEQ_LEN: tl.constexpr,
):
batch_id = tl.program_id(0)
kv_head = tl.program_id(1)
split_id = tl.program_id(2)
offs_h = tl.arange(0, BLOCK_H)
offs_d = tl.arange(0, BLOCK_D)
offs_n = tl.arange(0, BLOCK_N)
q_heads = kv_head * KV_GROUP + offs_h
mask_h = offs_h < KV_GROUP
partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads
partial_acc_offsets = partial_head_offsets[:, None] * BLOCK_D + offs_d[None, :]
seq_len = tl.load(SEQ_LENS + batch_id)
if MIN_SEQ_LEN and seq_len < MIN_SEQ_LEN:
return
m_i = tl.full((BLOCK_H,), -float("inf"), tl.float32)
l_i = tl.zeros((BLOCK_H,), tl.float32)
acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32)
split_begin = split_id * SPLIT_TOKENS
if split_begin >= seq_len:
tl.store(PARTIAL_M + partial_head_offsets, m_i, mask=mask_h)
tl.store(PARTIAL_L + partial_head_offsets, l_i, mask=mask_h)
tl.store(PARTIAL_ACC + partial_acc_offsets, acc, mask=mask_h[:, None])
return
q_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
q = tl.load(Q + q_offsets, mask=mask_h[:, None], other=0.0)
for split_offset in tl.range(0, SPLIT_TOKENS, BLOCK_N):
cur_n = split_begin + split_offset + offs_n
mask_n = cur_n < seq_len
logical_block = cur_n // 16
block_offset = cur_n - logical_block * 16
physical_block = tl.load(
BLOCK_TABLE + batch_id * MAX_BLOCKS_PER_SEQ + logical_block,
mask=mask_n,
other=0,
)
kv_offsets = (
(physical_block[:, None] * 16 + block_offset[:, None]) * NUM_KV_HEADS + kv_head
) * BLOCK_D + offs_d[None, :]
k = tl.load(K + kv_offsets, mask=mask_n[:, None], other=0.0)
qk = tl.dot(q, tl.trans(k)) * SCALE
qk = tl.where(mask_h[:, None] & mask_n[None, :], qk, -float("inf"))
v = tl.load(V + kv_offsets, mask=mask_n[:, None], other=0.0)
m_new = tl.maximum(m_i, tl.max(qk, axis=1))
p = tl.exp(qk - m_new[:, None])
alpha = tl.exp(m_i - m_new)
acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v)
l_i = l_i * alpha + tl.sum(p, axis=1)
m_i = m_new
tl.store(PARTIAL_M + partial_head_offsets, m_i, mask=mask_h)
tl.store(PARTIAL_L + partial_head_offsets, l_i, mask=mask_h)
tl.store(PARTIAL_ACC + partial_acc_offsets, acc, mask=mask_h[:, None])
@triton.jit
def _split_combine_decode_kernel(
SEQ_LENS,
PARTIAL_M,
PARTIAL_L,
PARTIAL_ACC,
OUT,
NUM_Q_HEADS: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
KV_GROUP: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_D: tl.constexpr,
BATCH_SIZE: tl.constexpr,
NUM_SPLITS: tl.constexpr,
MIN_SEQ_LEN: tl.constexpr,
):
batch_id = tl.program_id(0)
kv_head = tl.program_id(1)
seq_len = tl.load(SEQ_LENS + batch_id)
if MIN_SEQ_LEN and seq_len < MIN_SEQ_LEN:
return
offs_h = tl.arange(0, BLOCK_H)
offs_d = tl.arange(0, BLOCK_D)
q_heads = kv_head * KV_GROUP + offs_h
mask_h = offs_h < KV_GROUP
global_m = tl.full((BLOCK_H,), -float("inf"), tl.float32)
for split_id in tl.range(0, NUM_SPLITS):
partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads
split_m = tl.load(PARTIAL_M + partial_head_offsets, mask=mask_h, other=-float("inf"))
global_m = tl.maximum(global_m, split_m)
global_l = tl.zeros((BLOCK_H,), tl.float32)
acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32)
for split_id in tl.range(0, NUM_SPLITS):
partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads
split_m = tl.load(PARTIAL_M + partial_head_offsets, mask=mask_h, other=-float("inf"))
split_l = tl.load(PARTIAL_L + partial_head_offsets, mask=mask_h, other=0.0)
weight = tl.exp(split_m - global_m)
partial_acc_offsets = partial_head_offsets[:, None] * BLOCK_D + offs_d[None, :]
split_acc = tl.load(PARTIAL_ACC + partial_acc_offsets, mask=mask_h[:, None], other=0.0)
global_l += split_l * weight
acc += split_acc * weight[:, None]
denom = tl.where(global_l > 0.0, global_l, 1.0)
result = tl.where(global_l[:, None] > 0.0, acc / denom[:, None], 0.0)
out_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
tl.store(OUT + out_offsets, result, mask=mask_h[:, None])
def fish_decode_kvcache_attn_triton(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
block_table: torch.Tensor,
seq_lens: torch.Tensor,
out: torch.Tensor,
*,
scale: float,
max_seq_len: int,
small_path_max_seq_len: int,
long_split_tokens: int,
partial_m: torch.Tensor,
partial_l: torch.Tensor,
partial_acc: torch.Tensor,
) -> torch.Tensor:
if not is_available():
raise RuntimeError(f"Fish Triton attention is unavailable: {_LOAD_ERROR!r}")
batch_size, num_q_heads, head_dim = query.shape
block_size = key_cache.shape[1]
num_kv_heads = key_cache.shape[2]
if head_dim != _HEAD_DIM or block_size != _BLOCK_SIZE:
raise RuntimeError("Fish Triton attention only supports head_dim=128 and block_size=16")
if num_q_heads % num_kv_heads != 0:
raise RuntimeError("num_q_heads must be divisible by num_kv_heads")
kv_group = num_q_heads // num_kv_heads
block_h = triton.next_power_of_2(kv_group)
max_blocks_per_seq = block_table.shape[1]
if max_seq_len <= small_path_max_seq_len:
_small_decode_kernel[(batch_size, num_kv_heads)](
query,
key_cache,
value_cache,
block_table,
seq_lens,
out,
SCALE=float(scale),
MAX_SEQ_LEN=int(max_seq_len),
MAX_BLOCKS_PER_SEQ=max_blocks_per_seq,
NUM_Q_HEADS=num_q_heads,
NUM_KV_HEADS=num_kv_heads,
KV_GROUP=kv_group,
BLOCK_H=block_h,
BLOCK_D=head_dim,
BLOCK_N=_BLOCK_N,
GUARD_MAX_SEQ_LEN=0,
num_warps=4,
num_stages=3,
)
return out
_small_decode_kernel[(batch_size, num_kv_heads)](
query,
key_cache,
value_cache,
block_table,
seq_lens,
out,
SCALE=float(scale),
MAX_SEQ_LEN=small_path_max_seq_len,
MAX_BLOCKS_PER_SEQ=max_blocks_per_seq,
NUM_Q_HEADS=num_q_heads,
NUM_KV_HEADS=num_kv_heads,
KV_GROUP=kv_group,
BLOCK_H=block_h,
BLOCK_D=head_dim,
BLOCK_N=_BLOCK_N,
GUARD_MAX_SEQ_LEN=small_path_max_seq_len,
num_warps=4,
num_stages=3,
)
num_splits = triton.cdiv(int(max_seq_len), long_split_tokens)
expected_partial_acc_shape = (num_splits, batch_size, num_q_heads, head_dim)
if tuple(partial_acc.shape) != expected_partial_acc_shape:
raise RuntimeError(
"Fish Triton long attention workspace shape mismatch: "
f"expected partial_acc={expected_partial_acc_shape}, got {tuple(partial_acc.shape)}"
)
_split_partial_decode_kernel[(batch_size, num_kv_heads, num_splits)](
query,
key_cache,
value_cache,
block_table,
seq_lens,
partial_m,
partial_l,
partial_acc,
SCALE=float(scale),
MAX_BLOCKS_PER_SEQ=max_blocks_per_seq,
NUM_Q_HEADS=num_q_heads,
NUM_KV_HEADS=num_kv_heads,
KV_GROUP=kv_group,
BLOCK_H=block_h,
BLOCK_D=head_dim,
BLOCK_N=_BLOCK_N,
SPLIT_TOKENS=long_split_tokens,
BATCH_SIZE=batch_size,
MIN_SEQ_LEN=small_path_max_seq_len + 1,
num_warps=4,
num_stages=3,
)
_split_combine_decode_kernel[(batch_size, num_kv_heads)](
seq_lens,
partial_m,
partial_l,
partial_acc,
out,
NUM_Q_HEADS=num_q_heads,
NUM_KV_HEADS=num_kv_heads,
KV_GROUP=kv_group,
BLOCK_H=block_h,
BLOCK_D=head_dim,
BATCH_SIZE=batch_size,
NUM_SPLITS=num_splits,
MIN_SEQ_LEN=small_path_max_seq_len + 1,
num_warps=4,
num_stages=3,
)
return out
@@ -0,0 +1,68 @@
# Fish Speech S2 Pro high-concurrency dual-GPU deploy config.
#
# Stage0 SlowAR runs on GPU0 and Stage1 DAC runs on GPU1. Use this config
# when the deployment can trade an extra GPU for lower Stage0/Stage1
# interference between the AR and DAC stages.
async_chunk: true
enable_chunked_prefill: false
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
shm_threshold_bytes: 65536
codec_streaming: true
connector_get_sleep_s: 0.001
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
fish_speech_tensor_codes: true
fish_speech_single_initial_chunk: true
fish_speech_backlog_codec_chunk_frames: 50
fish_speech_backlog_load_threshold: 0.75
codec_chunk_frames: 25
codec_left_context_frames: 25
initial_codec_chunk_frames: 4
fish_speech_dac_dtype: fp16
fish_speech_dac_max_padded_frames: 256
fish_speech_dac_max_batch: 0
stages:
- stage_id: 0
max_num_seqs: 64
gpu_memory_utilization: 0.85
enforce_eager: false
trust_remote_code: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 16384
max_model_len: 16384
devices: "0"
output_connectors:
to_stage_1: connector_of_shared_memory
default_sampling_params:
temperature: 0.8
top_k: 30
top_p: 0.9
max_tokens: 2048
seed: 42
repetition_penalty: 1.0
- stage_id: 1
max_num_seqs: 64
gpu_memory_utilization: 0.25
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 16384
max_model_len: 16384
devices: "1"
input_connectors:
from_stage_0: connector_of_shared_memory
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
repetition_penalty: 1.0
@@ -0,0 +1,68 @@
# Fish Speech S2 Pro high-concurrency single-GPU deploy config.
#
# Stage0 SlowAR and Stage1 DAC share GPU0. This config keeps the optimized
# Fish runtime path explicit instead of changing the default
# fish_qwen3_omni.yaml deployment.
async_chunk: true
enable_chunked_prefill: false
connectors:
connector_of_shared_memory:
name: SharedMemoryConnector
extra:
shm_threshold_bytes: 65536
codec_streaming: true
connector_get_sleep_s: 0.001
connector_get_max_wait_first_chunk: 3000
connector_get_max_wait: 300
fish_speech_tensor_codes: true
fish_speech_single_initial_chunk: true
fish_speech_backlog_codec_chunk_frames: 50
fish_speech_backlog_load_threshold: 0.75
codec_chunk_frames: 25
codec_left_context_frames: 25
initial_codec_chunk_frames: 4
fish_speech_dac_dtype: fp16
fish_speech_dac_max_padded_frames: 256
fish_speech_dac_max_batch: 0
stages:
- stage_id: 0
max_num_seqs: 64
gpu_memory_utilization: 0.72
enforce_eager: false
trust_remote_code: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 16384
max_model_len: 16384
devices: "0"
output_connectors:
to_stage_1: connector_of_shared_memory
default_sampling_params:
temperature: 0.8
top_k: 30
top_p: 0.9
max_tokens: 2048
seed: 42
repetition_penalty: 1.0
- stage_id: 1
max_num_seqs: 64
gpu_memory_utilization: 0.18
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 16384
max_model_len: 16384
devices: "0"
input_connectors:
from_stage_0: connector_of_shared_memory
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 65536
seed: 42
repetition_penalty: 1.0
@@ -64,6 +64,21 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
self.requests_origin_status = {}
self.requests_num_chunks_sent: dict[str, int] = defaultdict(int)
@staticmethod
def _is_truthy_scalar(value: Any) -> bool:
if isinstance(value, torch.Tensor):
return value.numel() == 1 and bool(value.item())
return bool(value) if value is not None else False
@staticmethod
def _confirmed_num_computed_tokens(request: Request) -> int:
# vLLM async scheduling advances num_computed_tokens with output
# placeholders before the corresponding token is committed. Connector
# chunk send watermarks must use only committed tokens.
num_computed = int(getattr(request, "num_computed_tokens", 0))
num_placeholders = int(getattr(request, "num_output_placeholders", 0) or 0)
return max(0, num_computed - num_placeholders)
@classmethod
def create_connector(cls, model_config: Any):
connector_config = getattr(model_config, "stage_connector_config", None)
@@ -119,16 +134,19 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
request: Request object
"""
confirmed_num_computed_tokens = self._confirmed_num_computed_tokens(request)
# If the request is preempted, skip the already saved chunks.
if request.num_computed_tokens < self.requests_num_chunks_sent.get(request.external_req_id, 0):
if confirmed_num_computed_tokens < self.requests_num_chunks_sent.get(request.external_req_id, 0):
logger.warning(
f"Enqueue save_async for request {request.external_req_id}, "
f"request.num_computed_tokens={request.num_computed_tokens}, "
f"request.num_output_placeholders={getattr(request, 'num_output_placeholders', 0)}, "
f"previous_chunks_sent={self.requests_num_chunks_sent.get(request.external_req_id, 0)}"
)
return
self.requests_num_chunks_sent[request.external_req_id] = request.num_computed_tokens
self.requests_num_chunks_sent[request.external_req_id] = confirmed_num_computed_tokens
task = {
"pooling_output": pooling_output,
"request": request,
@@ -166,24 +184,36 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
self.get_req_chunk[req_id] += 1
meta = payload_data.get("meta", {})
payload_finished = self._is_truthy_scalar(meta.get("finished"))
if self.model_mode == "ar":
request.additional_information = payload_data
if meta.get("finished"):
if payload_finished:
self.finished_requests.add(req_id)
else:
if meta.get("finished"):
if payload_finished:
self.finished_requests.add(req_id)
new_ids = payload_data.get("codes", {}).get("audio")
if isinstance(new_ids, torch.Tensor):
has_tensor_codes = isinstance(new_ids, torch.Tensor)
use_tensor_codes = has_tensor_codes and new_ids.ndim >= 2
if use_tensor_codes:
request.prompt_token_ids = [0] if new_ids.numel() > 0 else []
elif has_tensor_codes:
new_ids = new_ids.tolist()
elif new_ids is None:
new_ids = []
request.prompt_token_ids = new_ids
request.prompt_token_ids = new_ids
if not use_tensor_codes:
request.prompt_token_ids = new_ids
prev_info = getattr(request, "additional_information", None)
info = dict(prev_info) if isinstance(prev_info, dict) else {}
for key, value in payload_data.items():
if key == "codes":
if use_tensor_codes and isinstance(value, dict):
existing_sub = info.get(key)
merged_sub = dict(existing_sub) if isinstance(existing_sub, dict) else {}
merged_sub.update(value)
info[key] = merged_sub
continue
if isinstance(value, dict):
existing_sub = info.get(key)
@@ -199,8 +229,13 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
request.num_computed_tokens = 0
# Empty chunk with more data expected: keep polling.
if not new_ids and not meta.get("finished"):
return True
has_new_ids = bool(new_ids.numel()) if use_tensor_codes else bool(new_ids)
if not has_new_ids and not payload_finished:
# The base recv loop treats False as "not ready yet" and
# requeues the request. Do not mark an empty non-terminal
# chunk as ready, otherwise Stage1 can consume before the
# first DAC frame arrives.
return False
# Mark as finished for consumption
self._finished_load_reqs.add(req_id)
@@ -82,10 +82,12 @@ class OmniMsgpackEncoder:
def _encode_tensor(self, tensor: torch.Tensor) -> dict[str, Any]:
"""Encode torch.Tensor to dict."""
t = tensor.detach().contiguous().cpu()
t = tensor.detach().cpu()
# Handle 0-dimensional (scalar) tensors by reshaping to 1D first
if t.dim() == 0:
t = t.reshape(1)
if not t.is_contiguous():
t = t.contiguous()
t = t.view(torch.uint8)
return {
_TENSOR_MARKER: True,
@@ -11,6 +11,7 @@ Install with: ``pip install fish-speech``
from __future__ import annotations
import inspect
import os
from collections.abc import Iterable
from typing import Any
@@ -33,6 +34,41 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
logger = init_logger(__name__)
def _connector_extra_config(vllm_config: VllmConfig) -> dict[str, Any]:
model_config = getattr(vllm_config, "model_config", None)
connector_cfg = getattr(model_config, "stage_connector_config", None)
if isinstance(connector_cfg, dict):
return connector_cfg.get("extra", connector_cfg)
extra = getattr(connector_cfg, "extra", None)
return extra if isinstance(extra, dict) else {}
def _get_int_config(extra_cfg: dict[str, Any], default: int, *names: str) -> int:
value = None
for name in names:
if name in extra_cfg:
value = extra_cfg[name]
break
if value is None:
return default
try:
return max(0, int(value))
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid Fish Speech DAC integer config {names[0]}={value!r}") from exc
def _get_dac_dtype(extra_cfg: dict[str, Any]) -> torch.dtype:
value = extra_cfg.get("fish_speech_dac_dtype", "float32")
value = str(value).strip().lower()
if value in {"fp16", "float16", "half"}:
return torch.float16
if value in {"bf16", "bfloat16"}:
return torch.bfloat16
if value in {"fp32", "float32", ""}:
return torch.float32
raise ValueError(f"Invalid Fish Speech DAC dtype: {value!r}")
class FishSpeechDACDecoder(nn.Module):
"""Stage-1 DAC decoder for Fish Speech S2 Pro (GenerationModelRunner).
@@ -58,6 +94,43 @@ class FishSpeechDACDecoder(nn.Module):
self._output_sample_rate: int = DAC_SAMPLE_RATE
self._hop_length: int = DAC_HOP_LENGTH
self._logged_codec_stats = False
self._codec_decode_takes_lengths: bool | None = None
extra_cfg = _connector_extra_config(vllm_config)
self._dac_dtype = _get_dac_dtype(extra_cfg)
self._decode_batch_max_padded_frames = _get_int_config(
extra_cfg,
0,
"fish_speech_dac_max_padded_frames",
"dac_max_padded_frames",
)
self._decode_batch_max_batch = _get_int_config(
extra_cfg,
0,
"fish_speech_dac_max_batch",
"dac_max_batch",
)
def _decode_codes(
self,
codes_bqf: torch.Tensor,
feature_lengths: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self._codec is not None
if self._codec_decode_takes_lengths:
return self._codec.decode(codes_bqf, feature_lengths)
if hasattr(self._codec, "from_indices"):
wav_batch = self._codec.from_indices(codes_bqf)
else:
wav_batch = self._codec.decode(codes_bqf)
max_frames = max(int(codes_bqf.shape[-1]), 1)
scale = wav_batch.shape[-1] / max_frames
audio_lengths = torch.clamp(
torch.round(feature_lengths.to(torch.float32) * scale).to(torch.long),
min=0,
max=wav_batch.shape[-1],
)
return wav_batch, audio_lengths
def _bake_weight_norm(self, codec: nn.Module) -> None:
baked = 0
@@ -149,14 +222,16 @@ class FishSpeechDACDecoder(nn.Module):
codec.quantizer.downsample = None
device = self.vllm_config.device_config.device
codec = codec.to(device=device, dtype=torch.float32)
codec = codec.to(device=device, dtype=self._dac_dtype)
codec.eval()
self._codec = codec
self._codec_decode_takes_lengths = len(inspect.signature(codec.decode).parameters) >= 2
logger.info(
"Fish Speech DAC codec loaded from %s (device=%s, sample_rate=%d)",
"Fish Speech DAC codec loaded from %s (device=%s, dtype=%s, sample_rate=%d)",
codec_path,
device,
self._dac_dtype,
self._output_sample_rate,
)
@@ -188,6 +263,28 @@ class FishSpeechDACDecoder(nn.Module):
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
return [ids]
def _codes_from_runtime_info(
self,
info: dict[str, Any] | None,
device: torch.device,
) -> torch.Tensor | None:
if not isinstance(info, dict):
return None
codes = info.get("codes", {}).get("audio")
if not isinstance(codes, torch.Tensor) or codes.numel() == 0:
return None
q = self._num_codebooks
codes = codes.to(device=device, dtype=torch.long, non_blocking=True).contiguous()
if codes.ndim != 2 or codes.shape[0] != q:
logger.warning(
"DAC tensor codes must have shape [num_codebooks, frames], got %s for num_codebooks=%d.",
tuple(codes.shape),
q,
)
return None
return codes
@torch.no_grad()
def forward(
self,
@@ -203,14 +300,14 @@ class FishSpeechDACDecoder(nn.Module):
input_ids layout per request: flat codes [num_codebooks * num_frames].
Codes are codebook-major: [cb0_f0, cb0_f1, ..., cb0_fN, cb1_f0, ...].
"""
self._ensure_codec_loaded()
assert self._codec is not None
q = self._num_codebooks
sr_val = self._output_sample_rate
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
empty = torch.zeros((0,), dtype=torch.float32)
self._ensure_codec_loaded()
assert self._codec is not None
if input_ids is None or input_ids.numel() == 0:
return OmniOutput(
text_hidden_states=None,
@@ -235,9 +332,22 @@ class FishSpeechDACDecoder(nn.Module):
left_context_size[i] = meta["left_context_size"]
for i, req_ids in enumerate(request_ids_list):
ctx_frames = left_context_size[i]
runtime_codes = (
self._codes_from_runtime_info(runtime_additional_information[i], ids.device)
if runtime_additional_information is not None and i < len(runtime_additional_information)
else None
)
if runtime_codes is not None:
frames = runtime_codes.shape[1]
parsed_ctx_frames[i] = ctx_frames
parsed_total_frames[i] = frames
valid_codes_qf.append(runtime_codes)
valid_indices.append(i)
continue
if req_ids.numel() < 1:
continue
ctx_frames = left_context_size[i]
flat = req_ids
n = flat.numel()
if n == 0 or n % q != 0:
@@ -279,25 +389,61 @@ class FishSpeechDACDecoder(nn.Module):
except Exception:
pass
feature_lengths = torch.tensor(
[codes_qf.shape[1] for codes_qf in valid_codes_qf],
device=valid_codes_qf[0].device,
dtype=torch.long,
)
max_frames = int(feature_lengths.max().item())
batch_size = len(valid_codes_qf)
wav_tensors: list[torch.Tensor | None] = [None] * len(valid_codes_qf)
codes_bqf = torch.zeros(
(batch_size, q, max_frames),
device=valid_codes_qf[0].device,
dtype=torch.long,
)
for i, codes_qf in enumerate(valid_codes_qf):
frame_count = int(feature_lengths[i].item())
codes_bqf[i, :, :frame_count] = codes_qf
def _iter_bounded_groups(group: list[tuple[int, torch.Tensor]]) -> Iterable[list[tuple[int, torch.Tensor]]]:
max_padded_frames = int(getattr(self, "_decode_batch_max_padded_frames", 0))
max_batch = int(getattr(self, "_decode_batch_max_batch", 0))
if max_padded_frames <= 0 and max_batch <= 0:
yield group
return
with torch.amp.autocast("cuda", enabled=False):
wav_batch, audio_lengths = self._codec.decode(codes_bqf, feature_lengths)
current: list[tuple[int, torch.Tensor]] = []
current_max_frames = 0
for item in sorted(group, key=lambda pair: int(pair[1].shape[1])):
frames = int(item[1].shape[1])
next_max_frames = max(current_max_frames, frames)
next_batch_size = len(current) + 1
exceeds_batch = max_batch > 0 and next_batch_size > max_batch
exceeds_work = max_padded_frames > 0 and next_batch_size * next_max_frames > max_padded_frames
if current and (exceeds_batch or exceeds_work):
yield current
current = [item]
current_max_frames = frames
else:
current.append(item)
current_max_frames = next_max_frames
if current:
yield current
def _decode_group(group: list[tuple[int, torch.Tensor]]) -> None:
actual_frames = [int(codes_qf.shape[1]) for _, codes_qf in group]
target_frames = max(actual_frames)
batch_size = len(group)
first_codes = group[0][1]
feature_lengths = torch.tensor(actual_frames, device=first_codes.device, dtype=torch.long)
codes_bqf = torch.zeros(
(batch_size, q, target_frames),
device=first_codes.device,
dtype=torch.long,
)
for row, (_, codes_qf) in enumerate(group):
frame_count = int(codes_qf.shape[1])
codes_bqf[row, :, :frame_count] = codes_qf
with torch.amp.autocast("cuda", enabled=False):
wav_batch, audio_lengths = self._decode_codes(codes_bqf, feature_lengths)
audio_lengths_list = (
audio_lengths.detach().to(device="cpu", dtype=torch.long).reshape(-1).tolist()
if audio_lengths.numel() > 0
else []
)
for row, (j, _) in enumerate(group):
audio_len = int(audio_lengths_list[row]) if len(audio_lengths_list) > row else int(wav_batch.shape[-1])
wav_tensors[j] = wav_batch[row, 0, :audio_len]
for bounded_group in _iter_bounded_groups(list(enumerate(valid_codes_qf))):
_decode_group(bounded_group)
audios: list[torch.Tensor] = [empty] * num_req
srs = [sr_tensor] * num_req
@@ -305,8 +451,8 @@ class FishSpeechDACDecoder(nn.Module):
for j, idx in enumerate(valid_indices):
ctx_frames = parsed_ctx_frames[idx]
total_frames = parsed_total_frames[idx]
audio_len = int(audio_lengths[j].item()) if audio_lengths.numel() > j else int(wav_batch.shape[-1])
wav = wav_batch[j, 0, :audio_len]
wav = wav_tensors[j]
assert wav is not None
# Trim context frames (left overlap for streaming).
if ctx_frames > 0:
# Decode length may deviate from (frames * hop_length) due to model
@@ -4,14 +4,13 @@
autoregressively after each Slow AR step. Analogous to Qwen3 TTS's
CodePredictor but with its own embedding table, RoPE, and output head.
Uses re-prefill (no KV cache): each AR step forwards the full growing
sequence through the 4-layer transformer. This trades ~O(T^2) attention
FLOPs (negligible for T=10, 4 layers) for zero KV cache management.
Uses a tiny per-call KV cache: each AR step forwards one codebook token while
reusing the previous Fast AR keys/values.
Optimisations:
- torch.compile on model forward (kernel fusion)
- Pre-allocated projection / embedding buffer
- Pre-allocated position_ids
- Pre-allocated per-layer KV cache for the short codebook loop
- Inline sampling
"""
@@ -51,7 +50,8 @@ logger = init_logger(__name__)
class _FastARAttention(nn.Module):
"""Multi-head attention using F.scaled_dot_product_attention (SDPA).
Supports fused QKV, RoPE, optional q/k normalization, and native GQA.
Supports fused QKV, RoPE, optional q/k normalization, native GQA, and a
small one-token decode path for the residual codebook loop.
Input: [B, seq_len, hidden_size].
"""
@@ -124,6 +124,43 @@ class _FastARAttention(nn.Module):
output, _ = self.o_proj(attn_out)
return output.view(bsz, seq_len, -1)
def forward_one(
self,
hidden_states: torch.Tensor,
position_ids: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
cache_pos: int,
) -> torch.Tensor:
bsz = int(hidden_states.shape[0])
qkv, _ = self.qkv_proj(hidden_states.reshape(bsz, -1))
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
if self.q_norm is not None:
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
q, k = self.rotary_emb(position_ids.reshape(-1), q, k)
q = q.view(bsz, self.num_heads, self.head_dim).unsqueeze(2)
k = k.view(bsz, self.num_kv_heads, self.head_dim)
v = v.view(bsz, self.num_kv_heads, self.head_dim)
k_cache[:bsz, :, cache_pos, :] = k
v_cache[:bsz, :, cache_pos, :] = v
attn_out = F.scaled_dot_product_attention(
q,
k_cache[:bsz, :, : cache_pos + 1, :],
v_cache[:bsz, :, : cache_pos + 1, :],
scale=self.scaling,
is_causal=False,
enable_gqa=self._use_gqa,
)
attn_out = attn_out.transpose(1, 2).reshape(bsz, -1)
output, _ = self.o_proj(attn_out)
return output
class _FastARMLP(nn.Module):
"""SiLU-gated MLP, matching Qwen3/LLaMA MLP structure."""
@@ -175,6 +212,25 @@ class _FastARDecoderLayer(nn.Module):
hidden_states = residual + hidden_states
return hidden_states
def forward_one(
self,
hidden_states: torch.Tensor,
position_ids: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
cache_pos: int,
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn.forward_one(hidden_states, position_ids, k_cache, v_cache, cache_pos)
hidden_states = residual + hidden_states
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
return hidden_states
# ===================================================================
# Fast AR Transformer Model
@@ -199,6 +255,25 @@ class FishSpeechFastARModel(nn.Module):
hidden_states = layer(hidden_states, position_ids)
return hidden_states
def forward_one(
self,
input_embed: torch.Tensor,
position_ids: torch.Tensor,
k_caches: torch.Tensor,
v_caches: torch.Tensor,
cache_pos: int,
) -> torch.Tensor:
hidden_states = input_embed
for layer_idx, layer in enumerate(self.layers):
hidden_states = layer.forward_one(
hidden_states,
position_ids,
k_caches[layer_idx],
v_caches[layer_idx],
cache_pos,
)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
@@ -259,13 +334,13 @@ class FishSpeechFastARModel(nn.Module):
class FishSpeechFastAR(nn.Module):
"""vLLM-native Fast AR for Fish Speech (residual codebooks).
Re-prefill approach: each AR step forwards the full growing sequence
through the 4-layer transformer. No KV cache needed.
Decode approach: each residual-codebook step forwards one token through
the 4-layer transformer with a per-call KV cache.
Optimisations over baseline:
1. torch.compile on model forward -- kernel fusion.
2. Pre-allocated embedding buffer [B, max_seq, H].
3. Pre-allocated position_ids.
1. Pre-allocated embedding buffer [B, max_seq, H].
2. Pre-allocated position_ids.
3. Per-call KV cache for the short residual-codebook loop.
4. Inline sampling.
"""
@@ -307,6 +382,8 @@ class FishSpeechFastAR(nn.Module):
# Pre-allocated buffers (lazily initialised on first forward).
self._embed_buf: torch.Tensor | None = None
self._pos_ids: torch.Tensor | None = None
self._k_cache: torch.Tensor | None = None
self._v_cache: torch.Tensor | None = None
self._compiled_model_fwd: object | None = None
self._compile_attempted = False
self._compile_failed = False
@@ -316,13 +393,27 @@ class FishSpeechFastAR(nn.Module):
max_seq = self._num_codebooks + 1 # hidden_state + num_codebooks codes
if (
self._embed_buf is not None
and self._pos_ids is not None
and self._k_cache is not None
and self._v_cache is not None
and self._embed_buf.shape[0] >= bsz
and self._embed_buf.device == device
and self._embed_buf.dtype == dtype
and self._pos_ids.shape[0] >= bsz
and self._pos_ids.shape[1] == max_seq
and self._pos_ids.device == device
and self._k_cache.shape[1] >= bsz
and self._k_cache.device == device
and self._k_cache.dtype == dtype
):
return
self._embed_buf = torch.zeros(bsz, max_seq, self._fast_dim, dtype=dtype, device=device)
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device)
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device).unsqueeze(0).expand(bsz, -1).contiguous()
num_layers = self.config.num_hidden_layers
num_kv_heads = self.config.num_key_value_heads
head_dim = self.config.head_dim
self._k_cache = torch.empty(num_layers, bsz, num_kv_heads, max_seq, head_dim, dtype=dtype, device=device)
self._v_cache = torch.empty_like(self._k_cache)
def _setup_compile(self) -> None:
if self._compile_attempted:
@@ -375,20 +466,10 @@ class FishSpeechFastAR(nn.Module):
torch.accelerator.synchronize(device)
@torch.inference_mode()
def _run_model(self, step_input: torch.Tensor, step_pos_ids: torch.Tensor, bsz: int) -> torch.Tensor:
if self._disable_compile_for_graph:
model_fwd = self._compiled_model_fwd or self.model.forward
else:
model_fwd = self._compiled_model_fwd if bsz == 1 else self.model.forward
try:
return model_fwd(step_input, step_pos_ids)
except Exception as exc:
if model_fwd is self.model.forward or self._compile_failed:
raise
self._compile_failed = True
self._compiled_model_fwd = self.model.forward
logger.warning("Fish Speech Fast AR torch.compile fallback to eager after runtime failure: %s", exc)
return self.model.forward(step_input, step_pos_ids)
def _run_model_one(self, input_embed: torch.Tensor, step_pos_ids: torch.Tensor, cache_pos: int) -> torch.Tensor:
assert self._k_cache is not None
assert self._v_cache is not None
return self.model.forward_one(input_embed, step_pos_ids, self._k_cache, self._v_cache, cache_pos)
@torch.inference_mode()
def forward(
@@ -401,6 +482,7 @@ class FishSpeechFastAR(nn.Module):
top_k: int = 30,
top_p: float = 0.9,
seed: int | None = None,
generator: torch.Generator | None = None,
) -> torch.Tensor:
"""Predict residual codebook codes 0..num_codebooks-1 autoregressively.
@@ -428,7 +510,6 @@ class FishSpeechFastAR(nn.Module):
all_codes[:, 0] = semantic_code
self._ensure_buffers(bsz, device, dtype)
self._setup_compile()
embed_buf = self._embed_buf
pos_ids = self._pos_ids
@@ -445,8 +526,7 @@ class FishSpeechFastAR(nn.Module):
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
# Create a seeded generator for deterministic residual codebook sampling.
generator = None
if seed is not None and use_sampling:
if generator is None and seed is not None and use_sampling:
generator = torch.Generator(device=device)
generator.manual_seed(seed)
@@ -455,15 +535,11 @@ class FishSpeechFastAR(nn.Module):
# codebooks only have 1024 entries. Truncate logits for steps > 0.
residual_codebook_size = 1024
for step in range(1, num_cb):
seq_len = step + 1
step_input = embed_buf[:bsz, :seq_len, :]
# Use a dense 2D position tensor for every batch size; stride-0
# views from expand() were fragile under compiled execution.
step_pos_ids = pos_ids[:seq_len].unsqueeze(0).repeat(bsz, 1)
self._run_model_one(embed_buf[:bsz, 0, :], pos_ids[:bsz, 0], 0)
hidden_out = self._run_model(step_input, step_pos_ids, bsz)
logits = self.fast_output(self.fast_norm(hidden_out[:, -1, :]))
for step in range(1, num_cb):
hidden_out = self._run_model_one(embed_buf[:bsz, step, :], pos_ids[:bsz, step], step)
logits = self.fast_output(self.fast_norm(hidden_out))
# Residual codebooks (step >= 1) only have 1024 entries.
if step >= 1:
@@ -199,6 +199,12 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
# Qwen3 transformer backbone.
self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
from vllm_omni.attention.fish_kvcache_attn import is_fish_kvcache_attn_enabled
if is_fish_kvcache_attn_enabled():
from vllm_omni.attention.fish_kvcache_backend import install_fish_kvcache_attn_backend
install_fish_kvcache_attn_backend(self.model)
# Fish Speech uses interleaved (GPT-J) RoPE, not NeoX style.
# vLLM's Qwen3Attention defaults to NeoX (is_neox_style=True).
@@ -672,14 +678,19 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
past_hidden = last_talker_hidden.reshape(bsz, -1).to(dtype=torch.bfloat16, device=dev)
# Run Fast AR to predict all num_codebooks codes.
do_sample = kwargs.get("do_sample")
temperature = kwargs.get("temperature")
top_k = kwargs.get("top_k")
top_p = kwargs.get("top_p")
audio_codes = self.fast_ar(
slow_ar_hidden=past_hidden,
semantic_token_id=input_ids.reshape(bsz),
do_sample=True,
temperature=0.8,
top_k=30,
top_p=0.9,
do_sample=True if do_sample is None else bool(do_sample),
temperature=0.8 if temperature is None else float(temperature),
top_k=30 if top_k is None else int(top_k),
top_p=0.9 if top_p is None else float(top_p),
seed=seed,
generator=kwargs.get("generator"),
) # [B, num_codebooks]
# Add codebook embeddings to the input embedding (from preprocess).
@@ -3,7 +3,6 @@
from typing import Any
import torch
from vllm.logger import init_logger
from vllm_omni.data_entry_keys import (
CodesStruct,
@@ -11,7 +10,85 @@ from vllm_omni.data_entry_keys import (
OmniPayloadStruct,
)
logger = init_logger(__name__)
def _get_connector_extra(transfer_manager: Any) -> dict[str, Any]:
connector = getattr(transfer_manager, "connector", None)
raw_cfg = getattr(connector, "config", {}) or {}
return raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
def _use_tensor_code_payload(transfer_manager: Any) -> bool:
cfg = _get_connector_extra(transfer_manager)
return bool(cfg.get("fish_speech_tensor_codes", False))
def _cfg_bool(cfg: dict[str, Any], key: str, default: bool = False) -> bool:
return bool(cfg.get(key, default))
def _cfg_int(cfg: dict[str, Any], key: str, default: int = 0) -> int:
value = cfg.get(key, default)
try:
return int(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid Fish Speech integer config {key}={value!r}") from exc
def _cfg_float(cfg: dict[str, Any], key: str, default: float) -> float:
value = cfg.get(key, default)
try:
return float(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid Fish Speech float config {key}={value!r}") from exc
def _active_codec_requests(transfer_manager: Any) -> int:
return sum(1 for value in transfer_manager.code_prompt_token_ids.values() if len(value) > 0)
def _ready_codec_requests(transfer_manager: Any, min_frames: int) -> int:
return sum(1 for value in transfer_manager.code_prompt_token_ids.values() if len(value) >= min_frames)
def _select_backlog_chunk_size(
transfer_manager: Any,
cfg: dict[str, Any],
base_chunk_size: int,
current_length: int,
) -> int:
backlog_chunk_size = _cfg_int(
cfg,
"fish_speech_backlog_codec_chunk_frames",
0,
)
if backlog_chunk_size <= base_chunk_size:
return base_chunk_size
capacity = max(1, int(getattr(transfer_manager, "scheduler_max_num_seqs", 1)))
active = _active_codec_requests(transfer_manager)
load_factor = min(active / capacity, 1.0)
threshold = _cfg_float(
cfg,
"fish_speech_backlog_load_threshold",
0.75,
)
if load_factor < threshold:
return base_chunk_size
ready_min = _cfg_int(
cfg,
"fish_speech_backlog_min_ready",
max(2, min(capacity, int(capacity * threshold))),
)
# Include the current request even when the caller has not yet committed
# the latest frame into the transfer-manager map.
ready = _ready_codec_requests(transfer_manager, base_chunk_size)
if current_length >= base_chunk_size:
ready = max(ready, 1)
if ready < ready_min:
return base_chunk_size
return backlog_chunk_size
def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None:
@@ -21,11 +98,18 @@ def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None:
return None
if audio_codes.ndim == 2:
frame = audio_codes[-1]
if frame.numel() == 0 or not bool(frame.any().item()):
valid = pooling_output.get("audio_code_valid")
if isinstance(valid, torch.Tensor) and valid.numel() > 0:
is_valid = bool(valid.reshape(-1)[-1].item())
elif valid is not None:
is_valid = bool(valid)
else:
is_valid = bool(frame.any().item())
if frame.numel() == 0 or not is_valid:
return None
return frame.to(torch.long).reshape(-1)
return frame.to(device="cpu", dtype=torch.long).reshape(-1)
if audio_codes.ndim == 1:
return audio_codes.to(torch.long).reshape(-1)
return audio_codes.to(device="cpu", dtype=torch.long).reshape(-1)
raise ValueError(f"Invalid audio_codes shape for Fish Speech async_chunk: {tuple(audio_codes.shape)}")
@@ -74,20 +158,20 @@ def slow_ar_to_dac_decoder_async_chunk(
"""
request_id = request.external_req_id
finished = bool(is_finished or request.is_finished())
cfg = _get_connector_extra(transfer_manager)
if isinstance(pooling_output, dict):
frame = _extract_last_frame(pooling_output)
if frame is not None:
transfer_manager.code_prompt_token_ids[request_id].append(frame.detach().to(device="cpu", dtype=torch.long))
transfer_manager.code_prompt_token_ids[request_id].append(frame.detach())
elif not finished:
return None
connector = getattr(transfer_manager, "connector", None)
raw_cfg = getattr(connector, "config", {}) or {}
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
chunk_size = int(cfg.get("codec_chunk_frames", 25))
left_context_size_config = int(cfg.get("codec_left_context_frames", 25))
initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0))
configured_initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0))
initial_chunk_size = configured_initial_chunk_size
# Per-request override.
additional_information = getattr(request, "additional_information", None)
@@ -100,7 +184,7 @@ def slow_ar_to_dac_decoder_async_chunk(
if entry.list_data is not None and len(entry.list_data) == 1:
initial_chunk_size = int(entry.list_data[0])
if chunk_size <= 0 or left_context_size_config < 0 or initial_chunk_size < 0:
if chunk_size <= 0 or left_context_size_config < 0 or configured_initial_chunk_size < 0 or initial_chunk_size < 0:
raise ValueError(
f"Invalid codec chunk config: codec_chunk_frames={chunk_size}, "
f"codec_left_context_frames={left_context_size_config}, "
@@ -110,6 +194,7 @@ def slow_ar_to_dac_decoder_async_chunk(
initial_chunk_size = chunk_size
length = len(transfer_manager.code_prompt_token_ids[request_id])
steady_chunk_size = _select_backlog_chunk_size(transfer_manager, cfg, chunk_size, length)
if length <= 0:
if finished:
@@ -119,9 +204,30 @@ def slow_ar_to_dac_decoder_async_chunk(
)
return None
in_initial_phase = initial_chunk_size > 0 and length <= chunk_size
single_initial_chunk = _cfg_bool(
cfg,
"fish_speech_single_initial_chunk",
False,
)
use_first_chunk = initial_chunk_size > 0 and initial_chunk_size < steady_chunk_size
if in_initial_phase:
if single_initial_chunk and use_first_chunk:
if length <= initial_chunk_size:
if not finished and length < initial_chunk_size:
return None
context_length = length if finished and length < initial_chunk_size else initial_chunk_size
else:
adjusted = length - initial_chunk_size
if adjusted <= 0:
return None
if not finished and adjusted % steady_chunk_size != 0:
return None
chunk_length = adjusted % steady_chunk_size
context_length = chunk_length if chunk_length != 0 else steady_chunk_size
end_index = min(length, left_context_size_config + context_length)
left_context_size = max(0, int(end_index - context_length))
window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:]
elif initial_chunk_size > 0 and length <= chunk_size:
already_sent = transfer_manager.put_req_chunk[request_id] * initial_chunk_size
pending = length - already_sent
if pending <= 0:
@@ -134,17 +240,23 @@ def slow_ar_to_dac_decoder_async_chunk(
else:
initial_coverage = (chunk_size // initial_chunk_size) * initial_chunk_size if initial_chunk_size > 0 else 0
adjusted = length - initial_coverage
chunk_length = adjusted % chunk_size
chunk_length = adjusted % steady_chunk_size
if chunk_length != 0 and not finished:
return None
context_length = chunk_length if chunk_length != 0 else chunk_size
context_length = chunk_length if chunk_length != 0 else steady_chunk_size
end_index = min(length, left_context_size_config + context_length)
left_context_size = max(0, int(end_index - context_length))
window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:]
# Pack into codebook-major flat codes.
# Pack into codebook-major codes. The tensor path avoids expanding codec
# indices into Python ints across the connector boundary; Stage1 schedules
# a single placeholder token and consumes the real codes from runtime info.
stacked_frames = torch.stack(window_frames, dim=0)
code_predictor_codes = stacked_frames.transpose(0, 1).reshape(-1)
codes_qf = stacked_frames.transpose(0, 1).contiguous()
if _use_tensor_code_payload(transfer_manager):
code_predictor_codes = codes_qf
else:
code_predictor_codes = codes_qf.reshape(-1)
return OmniPayloadStruct(
codes=CodesStruct(audio=code_predictor_codes),
+8
View File
@@ -509,6 +509,14 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
slot_mappings=slot_mappings_by_group,
)
self._maybe_attach_attention_metadata_extensions(
attn_metadata=attn_metadata,
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
max_query_len=max_num_scheduled_tokens,
pad_attn=pad_attn,
num_scheduled_tokens_np=num_scheduled_tokens_np,
)
(
input_ids,
@@ -278,6 +278,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
slot_mappings=slot_mappings_by_group,
)
self._maybe_attach_attention_metadata_extensions(
attn_metadata=attn_metadata,
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
max_query_len=max_num_scheduled_tokens,
pad_attn=pad_attn,
num_scheduled_tokens_np=num_scheduled_tokens_np,
)
(
input_ids,
@@ -730,6 +738,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
slot_mappings=slot_mappings_by_group,
use_spec_decode=self.speculative_config is not None,
)
self._maybe_attach_attention_metadata_extensions(
attn_metadata=attn_metadata,
num_reqs=num_reqs_padded,
num_reqs_padded=num_reqs_padded,
max_query_len=max_query_len,
pad_attn=True,
for_cudagraph_capture=is_graph_capturing,
)
with self.maybe_dummy_run_with_lora(
self.lora_config,
+48
View File
@@ -96,6 +96,7 @@ class OmniGPUModelRunner(GPUModelRunner):
@instrument(span_name="Loading (GPU)")
def load_model(self, *args, **kwargs) -> None:
super().load_model(*args, **kwargs)
self._prewarm_attention_capture_workspaces()
# TODO move this model specific logic to a separate class
# TTS model IS the talker (no .talker sub-attr); use getattr to support both Omni and TTS.
@@ -122,6 +123,45 @@ class OmniGPUModelRunner(GPUModelRunner):
self.last_talker_hidden = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False)
self.text_step = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False)
def _prewarm_attention_capture_workspaces(self) -> None:
capture_sizes = getattr(self.compilation_config, "cudagraph_capture_sizes", None)
if not capture_sizes:
return
from vllm_omni.attention.fish_kvcache_backend import prewarm_fish_kvcache_attn_capture_workspaces
prewarm_fish_kvcache_attn_capture_workspaces(
model_config=self.model_config,
device=self.device,
dtype=self.dtype,
capture_sizes=capture_sizes,
)
def _maybe_attach_attention_metadata_extensions(
self,
*,
attn_metadata: Any,
num_reqs: int,
num_reqs_padded: int,
max_query_len: int,
pad_attn: bool,
for_cudagraph_capture: bool = False,
num_scheduled_tokens_np: np.ndarray | None = None,
) -> None:
from vllm_omni.attention.fish_kvcache_backend import maybe_attach_fish_kvcache_seq_lens_upper_bound
maybe_attach_fish_kvcache_seq_lens_upper_bound(
model_config=self.model_config,
attn_metadata=attn_metadata,
input_batch=self.input_batch,
optimistic_seq_lens_cpu=self.optimistic_seq_lens_cpu,
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
max_query_len=max_query_len,
pad_attn=pad_attn,
for_cudagraph_capture=for_cudagraph_capture,
num_scheduled_tokens_np=num_scheduled_tokens_np,
)
def _build_model_sampler_output_token_ids(self) -> list[list[int]]:
"""Build decoded-token history for ``prefer_model_sampler`` models.
@@ -898,6 +938,14 @@ class OmniGPUModelRunner(GPUModelRunner):
slot_mappings=slot_mappings_by_group,
use_spec_decode=self.speculative_config is not None,
)
self._maybe_attach_attention_metadata_extensions(
attn_metadata=attn_metadata,
num_reqs=num_reqs_padded,
num_reqs_padded=num_reqs_padded,
max_query_len=max_query_len,
pad_attn=True,
for_cudagraph_capture=is_graph_capturing,
)
with self.maybe_dummy_run_with_lora(
self.lora_config,