[TTS][Perf] Optimize Fish Speech S2 Pro high-concurrency serving (#3773)
Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
1 parent
37eebff674
commit
d37912b071
23 files changed
+3453
-92
No files matched your search
@@ -143,6 +143,26 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh
|
||||
pip install fish-speech
|
||||
```
|
||||
|
||||
### Kvcache attention fast path
|
||||
|
||||
Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
|
||||
default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or
|
||||
`VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be
|
||||
installed.
|
||||
|
||||
```bash
|
||||
# Verify fast path availability.
|
||||
python - <<'PY'
|
||||
from vllm_omni.attention import fish_kvcache_attn
|
||||
|
||||
print(fish_kvcache_attn.is_available())
|
||||
print(fish_kvcache_attn.load_error())
|
||||
PY
|
||||
|
||||
# Optional: disable the runtime fast path.
|
||||
export VLLM_OMNI_FISH_KVCACHE_ATTN=0
|
||||
```
|
||||
|
||||
### Launch
|
||||
```bash
|
||||
vllm serve fishaudio/s2-pro --omni --port 8091
|
||||
|
||||
@@ -50,6 +50,26 @@ fi
|
||||
pip install fish-speech
|
||||
```
|
||||
|
||||
#### Kvcache attention fast path
|
||||
|
||||
Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
|
||||
default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or
|
||||
`VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be
|
||||
installed.
|
||||
|
||||
```bash
|
||||
# Verify fast path availability.
|
||||
python - <<'PY'
|
||||
from vllm_omni.attention import fish_kvcache_attn
|
||||
|
||||
print(fish_kvcache_attn.is_available())
|
||||
print(fish_kvcache_attn.load_error())
|
||||
PY
|
||||
|
||||
# Optional: disable the runtime fast path.
|
||||
export VLLM_OMNI_FISH_KVCACHE_ATTN=0
|
||||
```
|
||||
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -33,6 +33,20 @@ def test_tensor_serialization():
|
||||
assert torch.equal(tensor, deserialized)
|
||||
|
||||
|
||||
def test_noncontiguous_tensor_serialization():
|
||||
"""Test torch.Tensor serialization for strided views."""
|
||||
import torch
|
||||
|
||||
tensor = torch.arange(40, dtype=torch.long).reshape(4, 10).transpose(0, 1)
|
||||
assert not tensor.is_contiguous()
|
||||
assert tensor.stride(-1) != 1
|
||||
|
||||
serialized = OmniSerializer.serialize({"codes": tensor})
|
||||
deserialized = OmniSerializer.deserialize(serialized)
|
||||
|
||||
assert torch.equal(tensor, deserialized["codes"])
|
||||
|
||||
|
||||
def test_ndarray_serialization():
|
||||
"""Test numpy.ndarray serialization."""
|
||||
import numpy as np
|
||||
|
||||
@@ -37,6 +37,7 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
|
||||
status=status,
|
||||
prompt_token_ids=[],
|
||||
num_computed_tokens=0,
|
||||
num_output_placeholders=0,
|
||||
additional_information=None,
|
||||
is_finished=lambda: status == RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
@@ -44,9 +45,16 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
|
||||
|
||||
@pytest.fixture
|
||||
def build_adapter(monkeypatch, mocker: MockerFixture):
|
||||
def _build(*, stage_id: int = 1, model_mode: str = "ar", max_num_seqs: int = 2):
|
||||
def _build(
|
||||
*,
|
||||
stage_id: int = 1,
|
||||
model_mode: str = "ar",
|
||||
max_num_seqs: int = 2,
|
||||
connector_extra: dict | None = None,
|
||||
):
|
||||
connector = mocker.MagicMock()
|
||||
connector.stage_id = stage_id
|
||||
connector.config = {"extra": connector_extra or {}}
|
||||
connector.get.return_value = None
|
||||
connector.put.return_value = (True, 1, {})
|
||||
|
||||
@@ -127,6 +135,61 @@ def test_load_poll(build_adapter):
|
||||
assert "req-1" not in adapter._pending_load_reqs
|
||||
|
||||
|
||||
def test_load_poll_generation_tensor_codes_use_placeholder_prompt(build_adapter):
|
||||
adapter, connector = build_adapter(stage_id=1, model_mode="generation")
|
||||
request = _req("req-tensor", RequestStatus.WAITING, external_req_id="external-tensor")
|
||||
|
||||
codes = torch.tensor([[1, 2], [3, 4]], dtype=torch.long)
|
||||
payload: OmniPayload = {
|
||||
"codes": {"audio": codes},
|
||||
"meta": {
|
||||
"left_context_size": 1,
|
||||
"finished": torch.tensor(False, dtype=torch.bool),
|
||||
},
|
||||
}
|
||||
connector.get.return_value = (payload, 16)
|
||||
|
||||
adapter._poll_single_request(request)
|
||||
|
||||
assert request.prompt_token_ids == [0]
|
||||
assert request.num_computed_tokens == 0
|
||||
assert torch.equal(request.additional_information["codes"]["audio"], codes)
|
||||
assert request.additional_information["meta"]["left_context_size"] == 1
|
||||
assert "finished" not in request.additional_information["meta"]
|
||||
assert "req-tensor" in adapter._finished_load_reqs
|
||||
|
||||
|
||||
def test_load_poll_generation_empty_nonterminal_chunk_keeps_polling(build_adapter):
|
||||
adapter, connector = build_adapter(stage_id=1, model_mode="generation")
|
||||
request = _req("req-empty-tensor", RequestStatus.WAITING, external_req_id="external-empty")
|
||||
|
||||
empty_payload: OmniPayload = {
|
||||
"codes": {"audio": torch.empty((4, 0), dtype=torch.long)},
|
||||
"meta": {
|
||||
"left_context_size": 0,
|
||||
"finished": torch.tensor(False, dtype=torch.bool),
|
||||
},
|
||||
}
|
||||
ready_payload: OmniPayload = {
|
||||
"codes": {"audio": torch.tensor([[1, 2]], dtype=torch.long)},
|
||||
"meta": {
|
||||
"left_context_size": 0,
|
||||
"finished": torch.tensor(False, dtype=torch.bool),
|
||||
},
|
||||
}
|
||||
connector.get.side_effect = [(empty_payload, 16), (ready_payload, 16)]
|
||||
|
||||
assert adapter._poll_single_request(request) is False
|
||||
assert request.request_id not in adapter._finished_load_reqs
|
||||
assert request.request_id not in adapter.requests_with_ready_chunks
|
||||
assert adapter.get_req_chunk[request.request_id] == 1
|
||||
|
||||
assert adapter._poll_single_request(request) is True
|
||||
assert request.request_id in adapter._finished_load_reqs
|
||||
assert torch.equal(request.additional_information["codes"]["audio"], ready_payload["codes"]["audio"])
|
||||
assert adapter.get_req_chunk[request.request_id] == 2
|
||||
|
||||
|
||||
def test_save_async(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1)
|
||||
request = _req("req-1", RequestStatus.WAITING, external_req_id="external-1")
|
||||
@@ -140,6 +203,18 @@ def test_save_async(build_adapter):
|
||||
assert task["is_finished"] is False
|
||||
|
||||
|
||||
def test_save_async_uses_confirmed_tokens_for_async_scheduler_watermark(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1)
|
||||
request = _req("req-async", RequestStatus.WAITING, external_req_id="external-async")
|
||||
request.num_computed_tokens = 10
|
||||
request.num_output_placeholders = 2
|
||||
|
||||
adapter.save_async(pooling_output=None, request=request)
|
||||
|
||||
assert adapter.requests_num_chunks_sent["external-async"] == 8
|
||||
assert len(adapter._pending_save_reqs) == 1
|
||||
|
||||
|
||||
def test_send_single_request_struct_without_meta_does_not_crash(build_adapter, monkeypatch):
|
||||
"""Producer may return a struct with ``meta=None`` (e.g. payload that
|
||||
carries only ``embed`` or ``codes``). The sender's ``meta is not None``
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.fish_speech.configuration_fish_speech import (
|
||||
FishSpeechFastARConfig,
|
||||
FishSpeechSlowARConfig,
|
||||
)
|
||||
from vllm_omni.model_executor.models.fish_speech.fish_speech_fast_ar import FishSpeechFastAR
|
||||
from vllm_omni.model_executor.models.fish_speech.fish_speech_slow_ar import (
|
||||
FishSpeechSlowARForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
def test_fast_ar_reuses_dense_position_id_buffer(monkeypatch):
|
||||
fast_config = FishSpeechFastARConfig(
|
||||
vocab_size=16,
|
||||
num_codebooks=4,
|
||||
dim=8,
|
||||
n_head=2,
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
n_layer=1,
|
||||
intermediate_size=16,
|
||||
max_seq_len=5,
|
||||
)
|
||||
slow_config = FishSpeechSlowARConfig(
|
||||
vocab_size=32,
|
||||
dim=8,
|
||||
n_head=2,
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
n_layer=1,
|
||||
intermediate_size=16,
|
||||
codebook_size=16,
|
||||
num_codebooks=4,
|
||||
semantic_begin_id=4,
|
||||
semantic_end_id=19,
|
||||
)
|
||||
fast_ar = object.__new__(FishSpeechFastAR)
|
||||
nn.Module.__init__(fast_ar)
|
||||
fast_ar.config = fast_config
|
||||
fast_ar.slow_ar_config = slow_config
|
||||
fast_ar.fast_project_in = nn.Identity()
|
||||
fast_ar.fast_embeddings = nn.Embedding(fast_config.vocab_size, fast_config.hidden_size)
|
||||
fast_ar.fast_output = nn.Linear(fast_config.hidden_size, fast_config.vocab_size, bias=False)
|
||||
fast_ar.fast_norm = nn.Identity()
|
||||
fast_ar._num_codebooks = fast_config.num_codebooks
|
||||
fast_ar._fast_dim = fast_config.hidden_size
|
||||
fast_ar._embed_buf = None
|
||||
fast_ar._pos_ids = None
|
||||
fast_ar._k_cache = None
|
||||
fast_ar._v_cache = None
|
||||
fast_ar._compiled_model_fwd = None
|
||||
fast_ar._compile_attempted = True
|
||||
fast_ar._compile_failed = False
|
||||
fast_ar._disable_compile_for_graph = False
|
||||
|
||||
seen_position_ids: list[torch.Tensor] = []
|
||||
|
||||
def fake_run_model_one(step_input, step_pos_ids, cache_pos):
|
||||
seen_position_ids.append(step_pos_ids)
|
||||
return torch.zeros(
|
||||
step_input.shape[0],
|
||||
fast_config.hidden_size,
|
||||
dtype=step_input.dtype,
|
||||
device=step_input.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(fast_ar, "_run_model_one", fake_run_model_one)
|
||||
|
||||
hidden = torch.zeros(3, slow_config.hidden_size, dtype=torch.float32)
|
||||
semantic = torch.full((3,), slow_config.semantic_begin_id, dtype=torch.long)
|
||||
fast_ar(hidden, semantic, do_sample=False)
|
||||
|
||||
assert fast_ar._pos_ids is not None
|
||||
assert fast_ar._pos_ids.shape == (3, fast_config.num_codebooks + 1)
|
||||
assert len(seen_position_ids) == fast_config.num_codebooks
|
||||
for step, step_pos_ids in enumerate(seen_position_ids):
|
||||
assert step_pos_ids.shape == (3,)
|
||||
assert step_pos_ids.untyped_storage().data_ptr() == fast_ar._pos_ids.untyped_storage().data_ptr()
|
||||
assert step_pos_ids.stride() == (fast_ar._pos_ids.stride(0),)
|
||||
expected = torch.full((3,), step)
|
||||
assert torch.equal(step_pos_ids.cpu(), expected)
|
||||
|
||||
|
||||
def test_talker_mtp_does_not_mutate_input():
|
||||
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
|
||||
nn.Module.__init__(model)
|
||||
model._semantic_begin_id = 4
|
||||
model._semantic_end_id = 19
|
||||
model._codebook_size = 16
|
||||
model._num_codebooks = 4
|
||||
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
|
||||
model.fast_ar = lambda **_: torch.tensor(
|
||||
[
|
||||
[0, 1, 2, 3],
|
||||
[15, 14, 13, 12],
|
||||
],
|
||||
dtype=torch.long,
|
||||
)
|
||||
input_ids = torch.tensor([4, 2], dtype=torch.long)
|
||||
input_embeds = torch.randn(2, 8)
|
||||
input_embeds_before = input_embeds.clone()
|
||||
last_hidden = torch.randn(2, 8, dtype=torch.bfloat16)
|
||||
text_step = torch.zeros(2, 8, dtype=torch.bfloat16)
|
||||
|
||||
out1, _ = FishSpeechSlowARForConditionalGeneration.talker_mtp(
|
||||
model,
|
||||
input_ids,
|
||||
input_embeds,
|
||||
last_hidden,
|
||||
text_step,
|
||||
)
|
||||
|
||||
assert torch.equal(input_embeds, input_embeds_before)
|
||||
assert out1.untyped_storage().data_ptr() != input_embeds.untyped_storage().data_ptr()
|
||||
|
||||
|
||||
def test_talker_mtp_forwards_sampling_params():
|
||||
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
|
||||
nn.Module.__init__(model)
|
||||
model._semantic_begin_id = 4
|
||||
model._semantic_end_id = 19
|
||||
model._codebook_size = 16
|
||||
model._num_codebooks = 4
|
||||
model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8)
|
||||
seen = {}
|
||||
|
||||
def fake_fast_ar(**kwargs):
|
||||
seen.update(kwargs)
|
||||
return torch.tensor([[0, 1, 2, 3]], dtype=torch.long)
|
||||
|
||||
model.fast_ar = fake_fast_ar
|
||||
input_ids = torch.tensor([4], dtype=torch.long)
|
||||
input_embeds = torch.randn(1, 8)
|
||||
last_hidden = torch.randn(1, 8, dtype=torch.bfloat16)
|
||||
text_step = torch.zeros(1, 8, dtype=torch.bfloat16)
|
||||
|
||||
FishSpeechSlowARForConditionalGeneration.talker_mtp(
|
||||
model,
|
||||
input_ids,
|
||||
input_embeds,
|
||||
last_hidden,
|
||||
text_step,
|
||||
do_sample=False,
|
||||
temperature=0.1,
|
||||
top_k=5,
|
||||
top_p=0.7,
|
||||
)
|
||||
|
||||
assert seen["do_sample"] is False
|
||||
assert seen["temperature"] == 0.1
|
||||
assert seen["top_k"] == 5
|
||||
assert seen["top_p"] == 0.7
|
||||
@@ -1,5 +1,6 @@
|
||||
import functools
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -26,6 +27,39 @@ class _FakeCodec:
|
||||
return wav, audio_lengths
|
||||
|
||||
|
||||
class _RecordingCodec:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def decode(self, codes_bqf: torch.Tensor, feature_lengths: torch.Tensor):
|
||||
self.calls.append((tuple(codes_bqf.shape), tuple(int(x) for x in feature_lengths.tolist())))
|
||||
batch = codes_bqf.shape[0]
|
||||
wav_len = int(codes_bqf.shape[-1]) * 10
|
||||
wav = torch.arange(batch * wav_len, dtype=torch.float32).view(batch, 1, wav_len)
|
||||
audio_lengths = feature_lengths.to(torch.long) * 10
|
||||
return wav, audio_lengths
|
||||
|
||||
|
||||
class _FakeOneArgCodec:
|
||||
def decode(self, codes_bqf: torch.Tensor):
|
||||
batch = codes_bqf.shape[0]
|
||||
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
|
||||
|
||||
|
||||
class _FakeFromIndicesCodec:
|
||||
def __init__(self):
|
||||
self.from_indices_calls = 0
|
||||
|
||||
def from_indices(self, codes_bqf: torch.Tensor):
|
||||
self.from_indices_calls += 1
|
||||
batch = codes_bqf.shape[0]
|
||||
return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100)
|
||||
|
||||
def decode(self, codes_bqf: torch.Tensor):
|
||||
del codes_bqf
|
||||
raise AssertionError("decode() should not be used for index tensors when from_indices() exists")
|
||||
|
||||
|
||||
class _FakeTokenizer:
|
||||
def __init__(self, mapping, unk_token_id=-1):
|
||||
self._mapping = mapping
|
||||
@@ -40,6 +74,7 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = _FakeCodec()
|
||||
decoder._codec_decode_takes_lengths = True
|
||||
decoder._num_codebooks = 10
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
@@ -62,6 +97,132 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices():
|
||||
assert audios[1].shape[0] == 50
|
||||
|
||||
|
||||
def test_dac_decoder_uses_tensor_codes_from_runtime_info():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = _FakeCodec()
|
||||
decoder._codec_decode_takes_lengths = True
|
||||
decoder._num_codebooks = 2
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.tensor([0], dtype=torch.long),
|
||||
runtime_additional_information=[
|
||||
{
|
||||
"codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)},
|
||||
"meta": {"left_context_size": 1},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audios) == 1
|
||||
# 2 total frames with 1 frame of left context => proportional trim removes half the samples.
|
||||
assert audios[0].shape[0] == 50
|
||||
|
||||
|
||||
def test_dac_decoder_supports_codec_decode_without_lengths():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = _FakeOneArgCodec()
|
||||
decoder._codec_decode_takes_lengths = False
|
||||
decoder._num_codebooks = 10
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
|
||||
torch.arange(10, dtype=torch.long),
|
||||
torch.arange(20, dtype=torch.long),
|
||||
]
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.arange(30, dtype=torch.long),
|
||||
runtime_additional_information=[{}, {}],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert len(audios) == 2
|
||||
assert audios[0].shape[0] == 50
|
||||
assert audios[1].shape[0] == 100
|
||||
|
||||
|
||||
def test_dac_decoder_prefers_from_indices_for_one_arg_codec():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
codec = _FakeFromIndicesCodec()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = codec
|
||||
decoder._codec_decode_takes_lengths = False
|
||||
decoder._num_codebooks = 10
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
|
||||
torch.arange(10, dtype=torch.long),
|
||||
torch.arange(20, dtype=torch.long),
|
||||
]
|
||||
|
||||
out = decoder.forward(
|
||||
input_ids=torch.arange(30, dtype=torch.long),
|
||||
runtime_additional_information=[{}, {}],
|
||||
)
|
||||
|
||||
audios = out.multimodal_outputs["model_outputs"]
|
||||
assert codec.from_indices_calls == 1
|
||||
assert len(audios) == 2
|
||||
assert audios[0].shape[0] == 50
|
||||
assert audios[1].shape[0] == 100
|
||||
|
||||
|
||||
def test_dac_decoder_splits_groups_by_padded_frame_budget():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
codec = _RecordingCodec()
|
||||
decoder = object.__new__(FishSpeechDACDecoder)
|
||||
torch.nn.Module.__init__(decoder)
|
||||
decoder._codec = codec
|
||||
decoder._codec_decode_takes_lengths = True
|
||||
decoder._num_codebooks = 2
|
||||
decoder._output_sample_rate = 44100
|
||||
decoder._hop_length = 512
|
||||
decoder._logged_codec_stats = False
|
||||
decoder._decode_batch_max_padded_frames = 100
|
||||
decoder._decode_batch_max_batch = 0
|
||||
decoder._ensure_codec_loaded = lambda: None
|
||||
decoder._split_request_ids = lambda ids, seq_token_counts=None: [
|
||||
torch.arange(2 * 46, dtype=torch.long),
|
||||
torch.arange(2 * 45, dtype=torch.long),
|
||||
torch.arange(2 * 47, dtype=torch.long),
|
||||
]
|
||||
|
||||
decoder.forward(input_ids=torch.arange(2 * (46 + 45 + 47), dtype=torch.long))
|
||||
|
||||
assert codec.calls == [
|
||||
((2, 2, 46), (45, 46)),
|
||||
((1, 2, 47), (47,)),
|
||||
]
|
||||
|
||||
|
||||
def test_dac_decoder_dtype_can_be_configured():
|
||||
_, FishSpeechDACDecoder, _ = _fish_speech_regression_modules()
|
||||
cfg = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
model="unused",
|
||||
stage_connector_config={"extra": {"fish_speech_dac_dtype": "fp16"}},
|
||||
)
|
||||
)
|
||||
|
||||
decoder = FishSpeechDACDecoder(vllm_config=cfg)
|
||||
|
||||
assert decoder._dac_dtype == torch.float16
|
||||
|
||||
|
||||
def test_structured_voice_clone_prefill_adds_full_codebooks_with_decode_scale(monkeypatch):
|
||||
slow_ar_module, _, FishSpeechSlowARForConditionalGeneration = _fish_speech_regression_modules()
|
||||
model = object.__new__(FishSpeechSlowARForConditionalGeneration)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.stage_input_processors.fish_speech import (
|
||||
slow_ar_to_dac_decoder_async_chunk,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_transfer_manager(
|
||||
*,
|
||||
tensor_payload: bool = False,
|
||||
chunk_frames: int = 2,
|
||||
left_context: int = 1,
|
||||
initial_chunk_frames: int = 0,
|
||||
single_initial_chunk: bool = False,
|
||||
backlog_chunk_frames: int = 0,
|
||||
backlog_load_threshold: float = 0.75,
|
||||
max_num_seqs: int = 1,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
code_prompt_token_ids=defaultdict(list),
|
||||
put_req_chunk=defaultdict(int),
|
||||
scheduler_max_num_seqs=max_num_seqs,
|
||||
connector=SimpleNamespace(
|
||||
config={
|
||||
"extra": {
|
||||
"codec_chunk_frames": chunk_frames,
|
||||
"codec_left_context_frames": left_context,
|
||||
"initial_codec_chunk_frames": initial_chunk_frames,
|
||||
"fish_speech_tensor_codes": tensor_payload,
|
||||
"fish_speech_single_initial_chunk": single_initial_chunk,
|
||||
"fish_speech_backlog_codec_chunk_frames": backlog_chunk_frames,
|
||||
"fish_speech_backlog_load_threshold": backlog_load_threshold,
|
||||
}
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_request(req_id="req", *, finished=False):
|
||||
return SimpleNamespace(
|
||||
external_req_id=req_id,
|
||||
additional_information=None,
|
||||
is_finished=lambda: finished,
|
||||
)
|
||||
|
||||
|
||||
def _seed_frames(transfer_manager, req_id: str, n_frames: int, n_codebooks: int = 3):
|
||||
frame = torch.arange(1, n_codebooks + 1, dtype=torch.long)
|
||||
transfer_manager.code_prompt_token_ids[req_id] = [frame + i * n_codebooks for i in range(n_frames)]
|
||||
|
||||
|
||||
def _call_seeded(transfer_manager, req_id: str = "req", *, n_frames: int, finished: bool = False):
|
||||
_seed_frames(transfer_manager, req_id, n_frames)
|
||||
return slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{},
|
||||
_make_request(req_id, finished=finished),
|
||||
is_finished=finished,
|
||||
)
|
||||
|
||||
|
||||
def test_slow_ar_to_dac_async_chunk_can_emit_tensor_codes():
|
||||
transfer_manager = _make_transfer_manager(tensor_payload=True)
|
||||
request = _make_request()
|
||||
|
||||
assert (
|
||||
slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{"audio_codes": torch.tensor([[1, 2, 3]])},
|
||||
request,
|
||||
)
|
||||
is None
|
||||
)
|
||||
payload = slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{"audio_codes": torch.tensor([[4, 5, 6]])},
|
||||
request,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
audio_codes = payload.codes.audio
|
||||
assert isinstance(audio_codes, torch.Tensor)
|
||||
assert audio_codes.dtype == torch.long
|
||||
assert torch.equal(audio_codes, torch.tensor([[1, 4], [2, 5], [3, 6]], dtype=torch.long))
|
||||
assert payload.meta.left_context_size == 0
|
||||
|
||||
|
||||
def test_slow_ar_to_dac_async_chunk_uses_valid_mask_for_zero_codes():
|
||||
transfer_manager = _make_transfer_manager(tensor_payload=True, chunk_frames=1, left_context=0)
|
||||
request = _make_request()
|
||||
|
||||
assert (
|
||||
slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{
|
||||
"audio_codes": torch.tensor([[0, 0, 0]]),
|
||||
"audio_code_valid": torch.tensor([False]),
|
||||
},
|
||||
request,
|
||||
)
|
||||
is None
|
||||
)
|
||||
payload = slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{
|
||||
"audio_codes": torch.tensor([[4, 5, 6]]),
|
||||
"audio_code_valid": torch.tensor([True]),
|
||||
},
|
||||
request,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert torch.equal(payload.codes.audio, torch.tensor([[4], [5], [6]], dtype=torch.long))
|
||||
|
||||
|
||||
def test_slow_ar_to_dac_async_chunk_keeps_flat_codes_by_default():
|
||||
transfer_manager = _make_transfer_manager(tensor_payload=False)
|
||||
request = _make_request()
|
||||
|
||||
slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{"audio_codes": torch.tensor([[1, 2]])},
|
||||
request,
|
||||
)
|
||||
payload = slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{"audio_codes": torch.tensor([[3, 4]])},
|
||||
request,
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert torch.equal(payload.codes.audio, torch.tensor([1, 3, 2, 4], dtype=torch.long))
|
||||
|
||||
|
||||
def test_qwen3_style_single_initial_chunk_only_emits_first_chunk_early():
|
||||
transfer_manager = _make_transfer_manager(
|
||||
chunk_frames=5,
|
||||
left_context=5,
|
||||
initial_chunk_frames=2,
|
||||
single_initial_chunk=True,
|
||||
)
|
||||
|
||||
first = _call_seeded(transfer_manager, n_frames=2)
|
||||
assert first is not None
|
||||
assert first.meta.left_context_size == 0
|
||||
assert first.codes.audio.numel() == 3 * 2
|
||||
|
||||
# After the first small chunk, return to normal chunk boundaries:
|
||||
# initial_coverage=2, so the next emit is at 2 + 5 = 7 frames.
|
||||
assert _call_seeded(transfer_manager, n_frames=5) is None
|
||||
|
||||
second = _call_seeded(transfer_manager, n_frames=7)
|
||||
assert second is not None
|
||||
assert second.meta.left_context_size == 2
|
||||
assert second.codes.audio.numel() == 3 * 7
|
||||
|
||||
|
||||
def test_backlog_chunk_size_uses_larger_post_initial_boundary():
|
||||
transfer_manager = _make_transfer_manager(
|
||||
chunk_frames=5,
|
||||
left_context=5,
|
||||
initial_chunk_frames=2,
|
||||
single_initial_chunk=True,
|
||||
backlog_chunk_frames=8,
|
||||
backlog_load_threshold=0.5,
|
||||
max_num_seqs=4,
|
||||
)
|
||||
for idx in range(2):
|
||||
_seed_frames(transfer_manager, f"other-{idx}", 10)
|
||||
|
||||
first = _call_seeded(transfer_manager, "req", n_frames=2)
|
||||
assert first is not None
|
||||
|
||||
# Under backlog, steady chunk is 8 instead of 5. The next emit is
|
||||
# initial_coverage=2 plus 8 frames, so n=7 must still be held.
|
||||
assert _call_seeded(transfer_manager, "req", n_frames=7) is None
|
||||
second = _call_seeded(transfer_manager, "req", n_frames=10)
|
||||
assert second is not None
|
||||
assert second.meta.left_context_size == 2
|
||||
assert second.codes.audio.numel() == 3 * 10
|
||||
|
||||
|
||||
def test_backlog_chunk_size_keeps_multi_initial_boundary_aligned():
|
||||
transfer_manager = _make_transfer_manager(
|
||||
chunk_frames=25,
|
||||
left_context=25,
|
||||
initial_chunk_frames=10,
|
||||
single_initial_chunk=False,
|
||||
backlog_chunk_frames=50,
|
||||
backlog_load_threshold=0.5,
|
||||
max_num_seqs=4,
|
||||
)
|
||||
for idx in range(2):
|
||||
_seed_frames(transfer_manager, f"other-{idx}", 50)
|
||||
|
||||
first = _call_seeded(transfer_manager, "req", n_frames=10)
|
||||
assert first is not None
|
||||
transfer_manager.put_req_chunk["req"] += 1
|
||||
second = _call_seeded(transfer_manager, "req", n_frames=20)
|
||||
assert second is not None
|
||||
transfer_manager.put_req_chunk["req"] += 1
|
||||
|
||||
assert _call_seeded(transfer_manager, "req", n_frames=50) is None
|
||||
third = _call_seeded(transfer_manager, "req", n_frames=70)
|
||||
assert third is not None
|
||||
assert third.meta.left_context_size == 20
|
||||
assert third.codes.audio.numel() == 3 * 70
|
||||
|
||||
|
||||
def test_invalid_backlog_config_fails_loudly():
|
||||
transfer_manager = _make_transfer_manager(
|
||||
chunk_frames=5,
|
||||
left_context=5,
|
||||
backlog_chunk_frames="invalid",
|
||||
)
|
||||
request = _make_request()
|
||||
|
||||
with pytest.raises(ValueError, match="fish_speech_backlog_codec_chunk_frames"):
|
||||
slow_ar_to_dac_decoder_async_chunk(
|
||||
transfer_manager,
|
||||
{"audio_codes": torch.tensor([[1, 2, 3]])},
|
||||
request,
|
||||
)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,191 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
FISH_KVCACHE_LONG_SPLIT_TOKENS = 1024
|
||||
FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN = 1024
|
||||
|
||||
_FISH_KVCACHE_ATTN_ENV = "VLLM_OMNI_FISH_KVCACHE_ATTN"
|
||||
_ENABLED_VALUES = frozenset({"1", "true", "yes", "on", "required"})
|
||||
_DISABLED_VALUES = frozenset({"0", "false", "no", "off", "disabled", "disable"})
|
||||
_REQUIRED_VALUES = frozenset({"required"})
|
||||
_WORKSPACE_CACHE: dict[tuple[Any, ...], tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {}
|
||||
_WORKSPACE_CACHE_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _triton_backend() -> Any:
|
||||
from vllm_omni.attention import fish_kvcache_triton
|
||||
|
||||
return fish_kvcache_triton
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
return _triton_backend().is_available()
|
||||
|
||||
|
||||
def load_error() -> Exception | None:
|
||||
return _triton_backend().load_error()
|
||||
|
||||
|
||||
def is_fish_kvcache_attn_enabled() -> bool:
|
||||
value = os.environ.get(_FISH_KVCACHE_ATTN_ENV)
|
||||
if value is None:
|
||||
return True
|
||||
value = value.lower()
|
||||
if value in _DISABLED_VALUES:
|
||||
return False
|
||||
return value in _ENABLED_VALUES or value == ""
|
||||
|
||||
|
||||
def is_fish_kvcache_attn_required() -> bool:
|
||||
return os.environ.get(_FISH_KVCACHE_ATTN_ENV, "").lower() in _REQUIRED_VALUES
|
||||
|
||||
|
||||
def _is_sliding_window_disabled(sliding_window: Any) -> bool:
|
||||
if sliding_window is None:
|
||||
return True
|
||||
if isinstance(sliding_window, (list, tuple)) and len(sliding_window) == 2:
|
||||
return int(sliding_window[0]) == -1 and int(sliding_window[1]) == -1
|
||||
return False
|
||||
|
||||
|
||||
def can_use_fish_kvcache_attn(
|
||||
*,
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
block_table: torch.Tensor | None,
|
||||
seq_lens: torch.Tensor,
|
||||
max_query_len: int,
|
||||
max_seq_len: int,
|
||||
dcp_world_size: int,
|
||||
use_cascade: bool,
|
||||
alibi_slopes: Any,
|
||||
sliding_window: Any,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> bool:
|
||||
if not is_fish_kvcache_attn_enabled():
|
||||
return False
|
||||
if not is_available():
|
||||
return False
|
||||
if max_query_len != 1 or use_cascade or dcp_world_size != 1:
|
||||
return False
|
||||
if block_table is None or alibi_slopes is not None or not _is_sliding_window_disabled(sliding_window):
|
||||
return False
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
return False
|
||||
if query.dim() != 3 or key_cache.dim() != 4 or value_cache.dim() != 4:
|
||||
return False
|
||||
if block_table.dim() != 2 or seq_lens.dim() != 1:
|
||||
return False
|
||||
if block_table.shape[0] != query.shape[0] or seq_lens.shape[0] != query.shape[0]:
|
||||
return False
|
||||
if query.shape[-1] != 128 or key_cache.shape[-1] != 128:
|
||||
return False
|
||||
if key_cache.shape[1] != 16:
|
||||
return False
|
||||
if query.dtype not in (torch.float16, torch.bfloat16):
|
||||
return False
|
||||
if key_cache.dtype != query.dtype or value_cache.dtype != query.dtype:
|
||||
return False
|
||||
if block_table.dtype != torch.int32 or seq_lens.dtype != torch.int32:
|
||||
return False
|
||||
if max_seq_len <= 0:
|
||||
return False
|
||||
if max_seq_len > block_table.shape[1] * key_cache.shape[1]:
|
||||
return False
|
||||
if not (
|
||||
query.is_contiguous()
|
||||
and key_cache.is_contiguous()
|
||||
and value_cache.is_contiguous()
|
||||
and block_table.is_contiguous()
|
||||
and seq_lens.is_contiguous()
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_cuda_graph_capturing() -> bool:
|
||||
return bool(torch.cuda.is_current_stream_capturing())
|
||||
|
||||
|
||||
def _raise_workspace_capture_miss() -> None:
|
||||
raise RuntimeError("Fish kvcache attention workspace was not prewarmed before CUDA graph capture")
|
||||
|
||||
|
||||
def _get_decode_workspace(
|
||||
query: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
batch_size, num_q_heads, head_dim = query.shape
|
||||
if int(max_seq_len) <= FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN:
|
||||
key = (query.device.type, query.device.index, "empty")
|
||||
partial_shapes = ((0,), (0,), (0,))
|
||||
else:
|
||||
num_splits = (int(max_seq_len) + FISH_KVCACHE_LONG_SPLIT_TOKENS - 1) // FISH_KVCACHE_LONG_SPLIT_TOKENS
|
||||
key = (
|
||||
query.device.type,
|
||||
query.device.index,
|
||||
int(batch_size),
|
||||
int(num_q_heads),
|
||||
int(head_dim),
|
||||
int(num_splits),
|
||||
)
|
||||
partial_shapes = (
|
||||
(num_splits, batch_size, num_q_heads),
|
||||
(num_splits, batch_size, num_q_heads),
|
||||
(num_splits, batch_size, num_q_heads, head_dim),
|
||||
)
|
||||
with _WORKSPACE_CACHE_LOCK:
|
||||
workspace = _WORKSPACE_CACHE.get(key)
|
||||
if workspace is None:
|
||||
if _is_cuda_graph_capturing():
|
||||
_raise_workspace_capture_miss()
|
||||
workspace = (
|
||||
torch.empty(partial_shapes[0], device=query.device, dtype=torch.float32),
|
||||
torch.empty(partial_shapes[1], device=query.device, dtype=torch.float32),
|
||||
torch.empty(partial_shapes[2], device=query.device, dtype=torch.float32),
|
||||
)
|
||||
_WORKSPACE_CACHE[key] = workspace
|
||||
return workspace
|
||||
|
||||
|
||||
def prewarm_fish_kvcache_attn_workspace(
|
||||
query: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
_get_decode_workspace(query, int(max_seq_len))
|
||||
|
||||
|
||||
def fish_decode_kvcache_attn(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
*,
|
||||
scale: float,
|
||||
max_seq_len: int,
|
||||
) -> torch.Tensor:
|
||||
partial_m, partial_l, partial_acc = _get_decode_workspace(query, int(max_seq_len))
|
||||
return _triton_backend().fish_decode_kvcache_attn_triton(
|
||||
query,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
out,
|
||||
scale=float(scale),
|
||||
max_seq_len=int(max_seq_len),
|
||||
small_path_max_seq_len=FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN,
|
||||
long_split_tokens=FISH_KVCACHE_LONG_SPLIT_TOKENS,
|
||||
partial_m=partial_m,
|
||||
partial_l=partial_l,
|
||||
partial_acc=partial_acc,
|
||||
)
|
||||
@@ -0,0 +1,406 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.attention.fish_kvcache_attn import (
|
||||
FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN,
|
||||
can_use_fish_kvcache_attn,
|
||||
fish_decode_kvcache_attn,
|
||||
is_available,
|
||||
is_fish_kvcache_attn_enabled,
|
||||
is_fish_kvcache_attn_required,
|
||||
load_error,
|
||||
prewarm_fish_kvcache_attn_workspace,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_FISH_SLOW_AR_ARCH = "FishSpeechSlowARForConditionalGeneration"
|
||||
_FIRST_SMALL_HIT_LOGGED = False
|
||||
_FIRST_LONG_HIT_LOGGED = False
|
||||
_FIRST_MISS_LOGGED = False
|
||||
_SMALL_HIT_COUNT = 0
|
||||
_LONG_HIT_COUNT = 0
|
||||
_FALLBACK_COUNTS: Counter[str] = Counter()
|
||||
|
||||
|
||||
def _fish_kvcache_enabled() -> bool:
|
||||
return is_fish_kvcache_attn_enabled()
|
||||
|
||||
|
||||
def is_fish_kvcache_attn_active_for_model(model_config: Any) -> bool:
|
||||
return getattr(model_config, "model_arch", None) == _FISH_SLOW_AR_ARCH and is_fish_kvcache_attn_enabled()
|
||||
|
||||
|
||||
def reset_fish_kvcache_attn_stats() -> None:
|
||||
global _LONG_HIT_COUNT, _SMALL_HIT_COUNT
|
||||
_SMALL_HIT_COUNT = 0
|
||||
_LONG_HIT_COUNT = 0
|
||||
_FALLBACK_COUNTS.clear()
|
||||
|
||||
|
||||
def get_fish_kvcache_attn_stats() -> dict[str, Any]:
|
||||
return {
|
||||
"small_hit_count": _SMALL_HIT_COUNT,
|
||||
"long_hit_count": _LONG_HIT_COUNT,
|
||||
"fallback_count_by_reason": dict(_FALLBACK_COUNTS),
|
||||
}
|
||||
|
||||
|
||||
def _record_fallback(reason: str) -> None:
|
||||
_FALLBACK_COUNTS[reason] += 1
|
||||
|
||||
|
||||
def _small_decode_bucket(max_seq_len: int) -> int:
|
||||
if max_seq_len <= 64:
|
||||
return 64
|
||||
if max_seq_len <= 128:
|
||||
return 128
|
||||
if max_seq_len <= 256:
|
||||
return 256
|
||||
if max_seq_len <= 512:
|
||||
return 512
|
||||
return FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN
|
||||
|
||||
|
||||
def prewarm_fish_kvcache_attn_capture_workspaces(
|
||||
*,
|
||||
model_config: Any,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
capture_sizes: list[int] | tuple[int, ...],
|
||||
) -> int:
|
||||
"""Preallocate Fish attention workspaces used during CUDA graph capture."""
|
||||
if not _fish_kvcache_enabled() or not is_available():
|
||||
return 0
|
||||
if not is_fish_kvcache_attn_active_for_model(model_config):
|
||||
return 0
|
||||
|
||||
text_config = model_config.hf_config.text_config
|
||||
num_heads = int(text_config.num_attention_heads)
|
||||
head_dim = int(text_config.head_dim)
|
||||
max_seq_len = int(model_config.max_model_len)
|
||||
sizes = sorted({int(size) for size in capture_sizes if int(size) > 0})
|
||||
if not sizes:
|
||||
return 0
|
||||
|
||||
# The small path uses an empty workspace keyed only by device. Prewarm it
|
||||
# once, then prewarm every CUDA graph capture batch size for the long path.
|
||||
small_query = torch.empty((1, num_heads, head_dim), device=device, dtype=dtype)
|
||||
prewarm_fish_kvcache_attn_workspace(small_query, 1)
|
||||
prewarmed = 1
|
||||
|
||||
for batch_size in sizes:
|
||||
query = torch.empty((batch_size, num_heads, head_dim), device=device, dtype=dtype)
|
||||
prewarm_fish_kvcache_attn_workspace(query, max_seq_len)
|
||||
prewarmed += 1
|
||||
|
||||
logger.info(
|
||||
"Prewarmed Fish kvcache attention workspaces for capture sizes %s (num_heads=%d head_dim=%d max_seq_len=%d)",
|
||||
sizes,
|
||||
num_heads,
|
||||
head_dim,
|
||||
max_seq_len,
|
||||
)
|
||||
return prewarmed
|
||||
|
||||
|
||||
def _attach_seq_lens_cpu_upper_bound(attn_metadata: Any, seq_lens_cpu_upper_bound: torch.Tensor) -> None:
|
||||
if attn_metadata is None:
|
||||
return
|
||||
if isinstance(attn_metadata, dict):
|
||||
for value in attn_metadata.values():
|
||||
_attach_seq_lens_cpu_upper_bound(value, seq_lens_cpu_upper_bound)
|
||||
return
|
||||
if isinstance(attn_metadata, (list, tuple)):
|
||||
for value in attn_metadata:
|
||||
_attach_seq_lens_cpu_upper_bound(value, seq_lens_cpu_upper_bound)
|
||||
return
|
||||
setattr(attn_metadata, "seq_lens_cpu_upper_bound", seq_lens_cpu_upper_bound)
|
||||
|
||||
|
||||
def _graph_capture_seq_len_upper_bound(
|
||||
*,
|
||||
model_config: Any,
|
||||
max_query_len: int,
|
||||
for_cudagraph_capture: bool,
|
||||
) -> int | None:
|
||||
if not for_cudagraph_capture or int(max_query_len) != 1:
|
||||
return None
|
||||
max_model_len = int(getattr(model_config, "max_model_len", 0))
|
||||
return max(max_model_len, FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN)
|
||||
|
||||
|
||||
def maybe_attach_fish_kvcache_seq_lens_upper_bound(
|
||||
*,
|
||||
model_config: Any,
|
||||
attn_metadata: Any,
|
||||
input_batch: Any,
|
||||
optimistic_seq_lens_cpu: torch.Tensor,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
max_query_len: int,
|
||||
pad_attn: bool,
|
||||
for_cudagraph_capture: bool = False,
|
||||
num_scheduled_tokens_np: Any = None,
|
||||
) -> None:
|
||||
"""Attach CPU seq-len upper bounds for the Fish-only Triton decode fastpath."""
|
||||
if not is_fish_kvcache_attn_active_for_model(model_config) or int(max_query_len) != 1:
|
||||
return
|
||||
|
||||
upper_bound_rows = int(num_reqs_padded if pad_attn else num_reqs)
|
||||
capture_upper_bound = _graph_capture_seq_len_upper_bound(
|
||||
model_config=model_config,
|
||||
max_query_len=max_query_len,
|
||||
for_cudagraph_capture=for_cudagraph_capture,
|
||||
)
|
||||
if capture_upper_bound is not None:
|
||||
optimistic_seq_lens_cpu[:upper_bound_rows].fill_(capture_upper_bound)
|
||||
else:
|
||||
if num_scheduled_tokens_np is None:
|
||||
return
|
||||
seq_lens_cpu_upper_bound = torch.as_tensor(
|
||||
input_batch.num_computed_tokens_cpu[:num_reqs] + num_scheduled_tokens_np,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
)
|
||||
optimistic_seq_lens_cpu[:num_reqs] = seq_lens_cpu_upper_bound
|
||||
if upper_bound_rows > num_reqs:
|
||||
optimistic_seq_lens_cpu[num_reqs:upper_bound_rows].fill_(1)
|
||||
_attach_seq_lens_cpu_upper_bound(
|
||||
attn_metadata,
|
||||
optimistic_seq_lens_cpu[:upper_bound_rows],
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_max_seq_len(attn_metadata: Any, active_batch: int, seq_lens: Any) -> int | None:
|
||||
max_seq_len = int(attn_metadata.max_seq_len)
|
||||
if int(attn_metadata.max_query_len) != 1:
|
||||
return max_seq_len
|
||||
|
||||
upper_bound = getattr(attn_metadata, "seq_lens_cpu_upper_bound", None)
|
||||
if upper_bound is None:
|
||||
if is_fish_kvcache_attn_required():
|
||||
raise RuntimeError("Fish kvcache attention requires seq_lens_cpu_upper_bound for decode dispatch")
|
||||
return None
|
||||
if not isinstance(upper_bound, torch.Tensor) or upper_bound.device.type != "cpu":
|
||||
if is_fish_kvcache_attn_required():
|
||||
raise RuntimeError("Fish kvcache attention seq_lens_cpu_upper_bound must be a CPU tensor")
|
||||
return None
|
||||
if int(upper_bound.shape[0]) < active_batch:
|
||||
if is_fish_kvcache_attn_required():
|
||||
raise RuntimeError(
|
||||
"Fish kvcache attention seq_lens_cpu_upper_bound does not cover the active decode batch: "
|
||||
f"upper_bound_rows={int(upper_bound.shape[0])}, active_batch={active_batch}"
|
||||
)
|
||||
return None
|
||||
upper_bound = upper_bound[:active_batch]
|
||||
real_upper_bound = max(1, int(upper_bound.max().item()))
|
||||
if is_fish_kvcache_attn_required():
|
||||
real_max_seq_len = int(seq_lens.max().item())
|
||||
if real_upper_bound < real_max_seq_len:
|
||||
raise RuntimeError(
|
||||
"Fish kvcache attention seq_lens_cpu_upper_bound "
|
||||
f"underestimates real seq_lens: upper_bound={real_upper_bound}, "
|
||||
f"real={real_max_seq_len}"
|
||||
)
|
||||
if real_upper_bound <= FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN:
|
||||
return _small_decode_bucket(real_upper_bound)
|
||||
return real_upper_bound
|
||||
|
||||
|
||||
def _forward_with_fish_kvcache(
|
||||
impl: Any,
|
||||
original_forward: Callable[..., Any],
|
||||
layer: Any,
|
||||
query: Any,
|
||||
key: Any,
|
||||
value: Any,
|
||||
kv_cache: Any,
|
||||
attn_metadata: Any,
|
||||
output: Any,
|
||||
output_scale: Any = None,
|
||||
output_block_scale: Any = None,
|
||||
) -> Any:
|
||||
global _FIRST_LONG_HIT_LOGGED, _FIRST_MISS_LOGGED, _FIRST_SMALL_HIT_LOGGED
|
||||
global _LONG_HIT_COUNT, _SMALL_HIT_COUNT
|
||||
|
||||
if attn_metadata is not None and not attn_metadata.use_cascade:
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
key_cache, value_cache = kv_cache.unbind(0)
|
||||
q = query[:num_actual_tokens]
|
||||
out = output[:num_actual_tokens]
|
||||
active_batch = int(q.shape[0])
|
||||
block_table = attn_metadata.block_table[:active_batch] if attn_metadata.block_table is not None else None
|
||||
seq_lens = attn_metadata.seq_lens[:active_batch]
|
||||
dispatch_max_seq_len = _dispatch_max_seq_len(attn_metadata, active_batch, seq_lens)
|
||||
can_use = dispatch_max_seq_len is not None and can_use_fish_kvcache_attn(
|
||||
query=q,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
max_query_len=attn_metadata.max_query_len,
|
||||
max_seq_len=dispatch_max_seq_len,
|
||||
dcp_world_size=impl.dcp_world_size,
|
||||
use_cascade=attn_metadata.use_cascade,
|
||||
alibi_slopes=impl.alibi_slopes,
|
||||
sliding_window=impl.sliding_window,
|
||||
output_scale=output_scale,
|
||||
output_block_scale=output_block_scale,
|
||||
)
|
||||
if can_use:
|
||||
is_long_path = dispatch_max_seq_len > FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN
|
||||
if is_long_path:
|
||||
_LONG_HIT_COUNT += 1
|
||||
else:
|
||||
_SMALL_HIT_COUNT += 1
|
||||
should_log_hit = (is_long_path and not _FIRST_LONG_HIT_LOGGED) or (
|
||||
not is_long_path and not _FIRST_SMALL_HIT_LOGGED
|
||||
)
|
||||
if should_log_hit:
|
||||
if is_long_path:
|
||||
_FIRST_LONG_HIT_LOGGED = True
|
||||
path_name = "long"
|
||||
else:
|
||||
_FIRST_SMALL_HIT_LOGGED = True
|
||||
path_name = "small"
|
||||
logger.info(
|
||||
"Fish decode-only kvcache attention fast path hit: "
|
||||
"query_shape=%s key_cache_shape=%s block_table_shape=%s "
|
||||
"seq_lens_shape=%s max_seq_len=%s path=%s",
|
||||
tuple(q.shape),
|
||||
tuple(key_cache.shape),
|
||||
tuple(attn_metadata.block_table.shape),
|
||||
tuple(attn_metadata.seq_lens.shape),
|
||||
dispatch_max_seq_len,
|
||||
path_name,
|
||||
)
|
||||
fish_decode_kvcache_attn(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
out,
|
||||
scale=float(impl.scale),
|
||||
max_seq_len=dispatch_max_seq_len,
|
||||
)
|
||||
return output
|
||||
|
||||
is_decode_request = int(attn_metadata.max_query_len) == 1
|
||||
if is_decode_request and dispatch_max_seq_len is None:
|
||||
_record_fallback("missing_cpu_upper_bound")
|
||||
elif is_decode_request:
|
||||
_record_fallback("guard_miss")
|
||||
else:
|
||||
_record_fallback("non_decode")
|
||||
if is_decode_request and is_fish_kvcache_attn_required():
|
||||
raise RuntimeError(
|
||||
"Fish kvcache attention is required but guard rejected the request: "
|
||||
f"query_shape={tuple(q.shape)} query_dtype={q.dtype} "
|
||||
f"max_query_len={attn_metadata.max_query_len} max_seq_len={dispatch_max_seq_len} "
|
||||
f"dcp_world_size={impl.dcp_world_size} use_cascade={attn_metadata.use_cascade}"
|
||||
)
|
||||
|
||||
if not _FIRST_MISS_LOGGED:
|
||||
_FIRST_MISS_LOGGED = True
|
||||
logger.info(
|
||||
"Fish decode-only kvcache attention fast path miss: "
|
||||
"query_shape=%s query_dtype=%s key_cache_shape=%s key_dtype=%s "
|
||||
"block_table_shape=%s block_dtype=%s seq_lens_shape=%s seq_dtype=%s "
|
||||
"max_query_len=%s max_seq_len=%s dcp_world_size=%s use_cascade=%s",
|
||||
tuple(q.shape),
|
||||
q.dtype,
|
||||
tuple(key_cache.shape),
|
||||
key_cache.dtype,
|
||||
tuple(block_table.shape) if block_table is not None else None,
|
||||
getattr(block_table, "dtype", None),
|
||||
tuple(seq_lens.shape),
|
||||
seq_lens.dtype,
|
||||
attn_metadata.max_query_len,
|
||||
dispatch_max_seq_len,
|
||||
impl.dcp_world_size,
|
||||
attn_metadata.use_cascade,
|
||||
)
|
||||
|
||||
return original_forward(
|
||||
layer,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
output_scale=output_scale,
|
||||
output_block_scale=output_block_scale,
|
||||
)
|
||||
|
||||
|
||||
def install_fish_kvcache_attn_backend(model: Any) -> int:
|
||||
"""Install the Fish kvcache fast path on this Fish SlowAR model only."""
|
||||
if not _fish_kvcache_enabled():
|
||||
return 0
|
||||
if not is_available():
|
||||
_record_fallback("implementation_unavailable")
|
||||
if is_fish_kvcache_attn_required():
|
||||
raise RuntimeError(
|
||||
f"VLLM_OMNI_FISH_KVCACHE_ATTN=required but Fish kvcache attention is unavailable: {load_error()!r}"
|
||||
)
|
||||
logger.warning(
|
||||
"Fish kvcache attention is enabled but unavailable: %r",
|
||||
load_error(),
|
||||
)
|
||||
return 0
|
||||
|
||||
installed = 0
|
||||
for layer in model.layers:
|
||||
impl = layer.self_attn.attn.impl
|
||||
if getattr(impl, "_fish_kvcache_attn_installed", False):
|
||||
continue
|
||||
|
||||
original_forward = impl.forward
|
||||
|
||||
def fish_forward(
|
||||
this_impl: Any,
|
||||
layer: Any,
|
||||
query: Any,
|
||||
key: Any,
|
||||
value: Any,
|
||||
kv_cache: Any,
|
||||
attn_metadata: Any,
|
||||
output: Any,
|
||||
output_scale: Any = None,
|
||||
output_block_scale: Any = None,
|
||||
*,
|
||||
_original_forward: Callable[..., Any] = original_forward,
|
||||
) -> Any:
|
||||
return _forward_with_fish_kvcache(
|
||||
this_impl,
|
||||
_original_forward,
|
||||
layer,
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
output_scale=output_scale,
|
||||
output_block_scale=output_block_scale,
|
||||
)
|
||||
|
||||
impl.forward = types.MethodType(fish_forward, impl)
|
||||
impl._fish_kvcache_attn_installed = True
|
||||
installed += 1
|
||||
|
||||
if installed:
|
||||
logger.info("Installed Fish decode-only kvcache attention backend on %d attention layers", installed)
|
||||
elif is_fish_kvcache_attn_required():
|
||||
raise RuntimeError("Fish kvcache attention is required but installed 0 attention layers")
|
||||
return installed
|
||||
@@ -0,0 +1,359 @@
|
||||
# ruff: noqa: N803
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
try:
|
||||
import triton
|
||||
import triton.language as tl
|
||||
except Exception as exc: # pragma: no cover - depends on optional Triton install
|
||||
triton = None
|
||||
tl = None
|
||||
_LOAD_ERROR: Exception | None = exc
|
||||
else:
|
||||
_LOAD_ERROR = None
|
||||
|
||||
_BLOCK_SIZE = 16
|
||||
_HEAD_DIM = 128
|
||||
_BLOCK_N = 128
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
return _LOAD_ERROR is None and triton is not None and tl is not None
|
||||
|
||||
|
||||
def load_error() -> Exception | None:
|
||||
return _LOAD_ERROR
|
||||
|
||||
|
||||
if is_available():
|
||||
|
||||
@triton.jit
|
||||
def _small_decode_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
BLOCK_TABLE,
|
||||
SEQ_LENS,
|
||||
OUT,
|
||||
SCALE: tl.constexpr,
|
||||
MAX_SEQ_LEN: tl.constexpr,
|
||||
MAX_BLOCKS_PER_SEQ: tl.constexpr,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
KV_GROUP: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
GUARD_MAX_SEQ_LEN: tl.constexpr,
|
||||
):
|
||||
batch_id = tl.program_id(0)
|
||||
kv_head = tl.program_id(1)
|
||||
|
||||
offs_h = tl.arange(0, BLOCK_H)
|
||||
offs_d = tl.arange(0, BLOCK_D)
|
||||
offs_n = tl.arange(0, BLOCK_N)
|
||||
q_heads = kv_head * KV_GROUP + offs_h
|
||||
mask_h = offs_h < KV_GROUP
|
||||
|
||||
q_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
|
||||
q = tl.load(Q + q_offsets, mask=mask_h[:, None], other=0.0)
|
||||
|
||||
seq_len = tl.load(SEQ_LENS + batch_id)
|
||||
if GUARD_MAX_SEQ_LEN and seq_len > GUARD_MAX_SEQ_LEN:
|
||||
return
|
||||
|
||||
m_i = tl.full((BLOCK_H,), -float("inf"), tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), tl.float32)
|
||||
acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32)
|
||||
|
||||
start_n = 0
|
||||
while start_n < seq_len:
|
||||
cur_n = start_n + offs_n
|
||||
mask_n = cur_n < seq_len
|
||||
logical_block = cur_n // 16
|
||||
block_offset = cur_n - logical_block * 16
|
||||
physical_block = tl.load(
|
||||
BLOCK_TABLE + batch_id * MAX_BLOCKS_PER_SEQ + logical_block,
|
||||
mask=mask_n,
|
||||
other=0,
|
||||
)
|
||||
|
||||
kv_offsets = (
|
||||
(physical_block[:, None] * 16 + block_offset[:, None]) * NUM_KV_HEADS + kv_head
|
||||
) * BLOCK_D + offs_d[None, :]
|
||||
k = tl.load(K + kv_offsets, mask=mask_n[:, None], other=0.0)
|
||||
qk = tl.dot(q, tl.trans(k)) * SCALE
|
||||
qk = tl.where(mask_h[:, None] & mask_n[None, :], qk, -float("inf"))
|
||||
|
||||
v = tl.load(V + kv_offsets, mask=mask_n[:, None], other=0.0)
|
||||
m_new = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp(qk - m_new[:, None])
|
||||
alpha = tl.exp(m_i - m_new)
|
||||
acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v)
|
||||
l_i = l_i * alpha + tl.sum(p, axis=1)
|
||||
m_i = m_new
|
||||
start_n += BLOCK_N
|
||||
|
||||
denom = tl.where(l_i > 0.0, l_i, 1.0)
|
||||
result = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0)
|
||||
out_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
|
||||
tl.store(OUT + out_offsets, result, mask=mask_h[:, None])
|
||||
|
||||
@triton.jit
|
||||
def _split_partial_decode_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
BLOCK_TABLE,
|
||||
SEQ_LENS,
|
||||
PARTIAL_M,
|
||||
PARTIAL_L,
|
||||
PARTIAL_ACC,
|
||||
SCALE: tl.constexpr,
|
||||
MAX_BLOCKS_PER_SEQ: tl.constexpr,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
KV_GROUP: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
SPLIT_TOKENS: tl.constexpr,
|
||||
BATCH_SIZE: tl.constexpr,
|
||||
MIN_SEQ_LEN: tl.constexpr,
|
||||
):
|
||||
batch_id = tl.program_id(0)
|
||||
kv_head = tl.program_id(1)
|
||||
split_id = tl.program_id(2)
|
||||
|
||||
offs_h = tl.arange(0, BLOCK_H)
|
||||
offs_d = tl.arange(0, BLOCK_D)
|
||||
offs_n = tl.arange(0, BLOCK_N)
|
||||
q_heads = kv_head * KV_GROUP + offs_h
|
||||
mask_h = offs_h < KV_GROUP
|
||||
|
||||
partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads
|
||||
partial_acc_offsets = partial_head_offsets[:, None] * BLOCK_D + offs_d[None, :]
|
||||
|
||||
seq_len = tl.load(SEQ_LENS + batch_id)
|
||||
if MIN_SEQ_LEN and seq_len < MIN_SEQ_LEN:
|
||||
return
|
||||
|
||||
m_i = tl.full((BLOCK_H,), -float("inf"), tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), tl.float32)
|
||||
acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32)
|
||||
|
||||
split_begin = split_id * SPLIT_TOKENS
|
||||
if split_begin >= seq_len:
|
||||
tl.store(PARTIAL_M + partial_head_offsets, m_i, mask=mask_h)
|
||||
tl.store(PARTIAL_L + partial_head_offsets, l_i, mask=mask_h)
|
||||
tl.store(PARTIAL_ACC + partial_acc_offsets, acc, mask=mask_h[:, None])
|
||||
return
|
||||
|
||||
q_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
|
||||
q = tl.load(Q + q_offsets, mask=mask_h[:, None], other=0.0)
|
||||
|
||||
for split_offset in tl.range(0, SPLIT_TOKENS, BLOCK_N):
|
||||
cur_n = split_begin + split_offset + offs_n
|
||||
mask_n = cur_n < seq_len
|
||||
logical_block = cur_n // 16
|
||||
block_offset = cur_n - logical_block * 16
|
||||
physical_block = tl.load(
|
||||
BLOCK_TABLE + batch_id * MAX_BLOCKS_PER_SEQ + logical_block,
|
||||
mask=mask_n,
|
||||
other=0,
|
||||
)
|
||||
|
||||
kv_offsets = (
|
||||
(physical_block[:, None] * 16 + block_offset[:, None]) * NUM_KV_HEADS + kv_head
|
||||
) * BLOCK_D + offs_d[None, :]
|
||||
k = tl.load(K + kv_offsets, mask=mask_n[:, None], other=0.0)
|
||||
qk = tl.dot(q, tl.trans(k)) * SCALE
|
||||
qk = tl.where(mask_h[:, None] & mask_n[None, :], qk, -float("inf"))
|
||||
|
||||
v = tl.load(V + kv_offsets, mask=mask_n[:, None], other=0.0)
|
||||
m_new = tl.maximum(m_i, tl.max(qk, axis=1))
|
||||
p = tl.exp(qk - m_new[:, None])
|
||||
alpha = tl.exp(m_i - m_new)
|
||||
acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v)
|
||||
l_i = l_i * alpha + tl.sum(p, axis=1)
|
||||
m_i = m_new
|
||||
|
||||
tl.store(PARTIAL_M + partial_head_offsets, m_i, mask=mask_h)
|
||||
tl.store(PARTIAL_L + partial_head_offsets, l_i, mask=mask_h)
|
||||
tl.store(PARTIAL_ACC + partial_acc_offsets, acc, mask=mask_h[:, None])
|
||||
|
||||
@triton.jit
|
||||
def _split_combine_decode_kernel(
|
||||
SEQ_LENS,
|
||||
PARTIAL_M,
|
||||
PARTIAL_L,
|
||||
PARTIAL_ACC,
|
||||
OUT,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
KV_GROUP: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BATCH_SIZE: tl.constexpr,
|
||||
NUM_SPLITS: tl.constexpr,
|
||||
MIN_SEQ_LEN: tl.constexpr,
|
||||
):
|
||||
batch_id = tl.program_id(0)
|
||||
kv_head = tl.program_id(1)
|
||||
|
||||
seq_len = tl.load(SEQ_LENS + batch_id)
|
||||
if MIN_SEQ_LEN and seq_len < MIN_SEQ_LEN:
|
||||
return
|
||||
|
||||
offs_h = tl.arange(0, BLOCK_H)
|
||||
offs_d = tl.arange(0, BLOCK_D)
|
||||
q_heads = kv_head * KV_GROUP + offs_h
|
||||
mask_h = offs_h < KV_GROUP
|
||||
|
||||
global_m = tl.full((BLOCK_H,), -float("inf"), tl.float32)
|
||||
for split_id in tl.range(0, NUM_SPLITS):
|
||||
partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads
|
||||
split_m = tl.load(PARTIAL_M + partial_head_offsets, mask=mask_h, other=-float("inf"))
|
||||
global_m = tl.maximum(global_m, split_m)
|
||||
|
||||
global_l = tl.zeros((BLOCK_H,), tl.float32)
|
||||
acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32)
|
||||
for split_id in tl.range(0, NUM_SPLITS):
|
||||
partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads
|
||||
split_m = tl.load(PARTIAL_M + partial_head_offsets, mask=mask_h, other=-float("inf"))
|
||||
split_l = tl.load(PARTIAL_L + partial_head_offsets, mask=mask_h, other=0.0)
|
||||
weight = tl.exp(split_m - global_m)
|
||||
partial_acc_offsets = partial_head_offsets[:, None] * BLOCK_D + offs_d[None, :]
|
||||
split_acc = tl.load(PARTIAL_ACC + partial_acc_offsets, mask=mask_h[:, None], other=0.0)
|
||||
global_l += split_l * weight
|
||||
acc += split_acc * weight[:, None]
|
||||
|
||||
denom = tl.where(global_l > 0.0, global_l, 1.0)
|
||||
result = tl.where(global_l[:, None] > 0.0, acc / denom[:, None], 0.0)
|
||||
out_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :]
|
||||
tl.store(OUT + out_offsets, result, mask=mask_h[:, None])
|
||||
|
||||
|
||||
def fish_decode_kvcache_attn_triton(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
*,
|
||||
scale: float,
|
||||
max_seq_len: int,
|
||||
small_path_max_seq_len: int,
|
||||
long_split_tokens: int,
|
||||
partial_m: torch.Tensor,
|
||||
partial_l: torch.Tensor,
|
||||
partial_acc: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if not is_available():
|
||||
raise RuntimeError(f"Fish Triton attention is unavailable: {_LOAD_ERROR!r}")
|
||||
batch_size, num_q_heads, head_dim = query.shape
|
||||
block_size = key_cache.shape[1]
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
if head_dim != _HEAD_DIM or block_size != _BLOCK_SIZE:
|
||||
raise RuntimeError("Fish Triton attention only supports head_dim=128 and block_size=16")
|
||||
if num_q_heads % num_kv_heads != 0:
|
||||
raise RuntimeError("num_q_heads must be divisible by num_kv_heads")
|
||||
|
||||
kv_group = num_q_heads // num_kv_heads
|
||||
block_h = triton.next_power_of_2(kv_group)
|
||||
max_blocks_per_seq = block_table.shape[1]
|
||||
|
||||
if max_seq_len <= small_path_max_seq_len:
|
||||
_small_decode_kernel[(batch_size, num_kv_heads)](
|
||||
query,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
out,
|
||||
SCALE=float(scale),
|
||||
MAX_SEQ_LEN=int(max_seq_len),
|
||||
MAX_BLOCKS_PER_SEQ=max_blocks_per_seq,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
KV_GROUP=kv_group,
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_D=head_dim,
|
||||
BLOCK_N=_BLOCK_N,
|
||||
GUARD_MAX_SEQ_LEN=0,
|
||||
num_warps=4,
|
||||
num_stages=3,
|
||||
)
|
||||
return out
|
||||
|
||||
_small_decode_kernel[(batch_size, num_kv_heads)](
|
||||
query,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
out,
|
||||
SCALE=float(scale),
|
||||
MAX_SEQ_LEN=small_path_max_seq_len,
|
||||
MAX_BLOCKS_PER_SEQ=max_blocks_per_seq,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
KV_GROUP=kv_group,
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_D=head_dim,
|
||||
BLOCK_N=_BLOCK_N,
|
||||
GUARD_MAX_SEQ_LEN=small_path_max_seq_len,
|
||||
num_warps=4,
|
||||
num_stages=3,
|
||||
)
|
||||
num_splits = triton.cdiv(int(max_seq_len), long_split_tokens)
|
||||
expected_partial_acc_shape = (num_splits, batch_size, num_q_heads, head_dim)
|
||||
if tuple(partial_acc.shape) != expected_partial_acc_shape:
|
||||
raise RuntimeError(
|
||||
"Fish Triton long attention workspace shape mismatch: "
|
||||
f"expected partial_acc={expected_partial_acc_shape}, got {tuple(partial_acc.shape)}"
|
||||
)
|
||||
_split_partial_decode_kernel[(batch_size, num_kv_heads, num_splits)](
|
||||
query,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
partial_m,
|
||||
partial_l,
|
||||
partial_acc,
|
||||
SCALE=float(scale),
|
||||
MAX_BLOCKS_PER_SEQ=max_blocks_per_seq,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
KV_GROUP=kv_group,
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_D=head_dim,
|
||||
BLOCK_N=_BLOCK_N,
|
||||
SPLIT_TOKENS=long_split_tokens,
|
||||
BATCH_SIZE=batch_size,
|
||||
MIN_SEQ_LEN=small_path_max_seq_len + 1,
|
||||
num_warps=4,
|
||||
num_stages=3,
|
||||
)
|
||||
_split_combine_decode_kernel[(batch_size, num_kv_heads)](
|
||||
seq_lens,
|
||||
partial_m,
|
||||
partial_l,
|
||||
partial_acc,
|
||||
out,
|
||||
NUM_Q_HEADS=num_q_heads,
|
||||
NUM_KV_HEADS=num_kv_heads,
|
||||
KV_GROUP=kv_group,
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_D=head_dim,
|
||||
BATCH_SIZE=batch_size,
|
||||
NUM_SPLITS=num_splits,
|
||||
MIN_SEQ_LEN=small_path_max_seq_len + 1,
|
||||
num_warps=4,
|
||||
num_stages=3,
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,68 @@
|
||||
# Fish Speech S2 Pro high-concurrency dual-GPU deploy config.
|
||||
#
|
||||
# Stage0 SlowAR runs on GPU0 and Stage1 DAC runs on GPU1. Use this config
|
||||
# when the deployment can trade an extra GPU for lower Stage0/Stage1
|
||||
# interference between the AR and DAC stages.
|
||||
async_chunk: true
|
||||
enable_chunked_prefill: false
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.001
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
fish_speech_tensor_codes: true
|
||||
fish_speech_single_initial_chunk: true
|
||||
fish_speech_backlog_codec_chunk_frames: 50
|
||||
fish_speech_backlog_load_threshold: 0.75
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
initial_codec_chunk_frames: 4
|
||||
fish_speech_dac_dtype: fp16
|
||||
fish_speech_dac_max_padded_frames: 256
|
||||
fish_speech_dac_max_batch: 0
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 64
|
||||
gpu_memory_utilization: 0.85
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 16384
|
||||
max_model_len: 16384
|
||||
devices: "0"
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.8
|
||||
top_k: 30
|
||||
top_p: 0.9
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
repetition_penalty: 1.0
|
||||
|
||||
- stage_id: 1
|
||||
max_num_seqs: 64
|
||||
gpu_memory_utilization: 0.25
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 16384
|
||||
max_model_len: 16384
|
||||
devices: "1"
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
repetition_penalty: 1.0
|
||||
@@ -0,0 +1,68 @@
|
||||
# Fish Speech S2 Pro high-concurrency single-GPU deploy config.
|
||||
#
|
||||
# Stage0 SlowAR and Stage1 DAC share GPU0. This config keeps the optimized
|
||||
# Fish runtime path explicit instead of changing the default
|
||||
# fish_qwen3_omni.yaml deployment.
|
||||
async_chunk: true
|
||||
enable_chunked_prefill: false
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.001
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
fish_speech_tensor_codes: true
|
||||
fish_speech_single_initial_chunk: true
|
||||
fish_speech_backlog_codec_chunk_frames: 50
|
||||
fish_speech_backlog_load_threshold: 0.75
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
initial_codec_chunk_frames: 4
|
||||
fish_speech_dac_dtype: fp16
|
||||
fish_speech_dac_max_padded_frames: 256
|
||||
fish_speech_dac_max_batch: 0
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 64
|
||||
gpu_memory_utilization: 0.72
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 16384
|
||||
max_model_len: 16384
|
||||
devices: "0"
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.8
|
||||
top_k: 30
|
||||
top_p: 0.9
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
repetition_penalty: 1.0
|
||||
|
||||
- stage_id: 1
|
||||
max_num_seqs: 64
|
||||
gpu_memory_utilization: 0.18
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 16384
|
||||
max_model_len: 16384
|
||||
devices: "0"
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
repetition_penalty: 1.0
|
||||
@@ -64,6 +64,21 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
|
||||
self.requests_origin_status = {}
|
||||
self.requests_num_chunks_sent: dict[str, int] = defaultdict(int)
|
||||
|
||||
@staticmethod
|
||||
def _is_truthy_scalar(value: Any) -> bool:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return value.numel() == 1 and bool(value.item())
|
||||
return bool(value) if value is not None else False
|
||||
|
||||
@staticmethod
|
||||
def _confirmed_num_computed_tokens(request: Request) -> int:
|
||||
# vLLM async scheduling advances num_computed_tokens with output
|
||||
# placeholders before the corresponding token is committed. Connector
|
||||
# chunk send watermarks must use only committed tokens.
|
||||
num_computed = int(getattr(request, "num_computed_tokens", 0))
|
||||
num_placeholders = int(getattr(request, "num_output_placeholders", 0) or 0)
|
||||
return max(0, num_computed - num_placeholders)
|
||||
|
||||
@classmethod
|
||||
def create_connector(cls, model_config: Any):
|
||||
connector_config = getattr(model_config, "stage_connector_config", None)
|
||||
@@ -119,16 +134,19 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
|
||||
request: Request object
|
||||
"""
|
||||
|
||||
confirmed_num_computed_tokens = self._confirmed_num_computed_tokens(request)
|
||||
|
||||
# If the request is preempted, skip the already saved chunks.
|
||||
if request.num_computed_tokens < self.requests_num_chunks_sent.get(request.external_req_id, 0):
|
||||
if confirmed_num_computed_tokens < self.requests_num_chunks_sent.get(request.external_req_id, 0):
|
||||
logger.warning(
|
||||
f"Enqueue save_async for request {request.external_req_id}, "
|
||||
f"request.num_computed_tokens={request.num_computed_tokens}, "
|
||||
f"request.num_output_placeholders={getattr(request, 'num_output_placeholders', 0)}, "
|
||||
f"previous_chunks_sent={self.requests_num_chunks_sent.get(request.external_req_id, 0)}"
|
||||
)
|
||||
return
|
||||
|
||||
self.requests_num_chunks_sent[request.external_req_id] = request.num_computed_tokens
|
||||
self.requests_num_chunks_sent[request.external_req_id] = confirmed_num_computed_tokens
|
||||
task = {
|
||||
"pooling_output": pooling_output,
|
||||
"request": request,
|
||||
@@ -166,24 +184,36 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
|
||||
self.get_req_chunk[req_id] += 1
|
||||
|
||||
meta = payload_data.get("meta", {})
|
||||
payload_finished = self._is_truthy_scalar(meta.get("finished"))
|
||||
if self.model_mode == "ar":
|
||||
request.additional_information = payload_data
|
||||
if meta.get("finished"):
|
||||
if payload_finished:
|
||||
self.finished_requests.add(req_id)
|
||||
else:
|
||||
if meta.get("finished"):
|
||||
if payload_finished:
|
||||
self.finished_requests.add(req_id)
|
||||
|
||||
new_ids = payload_data.get("codes", {}).get("audio")
|
||||
if isinstance(new_ids, torch.Tensor):
|
||||
has_tensor_codes = isinstance(new_ids, torch.Tensor)
|
||||
use_tensor_codes = has_tensor_codes and new_ids.ndim >= 2
|
||||
if use_tensor_codes:
|
||||
request.prompt_token_ids = [0] if new_ids.numel() > 0 else []
|
||||
elif has_tensor_codes:
|
||||
new_ids = new_ids.tolist()
|
||||
elif new_ids is None:
|
||||
new_ids = []
|
||||
request.prompt_token_ids = new_ids
|
||||
request.prompt_token_ids = new_ids
|
||||
if not use_tensor_codes:
|
||||
request.prompt_token_ids = new_ids
|
||||
prev_info = getattr(request, "additional_information", None)
|
||||
info = dict(prev_info) if isinstance(prev_info, dict) else {}
|
||||
for key, value in payload_data.items():
|
||||
if key == "codes":
|
||||
if use_tensor_codes and isinstance(value, dict):
|
||||
existing_sub = info.get(key)
|
||||
merged_sub = dict(existing_sub) if isinstance(existing_sub, dict) else {}
|
||||
merged_sub.update(value)
|
||||
info[key] = merged_sub
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
existing_sub = info.get(key)
|
||||
@@ -199,8 +229,13 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
|
||||
request.num_computed_tokens = 0
|
||||
|
||||
# Empty chunk with more data expected: keep polling.
|
||||
if not new_ids and not meta.get("finished"):
|
||||
return True
|
||||
has_new_ids = bool(new_ids.numel()) if use_tensor_codes else bool(new_ids)
|
||||
if not has_new_ids and not payload_finished:
|
||||
# The base recv loop treats False as "not ready yet" and
|
||||
# requeues the request. Do not mark an empty non-terminal
|
||||
# chunk as ready, otherwise Stage1 can consume before the
|
||||
# first DAC frame arrives.
|
||||
return False
|
||||
|
||||
# Mark as finished for consumption
|
||||
self._finished_load_reqs.add(req_id)
|
||||
|
||||
@@ -82,10 +82,12 @@ class OmniMsgpackEncoder:
|
||||
|
||||
def _encode_tensor(self, tensor: torch.Tensor) -> dict[str, Any]:
|
||||
"""Encode torch.Tensor to dict."""
|
||||
t = tensor.detach().contiguous().cpu()
|
||||
t = tensor.detach().cpu()
|
||||
# Handle 0-dimensional (scalar) tensors by reshaping to 1D first
|
||||
if t.dim() == 0:
|
||||
t = t.reshape(1)
|
||||
if not t.is_contiguous():
|
||||
t = t.contiguous()
|
||||
t = t.view(torch.uint8)
|
||||
return {
|
||||
_TENSOR_MARKER: True,
|
||||
|
||||
@@ -11,6 +11,7 @@ Install with: ``pip install fish-speech``
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
@@ -33,6 +34,41 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _connector_extra_config(vllm_config: VllmConfig) -> dict[str, Any]:
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
connector_cfg = getattr(model_config, "stage_connector_config", None)
|
||||
if isinstance(connector_cfg, dict):
|
||||
return connector_cfg.get("extra", connector_cfg)
|
||||
extra = getattr(connector_cfg, "extra", None)
|
||||
return extra if isinstance(extra, dict) else {}
|
||||
|
||||
|
||||
def _get_int_config(extra_cfg: dict[str, Any], default: int, *names: str) -> int:
|
||||
value = None
|
||||
for name in names:
|
||||
if name in extra_cfg:
|
||||
value = extra_cfg[name]
|
||||
break
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
return max(0, int(value))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid Fish Speech DAC integer config {names[0]}={value!r}") from exc
|
||||
|
||||
|
||||
def _get_dac_dtype(extra_cfg: dict[str, Any]) -> torch.dtype:
|
||||
value = extra_cfg.get("fish_speech_dac_dtype", "float32")
|
||||
value = str(value).strip().lower()
|
||||
if value in {"fp16", "float16", "half"}:
|
||||
return torch.float16
|
||||
if value in {"bf16", "bfloat16"}:
|
||||
return torch.bfloat16
|
||||
if value in {"fp32", "float32", ""}:
|
||||
return torch.float32
|
||||
raise ValueError(f"Invalid Fish Speech DAC dtype: {value!r}")
|
||||
|
||||
|
||||
class FishSpeechDACDecoder(nn.Module):
|
||||
"""Stage-1 DAC decoder for Fish Speech S2 Pro (GenerationModelRunner).
|
||||
|
||||
@@ -58,6 +94,43 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
self._output_sample_rate: int = DAC_SAMPLE_RATE
|
||||
self._hop_length: int = DAC_HOP_LENGTH
|
||||
self._logged_codec_stats = False
|
||||
self._codec_decode_takes_lengths: bool | None = None
|
||||
extra_cfg = _connector_extra_config(vllm_config)
|
||||
self._dac_dtype = _get_dac_dtype(extra_cfg)
|
||||
self._decode_batch_max_padded_frames = _get_int_config(
|
||||
extra_cfg,
|
||||
0,
|
||||
"fish_speech_dac_max_padded_frames",
|
||||
"dac_max_padded_frames",
|
||||
)
|
||||
self._decode_batch_max_batch = _get_int_config(
|
||||
extra_cfg,
|
||||
0,
|
||||
"fish_speech_dac_max_batch",
|
||||
"dac_max_batch",
|
||||
)
|
||||
|
||||
def _decode_codes(
|
||||
self,
|
||||
codes_bqf: torch.Tensor,
|
||||
feature_lengths: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert self._codec is not None
|
||||
if self._codec_decode_takes_lengths:
|
||||
return self._codec.decode(codes_bqf, feature_lengths)
|
||||
|
||||
if hasattr(self._codec, "from_indices"):
|
||||
wav_batch = self._codec.from_indices(codes_bqf)
|
||||
else:
|
||||
wav_batch = self._codec.decode(codes_bqf)
|
||||
max_frames = max(int(codes_bqf.shape[-1]), 1)
|
||||
scale = wav_batch.shape[-1] / max_frames
|
||||
audio_lengths = torch.clamp(
|
||||
torch.round(feature_lengths.to(torch.float32) * scale).to(torch.long),
|
||||
min=0,
|
||||
max=wav_batch.shape[-1],
|
||||
)
|
||||
return wav_batch, audio_lengths
|
||||
|
||||
def _bake_weight_norm(self, codec: nn.Module) -> None:
|
||||
baked = 0
|
||||
@@ -149,14 +222,16 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
codec.quantizer.downsample = None
|
||||
|
||||
device = self.vllm_config.device_config.device
|
||||
codec = codec.to(device=device, dtype=torch.float32)
|
||||
codec = codec.to(device=device, dtype=self._dac_dtype)
|
||||
codec.eval()
|
||||
self._codec = codec
|
||||
self._codec_decode_takes_lengths = len(inspect.signature(codec.decode).parameters) >= 2
|
||||
|
||||
logger.info(
|
||||
"Fish Speech DAC codec loaded from %s (device=%s, sample_rate=%d)",
|
||||
"Fish Speech DAC codec loaded from %s (device=%s, dtype=%s, sample_rate=%d)",
|
||||
codec_path,
|
||||
device,
|
||||
self._dac_dtype,
|
||||
self._output_sample_rate,
|
||||
)
|
||||
|
||||
@@ -188,6 +263,28 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)]
|
||||
return [ids]
|
||||
|
||||
def _codes_from_runtime_info(
|
||||
self,
|
||||
info: dict[str, Any] | None,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
if not isinstance(info, dict):
|
||||
return None
|
||||
codes = info.get("codes", {}).get("audio")
|
||||
if not isinstance(codes, torch.Tensor) or codes.numel() == 0:
|
||||
return None
|
||||
|
||||
q = self._num_codebooks
|
||||
codes = codes.to(device=device, dtype=torch.long, non_blocking=True).contiguous()
|
||||
if codes.ndim != 2 or codes.shape[0] != q:
|
||||
logger.warning(
|
||||
"DAC tensor codes must have shape [num_codebooks, frames], got %s for num_codebooks=%d.",
|
||||
tuple(codes.shape),
|
||||
q,
|
||||
)
|
||||
return None
|
||||
return codes
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
@@ -203,14 +300,14 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
input_ids layout per request: flat codes [num_codebooks * num_frames].
|
||||
Codes are codebook-major: [cb0_f0, cb0_f1, ..., cb0_fN, cb1_f0, ...].
|
||||
"""
|
||||
self._ensure_codec_loaded()
|
||||
assert self._codec is not None
|
||||
|
||||
q = self._num_codebooks
|
||||
sr_val = self._output_sample_rate
|
||||
sr_tensor = torch.tensor(sr_val, dtype=torch.int32)
|
||||
empty = torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
self._ensure_codec_loaded()
|
||||
assert self._codec is not None
|
||||
|
||||
if input_ids is None or input_ids.numel() == 0:
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
@@ -235,9 +332,22 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
left_context_size[i] = meta["left_context_size"]
|
||||
|
||||
for i, req_ids in enumerate(request_ids_list):
|
||||
ctx_frames = left_context_size[i]
|
||||
runtime_codes = (
|
||||
self._codes_from_runtime_info(runtime_additional_information[i], ids.device)
|
||||
if runtime_additional_information is not None and i < len(runtime_additional_information)
|
||||
else None
|
||||
)
|
||||
if runtime_codes is not None:
|
||||
frames = runtime_codes.shape[1]
|
||||
parsed_ctx_frames[i] = ctx_frames
|
||||
parsed_total_frames[i] = frames
|
||||
valid_codes_qf.append(runtime_codes)
|
||||
valid_indices.append(i)
|
||||
continue
|
||||
|
||||
if req_ids.numel() < 1:
|
||||
continue
|
||||
ctx_frames = left_context_size[i]
|
||||
flat = req_ids
|
||||
n = flat.numel()
|
||||
if n == 0 or n % q != 0:
|
||||
@@ -279,25 +389,61 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
feature_lengths = torch.tensor(
|
||||
[codes_qf.shape[1] for codes_qf in valid_codes_qf],
|
||||
device=valid_codes_qf[0].device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
max_frames = int(feature_lengths.max().item())
|
||||
batch_size = len(valid_codes_qf)
|
||||
wav_tensors: list[torch.Tensor | None] = [None] * len(valid_codes_qf)
|
||||
|
||||
codes_bqf = torch.zeros(
|
||||
(batch_size, q, max_frames),
|
||||
device=valid_codes_qf[0].device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
for i, codes_qf in enumerate(valid_codes_qf):
|
||||
frame_count = int(feature_lengths[i].item())
|
||||
codes_bqf[i, :, :frame_count] = codes_qf
|
||||
def _iter_bounded_groups(group: list[tuple[int, torch.Tensor]]) -> Iterable[list[tuple[int, torch.Tensor]]]:
|
||||
max_padded_frames = int(getattr(self, "_decode_batch_max_padded_frames", 0))
|
||||
max_batch = int(getattr(self, "_decode_batch_max_batch", 0))
|
||||
if max_padded_frames <= 0 and max_batch <= 0:
|
||||
yield group
|
||||
return
|
||||
|
||||
with torch.amp.autocast("cuda", enabled=False):
|
||||
wav_batch, audio_lengths = self._codec.decode(codes_bqf, feature_lengths)
|
||||
current: list[tuple[int, torch.Tensor]] = []
|
||||
current_max_frames = 0
|
||||
for item in sorted(group, key=lambda pair: int(pair[1].shape[1])):
|
||||
frames = int(item[1].shape[1])
|
||||
next_max_frames = max(current_max_frames, frames)
|
||||
next_batch_size = len(current) + 1
|
||||
exceeds_batch = max_batch > 0 and next_batch_size > max_batch
|
||||
exceeds_work = max_padded_frames > 0 and next_batch_size * next_max_frames > max_padded_frames
|
||||
if current and (exceeds_batch or exceeds_work):
|
||||
yield current
|
||||
current = [item]
|
||||
current_max_frames = frames
|
||||
else:
|
||||
current.append(item)
|
||||
current_max_frames = next_max_frames
|
||||
if current:
|
||||
yield current
|
||||
|
||||
def _decode_group(group: list[tuple[int, torch.Tensor]]) -> None:
|
||||
actual_frames = [int(codes_qf.shape[1]) for _, codes_qf in group]
|
||||
target_frames = max(actual_frames)
|
||||
batch_size = len(group)
|
||||
first_codes = group[0][1]
|
||||
feature_lengths = torch.tensor(actual_frames, device=first_codes.device, dtype=torch.long)
|
||||
codes_bqf = torch.zeros(
|
||||
(batch_size, q, target_frames),
|
||||
device=first_codes.device,
|
||||
dtype=torch.long,
|
||||
)
|
||||
for row, (_, codes_qf) in enumerate(group):
|
||||
frame_count = int(codes_qf.shape[1])
|
||||
codes_bqf[row, :, :frame_count] = codes_qf
|
||||
|
||||
with torch.amp.autocast("cuda", enabled=False):
|
||||
wav_batch, audio_lengths = self._decode_codes(codes_bqf, feature_lengths)
|
||||
audio_lengths_list = (
|
||||
audio_lengths.detach().to(device="cpu", dtype=torch.long).reshape(-1).tolist()
|
||||
if audio_lengths.numel() > 0
|
||||
else []
|
||||
)
|
||||
for row, (j, _) in enumerate(group):
|
||||
audio_len = int(audio_lengths_list[row]) if len(audio_lengths_list) > row else int(wav_batch.shape[-1])
|
||||
wav_tensors[j] = wav_batch[row, 0, :audio_len]
|
||||
|
||||
for bounded_group in _iter_bounded_groups(list(enumerate(valid_codes_qf))):
|
||||
_decode_group(bounded_group)
|
||||
|
||||
audios: list[torch.Tensor] = [empty] * num_req
|
||||
srs = [sr_tensor] * num_req
|
||||
@@ -305,8 +451,8 @@ class FishSpeechDACDecoder(nn.Module):
|
||||
for j, idx in enumerate(valid_indices):
|
||||
ctx_frames = parsed_ctx_frames[idx]
|
||||
total_frames = parsed_total_frames[idx]
|
||||
audio_len = int(audio_lengths[j].item()) if audio_lengths.numel() > j else int(wav_batch.shape[-1])
|
||||
wav = wav_batch[j, 0, :audio_len]
|
||||
wav = wav_tensors[j]
|
||||
assert wav is not None
|
||||
# Trim context frames (left overlap for streaming).
|
||||
if ctx_frames > 0:
|
||||
# Decode length may deviate from (frames * hop_length) due to model
|
||||
|
||||
@@ -4,14 +4,13 @@
|
||||
autoregressively after each Slow AR step. Analogous to Qwen3 TTS's
|
||||
CodePredictor but with its own embedding table, RoPE, and output head.
|
||||
|
||||
Uses re-prefill (no KV cache): each AR step forwards the full growing
|
||||
sequence through the 4-layer transformer. This trades ~O(T^2) attention
|
||||
FLOPs (negligible for T=10, 4 layers) for zero KV cache management.
|
||||
Uses a tiny per-call KV cache: each AR step forwards one codebook token while
|
||||
reusing the previous Fast AR keys/values.
|
||||
|
||||
Optimisations:
|
||||
- torch.compile on model forward (kernel fusion)
|
||||
- Pre-allocated projection / embedding buffer
|
||||
- Pre-allocated position_ids
|
||||
- Pre-allocated per-layer KV cache for the short codebook loop
|
||||
- Inline sampling
|
||||
"""
|
||||
|
||||
@@ -51,7 +50,8 @@ logger = init_logger(__name__)
|
||||
class _FastARAttention(nn.Module):
|
||||
"""Multi-head attention using F.scaled_dot_product_attention (SDPA).
|
||||
|
||||
Supports fused QKV, RoPE, optional q/k normalization, and native GQA.
|
||||
Supports fused QKV, RoPE, optional q/k normalization, native GQA, and a
|
||||
small one-token decode path for the residual codebook loop.
|
||||
Input: [B, seq_len, hidden_size].
|
||||
"""
|
||||
|
||||
@@ -124,6 +124,43 @@ class _FastARAttention(nn.Module):
|
||||
output, _ = self.o_proj(attn_out)
|
||||
return output.view(bsz, seq_len, -1)
|
||||
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
cache_pos: int,
|
||||
) -> torch.Tensor:
|
||||
bsz = int(hidden_states.shape[0])
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states.reshape(bsz, -1))
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
if self.q_norm is not None:
|
||||
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
|
||||
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
|
||||
|
||||
q, k = self.rotary_emb(position_ids.reshape(-1), q, k)
|
||||
|
||||
q = q.view(bsz, self.num_heads, self.head_dim).unsqueeze(2)
|
||||
k = k.view(bsz, self.num_kv_heads, self.head_dim)
|
||||
v = v.view(bsz, self.num_kv_heads, self.head_dim)
|
||||
k_cache[:bsz, :, cache_pos, :] = k
|
||||
v_cache[:bsz, :, cache_pos, :] = v
|
||||
|
||||
attn_out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k_cache[:bsz, :, : cache_pos + 1, :],
|
||||
v_cache[:bsz, :, : cache_pos + 1, :],
|
||||
scale=self.scaling,
|
||||
is_causal=False,
|
||||
enable_gqa=self._use_gqa,
|
||||
)
|
||||
attn_out = attn_out.transpose(1, 2).reshape(bsz, -1)
|
||||
output, _ = self.o_proj(attn_out)
|
||||
return output
|
||||
|
||||
|
||||
class _FastARMLP(nn.Module):
|
||||
"""SiLU-gated MLP, matching Qwen3/LLaMA MLP structure."""
|
||||
@@ -175,6 +212,25 @@ class _FastARDecoderLayer(nn.Module):
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
def forward_one(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
cache_pos: int,
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn.forward_one(hidden_states, position_ids, k_cache, v_cache, cache_pos)
|
||||
hidden_states = residual + hidden_states
|
||||
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
|
||||
|
||||
# ===================================================================
|
||||
# Fast AR Transformer Model
|
||||
@@ -199,6 +255,25 @@ class FishSpeechFastARModel(nn.Module):
|
||||
hidden_states = layer(hidden_states, position_ids)
|
||||
return hidden_states
|
||||
|
||||
def forward_one(
|
||||
self,
|
||||
input_embed: torch.Tensor,
|
||||
position_ids: torch.Tensor,
|
||||
k_caches: torch.Tensor,
|
||||
v_caches: torch.Tensor,
|
||||
cache_pos: int,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = input_embed
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
hidden_states = layer.forward_one(
|
||||
hidden_states,
|
||||
position_ids,
|
||||
k_caches[layer_idx],
|
||||
v_caches[layer_idx],
|
||||
cache_pos,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
stacked_params_mapping = [
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
@@ -259,13 +334,13 @@ class FishSpeechFastARModel(nn.Module):
|
||||
class FishSpeechFastAR(nn.Module):
|
||||
"""vLLM-native Fast AR for Fish Speech (residual codebooks).
|
||||
|
||||
Re-prefill approach: each AR step forwards the full growing sequence
|
||||
through the 4-layer transformer. No KV cache needed.
|
||||
Decode approach: each residual-codebook step forwards one token through
|
||||
the 4-layer transformer with a per-call KV cache.
|
||||
|
||||
Optimisations over baseline:
|
||||
1. torch.compile on model forward -- kernel fusion.
|
||||
2. Pre-allocated embedding buffer [B, max_seq, H].
|
||||
3. Pre-allocated position_ids.
|
||||
1. Pre-allocated embedding buffer [B, max_seq, H].
|
||||
2. Pre-allocated position_ids.
|
||||
3. Per-call KV cache for the short residual-codebook loop.
|
||||
4. Inline sampling.
|
||||
"""
|
||||
|
||||
@@ -307,6 +382,8 @@ class FishSpeechFastAR(nn.Module):
|
||||
# Pre-allocated buffers (lazily initialised on first forward).
|
||||
self._embed_buf: torch.Tensor | None = None
|
||||
self._pos_ids: torch.Tensor | None = None
|
||||
self._k_cache: torch.Tensor | None = None
|
||||
self._v_cache: torch.Tensor | None = None
|
||||
self._compiled_model_fwd: object | None = None
|
||||
self._compile_attempted = False
|
||||
self._compile_failed = False
|
||||
@@ -316,13 +393,27 @@ class FishSpeechFastAR(nn.Module):
|
||||
max_seq = self._num_codebooks + 1 # hidden_state + num_codebooks codes
|
||||
if (
|
||||
self._embed_buf is not None
|
||||
and self._pos_ids is not None
|
||||
and self._k_cache is not None
|
||||
and self._v_cache is not None
|
||||
and self._embed_buf.shape[0] >= bsz
|
||||
and self._embed_buf.device == device
|
||||
and self._embed_buf.dtype == dtype
|
||||
and self._pos_ids.shape[0] >= bsz
|
||||
and self._pos_ids.shape[1] == max_seq
|
||||
and self._pos_ids.device == device
|
||||
and self._k_cache.shape[1] >= bsz
|
||||
and self._k_cache.device == device
|
||||
and self._k_cache.dtype == dtype
|
||||
):
|
||||
return
|
||||
self._embed_buf = torch.zeros(bsz, max_seq, self._fast_dim, dtype=dtype, device=device)
|
||||
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device)
|
||||
self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device).unsqueeze(0).expand(bsz, -1).contiguous()
|
||||
num_layers = self.config.num_hidden_layers
|
||||
num_kv_heads = self.config.num_key_value_heads
|
||||
head_dim = self.config.head_dim
|
||||
self._k_cache = torch.empty(num_layers, bsz, num_kv_heads, max_seq, head_dim, dtype=dtype, device=device)
|
||||
self._v_cache = torch.empty_like(self._k_cache)
|
||||
|
||||
def _setup_compile(self) -> None:
|
||||
if self._compile_attempted:
|
||||
@@ -375,20 +466,10 @@ class FishSpeechFastAR(nn.Module):
|
||||
torch.accelerator.synchronize(device)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(self, step_input: torch.Tensor, step_pos_ids: torch.Tensor, bsz: int) -> torch.Tensor:
|
||||
if self._disable_compile_for_graph:
|
||||
model_fwd = self._compiled_model_fwd or self.model.forward
|
||||
else:
|
||||
model_fwd = self._compiled_model_fwd if bsz == 1 else self.model.forward
|
||||
try:
|
||||
return model_fwd(step_input, step_pos_ids)
|
||||
except Exception as exc:
|
||||
if model_fwd is self.model.forward or self._compile_failed:
|
||||
raise
|
||||
self._compile_failed = True
|
||||
self._compiled_model_fwd = self.model.forward
|
||||
logger.warning("Fish Speech Fast AR torch.compile fallback to eager after runtime failure: %s", exc)
|
||||
return self.model.forward(step_input, step_pos_ids)
|
||||
def _run_model_one(self, input_embed: torch.Tensor, step_pos_ids: torch.Tensor, cache_pos: int) -> torch.Tensor:
|
||||
assert self._k_cache is not None
|
||||
assert self._v_cache is not None
|
||||
return self.model.forward_one(input_embed, step_pos_ids, self._k_cache, self._v_cache, cache_pos)
|
||||
|
||||
@torch.inference_mode()
|
||||
def forward(
|
||||
@@ -401,6 +482,7 @@ class FishSpeechFastAR(nn.Module):
|
||||
top_k: int = 30,
|
||||
top_p: float = 0.9,
|
||||
seed: int | None = None,
|
||||
generator: torch.Generator | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Predict residual codebook codes 0..num_codebooks-1 autoregressively.
|
||||
|
||||
@@ -428,7 +510,6 @@ class FishSpeechFastAR(nn.Module):
|
||||
all_codes[:, 0] = semantic_code
|
||||
|
||||
self._ensure_buffers(bsz, device, dtype)
|
||||
self._setup_compile()
|
||||
|
||||
embed_buf = self._embed_buf
|
||||
pos_ids = self._pos_ids
|
||||
@@ -445,8 +526,7 @@ class FishSpeechFastAR(nn.Module):
|
||||
inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0
|
||||
|
||||
# Create a seeded generator for deterministic residual codebook sampling.
|
||||
generator = None
|
||||
if seed is not None and use_sampling:
|
||||
if generator is None and seed is not None and use_sampling:
|
||||
generator = torch.Generator(device=device)
|
||||
generator.manual_seed(seed)
|
||||
|
||||
@@ -455,15 +535,11 @@ class FishSpeechFastAR(nn.Module):
|
||||
# codebooks only have 1024 entries. Truncate logits for steps > 0.
|
||||
residual_codebook_size = 1024
|
||||
|
||||
for step in range(1, num_cb):
|
||||
seq_len = step + 1
|
||||
step_input = embed_buf[:bsz, :seq_len, :]
|
||||
# Use a dense 2D position tensor for every batch size; stride-0
|
||||
# views from expand() were fragile under compiled execution.
|
||||
step_pos_ids = pos_ids[:seq_len].unsqueeze(0).repeat(bsz, 1)
|
||||
self._run_model_one(embed_buf[:bsz, 0, :], pos_ids[:bsz, 0], 0)
|
||||
|
||||
hidden_out = self._run_model(step_input, step_pos_ids, bsz)
|
||||
logits = self.fast_output(self.fast_norm(hidden_out[:, -1, :]))
|
||||
for step in range(1, num_cb):
|
||||
hidden_out = self._run_model_one(embed_buf[:bsz, step, :], pos_ids[:bsz, step], step)
|
||||
logits = self.fast_output(self.fast_norm(hidden_out))
|
||||
|
||||
# Residual codebooks (step >= 1) only have 1024 entries.
|
||||
if step >= 1:
|
||||
|
||||
@@ -199,6 +199,12 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
|
||||
|
||||
# Qwen3 transformer backbone.
|
||||
self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))
|
||||
from vllm_omni.attention.fish_kvcache_attn import is_fish_kvcache_attn_enabled
|
||||
|
||||
if is_fish_kvcache_attn_enabled():
|
||||
from vllm_omni.attention.fish_kvcache_backend import install_fish_kvcache_attn_backend
|
||||
|
||||
install_fish_kvcache_attn_backend(self.model)
|
||||
|
||||
# Fish Speech uses interleaved (GPT-J) RoPE, not NeoX style.
|
||||
# vLLM's Qwen3Attention defaults to NeoX (is_neox_style=True).
|
||||
@@ -672,14 +678,19 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module):
|
||||
past_hidden = last_talker_hidden.reshape(bsz, -1).to(dtype=torch.bfloat16, device=dev)
|
||||
|
||||
# Run Fast AR to predict all num_codebooks codes.
|
||||
do_sample = kwargs.get("do_sample")
|
||||
temperature = kwargs.get("temperature")
|
||||
top_k = kwargs.get("top_k")
|
||||
top_p = kwargs.get("top_p")
|
||||
audio_codes = self.fast_ar(
|
||||
slow_ar_hidden=past_hidden,
|
||||
semantic_token_id=input_ids.reshape(bsz),
|
||||
do_sample=True,
|
||||
temperature=0.8,
|
||||
top_k=30,
|
||||
top_p=0.9,
|
||||
do_sample=True if do_sample is None else bool(do_sample),
|
||||
temperature=0.8 if temperature is None else float(temperature),
|
||||
top_k=30 if top_k is None else int(top_k),
|
||||
top_p=0.9 if top_p is None else float(top_p),
|
||||
seed=seed,
|
||||
generator=kwargs.get("generator"),
|
||||
) # [B, num_codebooks]
|
||||
|
||||
# Add codebook embeddings to the input embedding (from preprocess).
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.data_entry_keys import (
|
||||
CodesStruct,
|
||||
@@ -11,7 +10,85 @@ from vllm_omni.data_entry_keys import (
|
||||
OmniPayloadStruct,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
def _get_connector_extra(transfer_manager: Any) -> dict[str, Any]:
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
raw_cfg = getattr(connector, "config", {}) or {}
|
||||
return raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
|
||||
|
||||
|
||||
def _use_tensor_code_payload(transfer_manager: Any) -> bool:
|
||||
cfg = _get_connector_extra(transfer_manager)
|
||||
return bool(cfg.get("fish_speech_tensor_codes", False))
|
||||
|
||||
|
||||
def _cfg_bool(cfg: dict[str, Any], key: str, default: bool = False) -> bool:
|
||||
return bool(cfg.get(key, default))
|
||||
|
||||
|
||||
def _cfg_int(cfg: dict[str, Any], key: str, default: int = 0) -> int:
|
||||
value = cfg.get(key, default)
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid Fish Speech integer config {key}={value!r}") from exc
|
||||
|
||||
|
||||
def _cfg_float(cfg: dict[str, Any], key: str, default: float) -> float:
|
||||
value = cfg.get(key, default)
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid Fish Speech float config {key}={value!r}") from exc
|
||||
|
||||
|
||||
def _active_codec_requests(transfer_manager: Any) -> int:
|
||||
return sum(1 for value in transfer_manager.code_prompt_token_ids.values() if len(value) > 0)
|
||||
|
||||
|
||||
def _ready_codec_requests(transfer_manager: Any, min_frames: int) -> int:
|
||||
return sum(1 for value in transfer_manager.code_prompt_token_ids.values() if len(value) >= min_frames)
|
||||
|
||||
|
||||
def _select_backlog_chunk_size(
|
||||
transfer_manager: Any,
|
||||
cfg: dict[str, Any],
|
||||
base_chunk_size: int,
|
||||
current_length: int,
|
||||
) -> int:
|
||||
backlog_chunk_size = _cfg_int(
|
||||
cfg,
|
||||
"fish_speech_backlog_codec_chunk_frames",
|
||||
0,
|
||||
)
|
||||
if backlog_chunk_size <= base_chunk_size:
|
||||
return base_chunk_size
|
||||
|
||||
capacity = max(1, int(getattr(transfer_manager, "scheduler_max_num_seqs", 1)))
|
||||
active = _active_codec_requests(transfer_manager)
|
||||
load_factor = min(active / capacity, 1.0)
|
||||
threshold = _cfg_float(
|
||||
cfg,
|
||||
"fish_speech_backlog_load_threshold",
|
||||
0.75,
|
||||
)
|
||||
if load_factor < threshold:
|
||||
return base_chunk_size
|
||||
|
||||
ready_min = _cfg_int(
|
||||
cfg,
|
||||
"fish_speech_backlog_min_ready",
|
||||
max(2, min(capacity, int(capacity * threshold))),
|
||||
)
|
||||
# Include the current request even when the caller has not yet committed
|
||||
# the latest frame into the transfer-manager map.
|
||||
ready = _ready_codec_requests(transfer_manager, base_chunk_size)
|
||||
if current_length >= base_chunk_size:
|
||||
ready = max(ready, 1)
|
||||
if ready < ready_min:
|
||||
return base_chunk_size
|
||||
|
||||
return backlog_chunk_size
|
||||
|
||||
|
||||
def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None:
|
||||
@@ -21,11 +98,18 @@ def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None:
|
||||
return None
|
||||
if audio_codes.ndim == 2:
|
||||
frame = audio_codes[-1]
|
||||
if frame.numel() == 0 or not bool(frame.any().item()):
|
||||
valid = pooling_output.get("audio_code_valid")
|
||||
if isinstance(valid, torch.Tensor) and valid.numel() > 0:
|
||||
is_valid = bool(valid.reshape(-1)[-1].item())
|
||||
elif valid is not None:
|
||||
is_valid = bool(valid)
|
||||
else:
|
||||
is_valid = bool(frame.any().item())
|
||||
if frame.numel() == 0 or not is_valid:
|
||||
return None
|
||||
return frame.to(torch.long).reshape(-1)
|
||||
return frame.to(device="cpu", dtype=torch.long).reshape(-1)
|
||||
if audio_codes.ndim == 1:
|
||||
return audio_codes.to(torch.long).reshape(-1)
|
||||
return audio_codes.to(device="cpu", dtype=torch.long).reshape(-1)
|
||||
raise ValueError(f"Invalid audio_codes shape for Fish Speech async_chunk: {tuple(audio_codes.shape)}")
|
||||
|
||||
|
||||
@@ -74,20 +158,20 @@ def slow_ar_to_dac_decoder_async_chunk(
|
||||
"""
|
||||
request_id = request.external_req_id
|
||||
finished = bool(is_finished or request.is_finished())
|
||||
cfg = _get_connector_extra(transfer_manager)
|
||||
|
||||
if isinstance(pooling_output, dict):
|
||||
frame = _extract_last_frame(pooling_output)
|
||||
if frame is not None:
|
||||
transfer_manager.code_prompt_token_ids[request_id].append(frame.detach().to(device="cpu", dtype=torch.long))
|
||||
transfer_manager.code_prompt_token_ids[request_id].append(frame.detach())
|
||||
elif not finished:
|
||||
return None
|
||||
|
||||
connector = getattr(transfer_manager, "connector", None)
|
||||
raw_cfg = getattr(connector, "config", {}) or {}
|
||||
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
|
||||
chunk_size = int(cfg.get("codec_chunk_frames", 25))
|
||||
left_context_size_config = int(cfg.get("codec_left_context_frames", 25))
|
||||
initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0))
|
||||
configured_initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0))
|
||||
|
||||
initial_chunk_size = configured_initial_chunk_size
|
||||
|
||||
# Per-request override.
|
||||
additional_information = getattr(request, "additional_information", None)
|
||||
@@ -100,7 +184,7 @@ def slow_ar_to_dac_decoder_async_chunk(
|
||||
if entry.list_data is not None and len(entry.list_data) == 1:
|
||||
initial_chunk_size = int(entry.list_data[0])
|
||||
|
||||
if chunk_size <= 0 or left_context_size_config < 0 or initial_chunk_size < 0:
|
||||
if chunk_size <= 0 or left_context_size_config < 0 or configured_initial_chunk_size < 0 or initial_chunk_size < 0:
|
||||
raise ValueError(
|
||||
f"Invalid codec chunk config: codec_chunk_frames={chunk_size}, "
|
||||
f"codec_left_context_frames={left_context_size_config}, "
|
||||
@@ -110,6 +194,7 @@ def slow_ar_to_dac_decoder_async_chunk(
|
||||
initial_chunk_size = chunk_size
|
||||
|
||||
length = len(transfer_manager.code_prompt_token_ids[request_id])
|
||||
steady_chunk_size = _select_backlog_chunk_size(transfer_manager, cfg, chunk_size, length)
|
||||
|
||||
if length <= 0:
|
||||
if finished:
|
||||
@@ -119,9 +204,30 @@ def slow_ar_to_dac_decoder_async_chunk(
|
||||
)
|
||||
return None
|
||||
|
||||
in_initial_phase = initial_chunk_size > 0 and length <= chunk_size
|
||||
single_initial_chunk = _cfg_bool(
|
||||
cfg,
|
||||
"fish_speech_single_initial_chunk",
|
||||
False,
|
||||
)
|
||||
use_first_chunk = initial_chunk_size > 0 and initial_chunk_size < steady_chunk_size
|
||||
|
||||
if in_initial_phase:
|
||||
if single_initial_chunk and use_first_chunk:
|
||||
if length <= initial_chunk_size:
|
||||
if not finished and length < initial_chunk_size:
|
||||
return None
|
||||
context_length = length if finished and length < initial_chunk_size else initial_chunk_size
|
||||
else:
|
||||
adjusted = length - initial_chunk_size
|
||||
if adjusted <= 0:
|
||||
return None
|
||||
if not finished and adjusted % steady_chunk_size != 0:
|
||||
return None
|
||||
chunk_length = adjusted % steady_chunk_size
|
||||
context_length = chunk_length if chunk_length != 0 else steady_chunk_size
|
||||
end_index = min(length, left_context_size_config + context_length)
|
||||
left_context_size = max(0, int(end_index - context_length))
|
||||
window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:]
|
||||
elif initial_chunk_size > 0 and length <= chunk_size:
|
||||
already_sent = transfer_manager.put_req_chunk[request_id] * initial_chunk_size
|
||||
pending = length - already_sent
|
||||
if pending <= 0:
|
||||
@@ -134,17 +240,23 @@ def slow_ar_to_dac_decoder_async_chunk(
|
||||
else:
|
||||
initial_coverage = (chunk_size // initial_chunk_size) * initial_chunk_size if initial_chunk_size > 0 else 0
|
||||
adjusted = length - initial_coverage
|
||||
chunk_length = adjusted % chunk_size
|
||||
chunk_length = adjusted % steady_chunk_size
|
||||
if chunk_length != 0 and not finished:
|
||||
return None
|
||||
context_length = chunk_length if chunk_length != 0 else chunk_size
|
||||
context_length = chunk_length if chunk_length != 0 else steady_chunk_size
|
||||
end_index = min(length, left_context_size_config + context_length)
|
||||
left_context_size = max(0, int(end_index - context_length))
|
||||
window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:]
|
||||
|
||||
# Pack into codebook-major flat codes.
|
||||
# Pack into codebook-major codes. The tensor path avoids expanding codec
|
||||
# indices into Python ints across the connector boundary; Stage1 schedules
|
||||
# a single placeholder token and consumes the real codes from runtime info.
|
||||
stacked_frames = torch.stack(window_frames, dim=0)
|
||||
code_predictor_codes = stacked_frames.transpose(0, 1).reshape(-1)
|
||||
codes_qf = stacked_frames.transpose(0, 1).contiguous()
|
||||
if _use_tensor_code_payload(transfer_manager):
|
||||
code_predictor_codes = codes_qf
|
||||
else:
|
||||
code_predictor_codes = codes_qf.reshape(-1)
|
||||
|
||||
return OmniPayloadStruct(
|
||||
codes=CodesStruct(audio=code_predictor_codes),
|
||||
|
||||
@@ -509,6 +509,14 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
|
||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
self._maybe_attach_attention_metadata_extensions(
|
||||
attn_metadata=attn_metadata,
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
max_query_len=max_num_scheduled_tokens,
|
||||
pad_attn=pad_attn,
|
||||
num_scheduled_tokens_np=num_scheduled_tokens_np,
|
||||
)
|
||||
|
||||
(
|
||||
input_ids,
|
||||
|
||||
@@ -278,6 +278,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
cascade_attn_prefix_lens=cascade_attn_prefix_lens,
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
)
|
||||
self._maybe_attach_attention_metadata_extensions(
|
||||
attn_metadata=attn_metadata,
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
max_query_len=max_num_scheduled_tokens,
|
||||
pad_attn=pad_attn,
|
||||
num_scheduled_tokens_np=num_scheduled_tokens_np,
|
||||
)
|
||||
|
||||
(
|
||||
input_ids,
|
||||
@@ -730,6 +738,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
use_spec_decode=self.speculative_config is not None,
|
||||
)
|
||||
self._maybe_attach_attention_metadata_extensions(
|
||||
attn_metadata=attn_metadata,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
max_query_len=max_query_len,
|
||||
pad_attn=True,
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
self.lora_config,
|
||||
|
||||
@@ -96,6 +96,7 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
@instrument(span_name="Loading (GPU)")
|
||||
def load_model(self, *args, **kwargs) -> None:
|
||||
super().load_model(*args, **kwargs)
|
||||
self._prewarm_attention_capture_workspaces()
|
||||
|
||||
# TODO move this model specific logic to a separate class
|
||||
# TTS model IS the talker (no .talker sub-attr); use getattr to support both Omni and TTS.
|
||||
@@ -122,6 +123,45 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
self.last_talker_hidden = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False)
|
||||
self.text_step = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False)
|
||||
|
||||
def _prewarm_attention_capture_workspaces(self) -> None:
|
||||
capture_sizes = getattr(self.compilation_config, "cudagraph_capture_sizes", None)
|
||||
if not capture_sizes:
|
||||
return
|
||||
from vllm_omni.attention.fish_kvcache_backend import prewarm_fish_kvcache_attn_capture_workspaces
|
||||
|
||||
prewarm_fish_kvcache_attn_capture_workspaces(
|
||||
model_config=self.model_config,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
capture_sizes=capture_sizes,
|
||||
)
|
||||
|
||||
def _maybe_attach_attention_metadata_extensions(
|
||||
self,
|
||||
*,
|
||||
attn_metadata: Any,
|
||||
num_reqs: int,
|
||||
num_reqs_padded: int,
|
||||
max_query_len: int,
|
||||
pad_attn: bool,
|
||||
for_cudagraph_capture: bool = False,
|
||||
num_scheduled_tokens_np: np.ndarray | None = None,
|
||||
) -> None:
|
||||
from vllm_omni.attention.fish_kvcache_backend import maybe_attach_fish_kvcache_seq_lens_upper_bound
|
||||
|
||||
maybe_attach_fish_kvcache_seq_lens_upper_bound(
|
||||
model_config=self.model_config,
|
||||
attn_metadata=attn_metadata,
|
||||
input_batch=self.input_batch,
|
||||
optimistic_seq_lens_cpu=self.optimistic_seq_lens_cpu,
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
max_query_len=max_query_len,
|
||||
pad_attn=pad_attn,
|
||||
for_cudagraph_capture=for_cudagraph_capture,
|
||||
num_scheduled_tokens_np=num_scheduled_tokens_np,
|
||||
)
|
||||
|
||||
def _build_model_sampler_output_token_ids(self) -> list[list[int]]:
|
||||
"""Build decoded-token history for ``prefer_model_sampler`` models.
|
||||
|
||||
@@ -898,6 +938,14 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
slot_mappings=slot_mappings_by_group,
|
||||
use_spec_decode=self.speculative_config is not None,
|
||||
)
|
||||
self._maybe_attach_attention_metadata_extensions(
|
||||
attn_metadata=attn_metadata,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
max_query_len=max_query_len,
|
||||
pad_attn=True,
|
||||
for_cudagraph_capture=is_graph_capturing,
|
||||
)
|
||||
|
||||
with self.maybe_dummy_run_with_lora(
|
||||
self.lora_config,
|
||||
|
||||
Reference in new issue
Block a user