From d37912b07164356ad132fb7243dea5430648e33c Mon Sep 17 00:00:00 2001 From: Sy03 <1370724210@qq.com> Date: Thu, 28 May 2026 02:25:49 +0800 Subject: [PATCH] [TTS][Perf] Optimize Fish Speech S2 Pro high-concurrency serving (#3773) Signed-off-by: Sy03 <1370724210@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> --- .../online_serving/text_to_speech/README.md | 20 + recipes/fishaudio/Fish-Speech-S2-Pro.md | 20 + tests/attention/test_fish_kvcache_attn.py | 1137 +++++++++++++++++ .../omni_connectors/test_basic_connectors.py | 14 + .../test_chunk_transfer_adapter.py | 77 +- ...est_fish_speech_fast_ar_runtime_buffers.py | 157 +++ .../test_fish_speech_regressions.py | 161 +++ .../test_fish_speech_async_chunk.py | 229 ++++ vllm_omni/attention/__init__.py | 2 + vllm_omni/attention/fish_kvcache_attn.py | 191 +++ vllm_omni/attention/fish_kvcache_backend.py | 406 ++++++ vllm_omni/attention/fish_kvcache_triton.py | 359 ++++++ ..._qwen3_omni_high_concurrency_dual_gpu.yaml | 68 + ...wen3_omni_high_concurrency_single_gpu.yaml | 68 + .../chunk_transfer_adapter.py | 51 +- .../omni_connectors/utils/serialization.py | 4 +- .../fish_speech/fish_speech_dac_decoder.py | 196 ++- .../models/fish_speech/fish_speech_fast_ar.py | 148 ++- .../models/fish_speech/fish_speech_slow_ar.py | 19 +- .../stage_input_processors/fish_speech.py | 146 ++- vllm_omni/worker/gpu_ar_model_runner.py | 8 + .../worker/gpu_generation_model_runner.py | 16 + vllm_omni/worker/gpu_model_runner.py | 48 + 23 files changed, 3453 insertions(+), 92 deletions(-) create mode 100644 tests/attention/test_fish_kvcache_attn.py create mode 100644 tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py create mode 100644 tests/model_executor/stage_input_processors/test_fish_speech_async_chunk.py create mode 100644 vllm_omni/attention/__init__.py create mode 100644 vllm_omni/attention/fish_kvcache_attn.py create mode 100644 vllm_omni/attention/fish_kvcache_backend.py create mode 100644 vllm_omni/attention/fish_kvcache_triton.py create mode 100644 vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml create mode 100644 vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index 922156d4c..cc4afbaeb 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -143,6 +143,26 @@ bash examples/online_serving/text_to_speech/glm_tts/run_gradio_demo.sh pip install fish-speech ``` +### Kvcache attention fast path + +Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by +default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or +`VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be +installed. + +```bash +# Verify fast path availability. +python - <<'PY' +from vllm_omni.attention import fish_kvcache_attn + +print(fish_kvcache_attn.is_available()) +print(fish_kvcache_attn.load_error()) +PY + +# Optional: disable the runtime fast path. +export VLLM_OMNI_FISH_KVCACHE_ATTN=0 +``` + ### Launch ```bash vllm serve fishaudio/s2-pro --omni --port 8091 diff --git a/recipes/fishaudio/Fish-Speech-S2-Pro.md b/recipes/fishaudio/Fish-Speech-S2-Pro.md index 2a88bd169..dca9f537e 100644 --- a/recipes/fishaudio/Fish-Speech-S2-Pro.md +++ b/recipes/fishaudio/Fish-Speech-S2-Pro.md @@ -50,6 +50,26 @@ fi pip install fish-speech ``` +#### Kvcache attention fast path + +Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by +default on CUDA builds. Set `VLLM_OMNI_FISH_KVCACHE_ATTN=0` to disable it, or +`VLLM_OMNI_FISH_KVCACHE_ATTN=required` to fail fast if the fast path cannot be +installed. + +```bash +# Verify fast path availability. +python - <<'PY' +from vllm_omni.attention import fish_kvcache_attn + +print(fish_kvcache_attn.is_available()) +print(fish_kvcache_attn.load_error()) +PY + +# Optional: disable the runtime fast path. +export VLLM_OMNI_FISH_KVCACHE_ATTN=0 +``` + #### Command ```bash diff --git a/tests/attention/test_fish_kvcache_attn.py b/tests/attention/test_fish_kvcache_attn.py new file mode 100644 index 000000000..89b43e627 --- /dev/null +++ b/tests/attention/test_fish_kvcache_attn.py @@ -0,0 +1,1137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm_omni.attention import fish_kvcache_attn, fish_kvcache_backend + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _FakeImpl: + dcp_world_size = 1 + alibi_slopes = None + sliding_window = None + scale = 1.0 + + def __init__(self): + self.original_calls = 0 + + def forward( + self, + layer, + query, + key, + value, + kv_cache, + attn_metadata, + output, + output_scale=None, + output_block_scale=None, + ): + del layer, query, key, value, kv_cache, attn_metadata, output_scale, output_block_scale + self.original_calls += 1 + output.fill_(2) + return output + + +class _FakeAttentionLayer: + def __init__(self): + self.impl = _FakeImpl() + + +class _FakeSelfAttn: + def __init__(self): + self.attn = _FakeAttentionLayer() + + +class _FakeLayer: + def __init__(self): + self.self_attn = _FakeSelfAttn() + + +class _FakeModel: + def __init__(self): + self.layers = [_FakeLayer()] + + +_DEFAULT_UPPER_BOUND = object() + + +def _metadata( + *, + batch_size: int = 2, + num_actual_tokens: int | None = None, + max_query_len: int = 1, + max_seq_len: int = 128, + use_cascade: bool = False, + seq_lens_cpu_upper_bound: torch.Tensor | None | object = _DEFAULT_UPPER_BOUND, + seq_lens: torch.Tensor | None = None, +): + if num_actual_tokens is None: + num_actual_tokens = batch_size + if seq_lens is None: + seq_lens = torch.ones((batch_size,), dtype=torch.int32) + if seq_lens_cpu_upper_bound is _DEFAULT_UPPER_BOUND and max_query_len == 1: + seq_lens_cpu_upper_bound = seq_lens.detach().cpu() + elif seq_lens_cpu_upper_bound is _DEFAULT_UPPER_BOUND: + seq_lens_cpu_upper_bound = None + return type( + "Metadata", + (), + { + "use_cascade": use_cascade, + "num_actual_tokens": num_actual_tokens, + "block_table": torch.zeros((batch_size, 8), dtype=torch.int32), + "seq_lens": seq_lens, + "seq_lens_cpu_upper_bound": seq_lens_cpu_upper_bound, + "max_query_len": max_query_len, + "max_seq_len": max_seq_len, + }, + )() + + +def test_fish_kvcache_enabled_by_default(monkeypatch): + monkeypatch.delenv("VLLM_OMNI_FISH_KVCACHE_ATTN", raising=False) + + assert fish_kvcache_attn.is_fish_kvcache_attn_enabled() + assert fish_kvcache_backend._fish_kvcache_enabled() + assert not fish_kvcache_attn.is_fish_kvcache_attn_required() + + +@pytest.mark.parametrize("value", ["", "1", "true", "yes", "on", "required"]) +def test_fish_kvcache_enabled_values_are_consistent(monkeypatch, value): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", value) + assert fish_kvcache_attn.is_fish_kvcache_attn_enabled() + assert fish_kvcache_backend._fish_kvcache_enabled() + assert fish_kvcache_attn.is_fish_kvcache_attn_required() is (value == "required") + + +@pytest.mark.parametrize("value", ["0", "false", "no", "off", "disabled", "disable"]) +def test_fish_kvcache_disabled_values_are_consistent(monkeypatch, value): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", value) + + assert not fish_kvcache_attn.is_fish_kvcache_attn_enabled() + assert not fish_kvcache_backend._fish_kvcache_enabled() + assert not fish_kvcache_attn.is_fish_kvcache_attn_required() + + +@pytest.mark.parametrize("sliding_window", [None, (-1, -1)]) +def test_fish_kvcache_attn_guard_accepts_decode_only_shape(monkeypatch, sliding_window): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_attn, "is_available", lambda: True) + + assert fish_kvcache_attn.can_use_fish_kvcache_attn( + query=torch.empty((4, 32, 128), dtype=torch.float16), + key_cache=torch.empty((8, 16, 8, 128), dtype=torch.float16), + value_cache=torch.empty((8, 16, 8, 128), dtype=torch.float16), + block_table=torch.zeros((4, 8), dtype=torch.int32), + seq_lens=torch.ones((4,), dtype=torch.int32), + max_query_len=1, + max_seq_len=128, + dcp_world_size=1, + use_cascade=False, + alibi_slopes=None, + sliding_window=sliding_window, + ) + + +def test_fish_kvcache_attn_guard_rejects_unsupported_block_size(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_attn, "is_available", lambda: True) + + assert not fish_kvcache_attn.can_use_fish_kvcache_attn( + query=torch.empty((4, 32, 128), dtype=torch.float16), + key_cache=torch.empty((8, 32, 8, 128), dtype=torch.float16), + value_cache=torch.empty((8, 32, 8, 128), dtype=torch.float16), + block_table=torch.zeros((4, 8), dtype=torch.int32), + seq_lens=torch.ones((4,), dtype=torch.int32), + max_query_len=1, + max_seq_len=128, + dcp_world_size=1, + use_cascade=False, + alibi_slopes=None, + sliding_window=None, + ) + + +def test_fish_kvcache_backend_wraps_only_model_instance(monkeypatch): + fish_kvcache_backend.reset_fish_kvcache_attn_stats() + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: True) + + hits = [] + + def fake_decode(query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len): + del key_cache, value_cache, block_table, seq_lens, scale, max_seq_len + hits.append(tuple(query.shape)) + out.fill_(1) + return out + + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + other_impl = _FakeImpl() + + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + assert not hasattr(other_impl, "_fish_kvcache_attn_installed") + + metadata = _metadata() + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + result = model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + metadata, + output, + ) + + assert result is output + assert hits == [(2, 32, 128)] + assert model.layers[0].self_attn.attn.impl.original_calls == 0 + assert torch.equal(output, torch.ones_like(output)) + assert fish_kvcache_backend.get_fish_kvcache_attn_stats()["small_hit_count"] == 1 + + +def test_fish_kvcache_backend_install_is_idempotent(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: False) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + wrapped_forward = model.layers[0].self_attn.attn.impl.forward + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 0 + assert model.layers[0].self_attn.attn.impl.forward is wrapped_forward + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata(), + output, + ) + + assert model.layers[0].self_attn.attn.impl.original_calls == 1 + + +def test_fish_kvcache_backend_falls_back_to_original_forward_on_guard_miss(monkeypatch): + fish_kvcache_backend.reset_fish_kvcache_attn_stats() + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: False) + + decode_calls = [] + + def fake_decode(*args, **kwargs): + del args, kwargs + decode_calls.append(True) + + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + result = model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata(), + output, + ) + + assert result is output + assert model.layers[0].self_attn.attn.impl.original_calls == 1 + assert decode_calls == [] + assert torch.equal(output, torch.full_like(output, 2)) + stats = fish_kvcache_backend.get_fish_kvcache_attn_stats() + assert stats["fallback_count_by_reason"] == {"guard_miss": 1} + + +def test_fish_kvcache_backend_uses_decode_seq_lens_upper_bound(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + guard_max_seq_lens = [] + + def fake_can_use(**kwargs): + guard_max_seq_lens.append(kwargs["max_seq_len"]) + return True + + decode_max_seq_lens = [] + + def fake_decode(query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len): + del query, key_cache, value_cache, block_table, seq_lens, scale + decode_max_seq_lens.append(max_seq_len) + out.fill_(1) + return out + + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", fake_can_use) + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + max_query_len=1, + max_seq_len=16384, + seq_lens_cpu_upper_bound=torch.tensor([512, 768], dtype=torch.int32), + ), + output, + ) + + assert guard_max_seq_lens == [1024] + assert decode_max_seq_lens == [1024] + + +def test_fish_kvcache_backend_uses_cpu_upper_bound_above_metadata_max(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + guard_max_seq_lens = [] + + def fake_can_use(**kwargs): + guard_max_seq_lens.append(kwargs["max_seq_len"]) + return True + + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", fake_can_use) + monkeypatch.setattr( + fish_kvcache_backend, + "fish_decode_kvcache_attn", + lambda query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len: out.fill_(1), + ) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + max_query_len=1, + max_seq_len=64, + seq_lens_cpu_upper_bound=torch.tensor([1024, 1024], dtype=torch.int32), + ), + output, + ) + + assert guard_max_seq_lens == [1024] + + +def test_fish_kvcache_backend_slices_upper_bound_by_active_batch(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + guard_max_seq_lens = [] + + def fake_can_use(**kwargs): + guard_max_seq_lens.append(kwargs["max_seq_len"]) + return True + + def fake_decode(query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len): + del query, key_cache, value_cache, block_table, seq_lens, scale, max_seq_len + out.fill_(1) + return out + + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", fake_can_use) + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + batch_size=4, + num_actual_tokens=2, + max_query_len=1, + max_seq_len=16384, + seq_lens_cpu_upper_bound=torch.tensor([512, 768, 4096, 4096], dtype=torch.int32), + ), + output, + ) + + assert guard_max_seq_lens == [1024] + + +def test_fish_kvcache_backend_falls_back_on_short_upper_bound_shape(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + guard_max_seq_lens = [] + + def fake_can_use(**kwargs): + guard_max_seq_lens.append(kwargs["max_seq_len"]) + return True + + def fake_decode(query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len): + del query, key_cache, value_cache, block_table, seq_lens, scale, max_seq_len + out.fill_(1) + return out + + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", fake_can_use) + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + result = model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + batch_size=2, + max_query_len=1, + max_seq_len=16384, + seq_lens_cpu_upper_bound=torch.tensor([512], dtype=torch.int32), + ), + output, + ) + + assert result is output + assert guard_max_seq_lens == [] + assert model.layers[0].self_attn.attn.impl.original_calls == 1 + assert torch.equal(output, torch.full_like(output, 2)) + + +def test_fish_kvcache_backend_falls_back_on_non_cpu_upper_bound(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + guard_max_seq_lens = [] + + def fake_can_use(**kwargs): + guard_max_seq_lens.append(kwargs["max_seq_len"]) + return True + + def fake_decode(query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len): + del query, key_cache, value_cache, block_table, seq_lens, scale, max_seq_len + out.fill_(1) + return out + + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", fake_can_use) + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + result = model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + batch_size=2, + max_query_len=1, + max_seq_len=16384, + seq_lens_cpu_upper_bound=torch.empty((2,), device="meta", dtype=torch.int32), + ), + output, + ) + + assert result is output + assert guard_max_seq_lens == [] + assert model.layers[0].self_attn.attn.impl.original_calls == 1 + assert torch.equal(output, torch.full_like(output, 2)) + + +def test_fish_kvcache_backend_required_rejects_non_cpu_upper_bound(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "required") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: True) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + with pytest.raises(RuntimeError, match="must be a CPU tensor"): + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + batch_size=2, + max_query_len=1, + max_seq_len=16384, + seq_lens_cpu_upper_bound=torch.empty((2,), device="meta", dtype=torch.int32), + ), + output, + ) + + +def test_fish_kvcache_backend_required_rejects_underestimated_upper_bound(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "required") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: True) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + with pytest.raises(RuntimeError, match="underestimates"): + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata( + max_query_len=1, + max_seq_len=16384, + seq_lens=torch.tensor([512, 2048], dtype=torch.int32), + seq_lens_cpu_upper_bound=torch.tensor([512, 768], dtype=torch.int32), + ), + output, + ) + + +def test_fish_kvcache_backend_required_rejects_missing_upper_bound(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "required") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: True) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + with pytest.raises(RuntimeError, match="requires seq_lens_cpu_upper_bound"): + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata(max_query_len=1, max_seq_len=16384, seq_lens_cpu_upper_bound=None), + output, + ) + + +def test_fish_kvcache_backend_required_rejects_zero_installed_layers(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "required") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + with pytest.raises(RuntimeError, match="installed 0 attention layers"): + fish_kvcache_backend.install_fish_kvcache_attn_backend(type("EmptyModel", (), {"layers": []})()) + + +def test_fish_kvcache_backend_required_raises_on_guard_miss(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "required") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: False) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + with pytest.raises(RuntimeError, match="required"): + model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata(), + output, + ) + + +def test_fish_kvcache_backend_required_allows_prefill_fallback(monkeypatch): + fish_kvcache_backend.reset_fish_kvcache_attn_stats() + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "required") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", lambda **_: False) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + result = model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata(max_query_len=2, max_seq_len=2), + output, + ) + + assert result is output + assert model.layers[0].self_attn.attn.impl.original_calls == 1 + assert torch.equal(output, torch.full_like(output, 2)) + stats = fish_kvcache_backend.get_fish_kvcache_attn_stats() + assert stats["fallback_count_by_reason"] == {"non_decode": 1} + + +def test_fish_kvcache_backend_falls_back_without_cpu_upper_bound(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None) + + guard_max_seq_lens = [] + + def fake_can_use(**kwargs): + guard_max_seq_lens.append(kwargs["max_seq_len"]) + return True + + decode_max_seq_lens = [] + + def fake_decode(query, key_cache, value_cache, block_table, seq_lens, out, *, scale, max_seq_len): + del query, key_cache, value_cache, block_table, seq_lens, scale + decode_max_seq_lens.append(max_seq_len) + out.fill_(1) + return out + + monkeypatch.setattr(fish_kvcache_backend, "can_use_fish_kvcache_attn", fake_can_use) + monkeypatch.setattr(fish_kvcache_backend, "fish_decode_kvcache_attn", fake_decode) + + model = _FakeModel() + assert fish_kvcache_backend.install_fish_kvcache_attn_backend(model) == 1 + + query = torch.zeros((2, 32, 128), dtype=torch.float16) + output = torch.zeros_like(query) + kv_cache = torch.zeros((2, 8, 16, 8, 128), dtype=torch.float16) + + result = model.layers[0].self_attn.attn.impl.forward( + None, + query, + None, + None, + kv_cache, + _metadata(max_query_len=1, max_seq_len=16384, seq_lens_cpu_upper_bound=None), + output, + ) + + assert result is output + assert guard_max_seq_lens == [] + assert decode_max_seq_lens == [] + assert model.layers[0].self_attn.attn.impl.original_calls == 1 + assert torch.equal(output, torch.full_like(output, 2)) + + +def test_fish_kvcache_attn_guard_accepts_decode_metadata_above_small_cap(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_attn, "is_available", lambda: True) + + assert fish_kvcache_attn.can_use_fish_kvcache_attn( + query=torch.empty((4, 32, 128), dtype=torch.float16), + key_cache=torch.empty((1024, 16, 8, 128), dtype=torch.float16), + value_cache=torch.empty((1024, 16, 8, 128), dtype=torch.float16), + block_table=torch.zeros((4, 1024), dtype=torch.int32), + seq_lens=torch.ones((4,), dtype=torch.int32), + max_query_len=1, + max_seq_len=16384, + dcp_world_size=1, + use_cascade=False, + alibi_slopes=None, + sliding_window=None, + ) + + +def test_fish_kvcache_attn_guard_rejects_seq_len_above_block_table_capacity(monkeypatch): + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_attn, "is_available", lambda: True) + + assert not fish_kvcache_attn.can_use_fish_kvcache_attn( + query=torch.empty((4, 32, 128), dtype=torch.float16), + key_cache=torch.empty((8, 16, 8, 128), dtype=torch.float16), + value_cache=torch.empty((8, 16, 8, 128), dtype=torch.float16), + block_table=torch.zeros((4, 8), dtype=torch.int32), + seq_lens=torch.ones((4,), dtype=torch.int32), + max_query_len=1, + max_seq_len=129, + dcp_world_size=1, + use_cascade=False, + alibi_slopes=None, + sliding_window=None, + ) + + +def test_fish_kvcache_decode_reuses_long_workspace(monkeypatch): + fish_kvcache_attn._WORKSPACE_CACHE.clear() + calls = [] + + def fake_decode( + query, + key_cache, + value_cache, + block_table, + seq_lens, + out, + scale, + max_seq_len, + small_path_max_seq_len, + long_split_tokens, + partial_m, + partial_l, + partial_acc, + ): + del query, key_cache, value_cache, block_table, seq_lens, scale, max_seq_len + del small_path_max_seq_len, long_split_tokens + calls.append((partial_m.data_ptr(), partial_l.data_ptr(), partial_acc.data_ptr(), tuple(partial_acc.shape))) + return out + + monkeypatch.setattr( + fish_kvcache_attn._triton_backend(), + "fish_decode_kvcache_attn_triton", + fake_decode, + ) + + q = torch.zeros((2, 4, 128), dtype=torch.float16) + k_cache = torch.zeros((128, 16, 2, 128), dtype=torch.float16) + v_cache = torch.zeros_like(k_cache) + block_table = torch.zeros((2, 128), dtype=torch.int32) + seq_lens = torch.full((2,), 2048, dtype=torch.int32) + out = torch.empty_like(q) + + for _ in range(2): + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=128**-0.5, + max_seq_len=2048, + ) + + assert len(calls) == 2 + assert calls[0] == calls[1] + assert calls[0][3] == (2, 2, 4, 128) + + +def test_fish_kvcache_decode_workspace_miss_rejects_cuda_graph_capture(monkeypatch): + fish_kvcache_attn._WORKSPACE_CACHE.clear() + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + + q = torch.zeros((2, 4, 128), dtype=torch.float16) + + with pytest.raises(RuntimeError, match="workspace was not prewarmed"): + fish_kvcache_attn._get_decode_workspace(q, 2048) + + +def test_fish_kvcache_backend_prewarms_capture_workspaces(monkeypatch): + fish_kvcache_attn._WORKSPACE_CACHE.clear() + monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1") + monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True) + + text_config = type( + "TextConfig", + (), + {"num_attention_heads": 4, "head_dim": 128}, + )() + hf_config = type("HFConfig", (), {"text_config": text_config})() + model_config = type( + "ModelConfig", + (), + { + "hf_config": hf_config, + "max_model_len": 2048, + "model_arch": "FishSpeechSlowARForConditionalGeneration", + }, + )() + + assert ( + fish_kvcache_backend.prewarm_fish_kvcache_attn_capture_workspaces( + model_config=model_config, + device=torch.device("cpu"), + dtype=torch.float16, + capture_sizes=[1, 4], + ) + == 3 + ) + + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + q = torch.zeros((4, 4, 128), dtype=torch.float16) + partial_m, partial_l, partial_acc = fish_kvcache_attn._get_decode_workspace(q, 2048) + + assert tuple(partial_m.shape) == (2, 4, 4) + assert tuple(partial_l.shape) == (2, 4, 4) + assert tuple(partial_acc.shape) == (2, 4, 4, 128) + + +def _reference_decode_attention( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + block_table: torch.Tensor, + seq_lens: torch.Tensor, + scale: float, +) -> torch.Tensor: + block_size = int(k_cache.shape[1]) + num_q_heads = int(q.shape[1]) + num_kv_heads = int(k_cache.shape[2]) + outputs = [] + for batch_idx in range(q.shape[0]): + seq_len = int(seq_lens[batch_idx].item()) + if seq_len <= 0: + outputs.append(torch.zeros_like(q[batch_idx], dtype=torch.float32)) + continue + k_rows = [] + v_rows = [] + for logical_block in range((seq_len + block_size - 1) // block_size): + physical_block = int(block_table[batch_idx, logical_block].item()) + start = logical_block * block_size + take = min(block_size, seq_len - start) + k_rows.append(k_cache[physical_block, :take]) + v_rows.append(v_cache[physical_block, :take]) + k = torch.cat(k_rows, dim=0).to(torch.float32) + v = torch.cat(v_rows, dim=0).to(torch.float32) + per_head = [] + for q_head in range(num_q_heads): + kv_head = q_head // (num_q_heads // num_kv_heads) + scores = torch.matmul(k[:, kv_head, :], q[batch_idx, q_head].to(torch.float32)) * scale + weights = torch.softmax(scores, dim=0) + per_head.append(torch.matmul(weights, v[:, kv_head, :])) + outputs.append(torch.stack(per_head, dim=0)) + return torch.stack(outputs, dim=0).to(q.dtype) + + +@pytest.mark.parametrize("seq_len", [64, 128, 256, 512, 1024, 1025, 2048]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("batch_size", [1, 3]) +@pytest.mark.skipif( + not torch.cuda.is_available() or not fish_kvcache_attn.is_available(), + reason="Fish kvcache Triton attention is not available", +) +def test_fish_kvcache_native_op_matches_reference(seq_len, dtype, batch_size): + torch.manual_seed(0) + device = torch.device("cuda") + block_size = 16 + num_q_heads = 4 + num_kv_heads = 2 + head_dim = 128 + max_blocks = (seq_len + block_size - 1) // block_size + num_blocks = batch_size * max_blocks + 3 + scale = head_dim**-0.5 + + q = torch.randn((batch_size, num_q_heads, head_dim), device=device, dtype=dtype) + k_cache = torch.randn((num_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + v_cache = torch.randn((num_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + block_table = torch.empty((batch_size, max_blocks), device=device, dtype=torch.int32) + for batch_idx in range(batch_size): + pages = torch.arange(batch_idx * max_blocks, (batch_idx + 1) * max_blocks, device=device, dtype=torch.int32) + block_table[batch_idx] = torch.flip(pages, dims=[0]) + seq_lens = torch.full((batch_size,), seq_len, device=device, dtype=torch.int32) + + out = torch.empty_like(q) + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=scale, + max_seq_len=seq_len, + ) + + expected = _reference_decode_attention(q, k_cache, v_cache, block_table, seq_lens, scale) + torch.testing.assert_close(out, expected, atol=3e-2, rtol=3e-2) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or not fish_kvcache_attn.is_available(), + reason="Fish kvcache Triton attention is not available", +) +def test_fish_kvcache_native_op_zero_seq_len_writes_zero(): + device = torch.device("cuda") + dtype = torch.float16 + block_size = 16 + num_q_heads = 4 + num_kv_heads = 2 + head_dim = 128 + max_seq_len = 512 + max_blocks = max_seq_len // block_size + scale = head_dim**-0.5 + + q = torch.randn((1, num_q_heads, head_dim), device=device, dtype=dtype) + k_cache = torch.randn((max_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + v_cache = torch.randn_like(k_cache) + block_table = torch.arange(max_blocks, device=device, dtype=torch.int32).reshape(1, max_blocks) + seq_lens = torch.zeros((1,), device=device, dtype=torch.int32) + out = torch.full_like(q, float("nan")) + + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=scale, + max_seq_len=max_seq_len, + ) + + assert torch.isfinite(out).all() + assert torch.equal(out, torch.zeros_like(out)) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or not fish_kvcache_attn.is_available(), + reason="Fish kvcache Triton attention is not available", +) +def test_fish_kvcache_native_op_handles_mixed_short_and_long_seq_lens(): + torch.manual_seed(0) + device = torch.device("cuda") + dtype = torch.float16 + block_size = 16 + num_q_heads = 4 + num_kv_heads = 2 + head_dim = 128 + seq_lens = torch.tensor([512, 2048, 1025], device=device, dtype=torch.int32) + max_seq_len = int(seq_lens.max().item()) + batch_size = int(seq_lens.numel()) + max_blocks = (max_seq_len + block_size - 1) // block_size + num_blocks = batch_size * max_blocks + scale = head_dim**-0.5 + + q = torch.randn((batch_size, num_q_heads, head_dim), device=device, dtype=dtype) + k_cache = torch.randn((num_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + v_cache = torch.randn((num_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + block_table = torch.empty((batch_size, max_blocks), device=device, dtype=torch.int32) + for batch_idx in range(batch_size): + pages = torch.arange(batch_idx * max_blocks, (batch_idx + 1) * max_blocks, device=device, dtype=torch.int32) + block_table[batch_idx] = pages + + out = torch.empty_like(q) + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=scale, + max_seq_len=max_seq_len, + ) + + expected = _reference_decode_attention(q, k_cache, v_cache, block_table, seq_lens, scale) + torch.testing.assert_close(out, expected, atol=3e-2, rtol=3e-2) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or not fish_kvcache_attn.is_available(), + reason="Fish kvcache Triton attention is not available", +) +def test_fish_kvcache_native_op_handles_zero_short_and_long_seq_lens(): + torch.manual_seed(0) + device = torch.device("cuda") + dtype = torch.float16 + block_size = 16 + num_q_heads = 4 + num_kv_heads = 2 + head_dim = 128 + seq_lens = torch.tensor([0, 512, 2048], device=device, dtype=torch.int32) + max_seq_len = int(seq_lens.max().item()) + batch_size = int(seq_lens.numel()) + max_blocks = (max_seq_len + block_size - 1) // block_size + num_blocks = batch_size * max_blocks + scale = head_dim**-0.5 + + q = torch.randn((batch_size, num_q_heads, head_dim), device=device, dtype=dtype) + k_cache = torch.randn((num_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + v_cache = torch.randn((num_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + block_table = torch.empty((batch_size, max_blocks), device=device, dtype=torch.int32) + for batch_idx in range(batch_size): + pages = torch.arange(batch_idx * max_blocks, (batch_idx + 1) * max_blocks, device=device, dtype=torch.int32) + block_table[batch_idx] = pages + out = torch.full_like(q, float("nan")) + + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=scale, + max_seq_len=max_seq_len, + ) + + expected = _reference_decode_attention(q, k_cache, v_cache, block_table, seq_lens, scale) + assert torch.isfinite(out).all() + torch.testing.assert_close(out, expected, atol=3e-2, rtol=3e-2) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or not fish_kvcache_attn.is_available(), + reason="Fish kvcache Triton attention is not available", +) +def test_fish_kvcache_native_op_can_be_captured_by_cuda_graph(): + device = torch.device("cuda") + dtype = torch.float16 + q = torch.randn((2, 4, 128), device=device, dtype=dtype) + k_cache = torch.randn((8, 16, 2, 128), device=device, dtype=dtype) + v_cache = torch.randn((8, 16, 2, 128), device=device, dtype=dtype) + block_table = torch.arange(8, device=device, dtype=torch.int32).reshape(2, 4) + seq_lens = torch.full((2,), 64, device=device, dtype=torch.int32) + out = torch.empty_like(q) + expected = torch.empty_like(q) + + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + expected, + scale=128**-0.5, + max_seq_len=64, + ) + torch.accelerator.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=128**-0.5, + max_seq_len=64, + ) + graph.replay() + torch.accelerator.synchronize() + + torch.testing.assert_close(out, expected, atol=3e-2, rtol=3e-2) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or not fish_kvcache_attn.is_available(), + reason="Fish kvcache Triton attention is not available", +) +def test_fish_kvcache_native_long_op_can_be_captured_by_cuda_graph(): + device = torch.device("cuda") + dtype = torch.float16 + seq_len = 2048 + block_size = 16 + num_q_heads = 4 + num_kv_heads = 2 + head_dim = 128 + max_blocks = (seq_len + block_size - 1) // block_size + + q = torch.randn((2, num_q_heads, head_dim), device=device, dtype=dtype) + k_cache = torch.randn((2 * max_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + v_cache = torch.randn((2 * max_blocks, block_size, num_kv_heads, head_dim), device=device, dtype=dtype) + block_table = torch.arange(2 * max_blocks, device=device, dtype=torch.int32).reshape(2, max_blocks) + seq_lens = torch.full((2,), seq_len, device=device, dtype=torch.int32) + out = torch.empty_like(q) + expected = torch.empty_like(q) + + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + expected, + scale=head_dim**-0.5, + max_seq_len=seq_len, + ) + torch.accelerator.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + fish_kvcache_attn.fish_decode_kvcache_attn( + q, + k_cache, + v_cache, + block_table, + seq_lens, + out, + scale=head_dim**-0.5, + max_seq_len=seq_len, + ) + graph.replay() + torch.accelerator.synchronize() + + torch.testing.assert_close(out, expected, atol=3e-2, rtol=3e-2) diff --git a/tests/distributed/omni_connectors/test_basic_connectors.py b/tests/distributed/omni_connectors/test_basic_connectors.py index 662d41fe0..4d0661be8 100644 --- a/tests/distributed/omni_connectors/test_basic_connectors.py +++ b/tests/distributed/omni_connectors/test_basic_connectors.py @@ -33,6 +33,20 @@ def test_tensor_serialization(): assert torch.equal(tensor, deserialized) +def test_noncontiguous_tensor_serialization(): + """Test torch.Tensor serialization for strided views.""" + import torch + + tensor = torch.arange(40, dtype=torch.long).reshape(4, 10).transpose(0, 1) + assert not tensor.is_contiguous() + assert tensor.stride(-1) != 1 + + serialized = OmniSerializer.serialize({"codes": tensor}) + deserialized = OmniSerializer.deserialize(serialized) + + assert torch.equal(tensor, deserialized["codes"]) + + def test_ndarray_serialization(): """Test numpy.ndarray serialization.""" import numpy as np diff --git a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py index 9f10aea0c..998a8ebb9 100644 --- a/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py +++ b/tests/distributed/omni_connectors/test_chunk_transfer_adapter.py @@ -37,6 +37,7 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None) status=status, prompt_token_ids=[], num_computed_tokens=0, + num_output_placeholders=0, additional_information=None, is_finished=lambda: status == RequestStatus.FINISHED_STOPPED, ) @@ -44,9 +45,16 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None) @pytest.fixture def build_adapter(monkeypatch, mocker: MockerFixture): - def _build(*, stage_id: int = 1, model_mode: str = "ar", max_num_seqs: int = 2): + def _build( + *, + stage_id: int = 1, + model_mode: str = "ar", + max_num_seqs: int = 2, + connector_extra: dict | None = None, + ): connector = mocker.MagicMock() connector.stage_id = stage_id + connector.config = {"extra": connector_extra or {}} connector.get.return_value = None connector.put.return_value = (True, 1, {}) @@ -127,6 +135,61 @@ def test_load_poll(build_adapter): assert "req-1" not in adapter._pending_load_reqs +def test_load_poll_generation_tensor_codes_use_placeholder_prompt(build_adapter): + adapter, connector = build_adapter(stage_id=1, model_mode="generation") + request = _req("req-tensor", RequestStatus.WAITING, external_req_id="external-tensor") + + codes = torch.tensor([[1, 2], [3, 4]], dtype=torch.long) + payload: OmniPayload = { + "codes": {"audio": codes}, + "meta": { + "left_context_size": 1, + "finished": torch.tensor(False, dtype=torch.bool), + }, + } + connector.get.return_value = (payload, 16) + + adapter._poll_single_request(request) + + assert request.prompt_token_ids == [0] + assert request.num_computed_tokens == 0 + assert torch.equal(request.additional_information["codes"]["audio"], codes) + assert request.additional_information["meta"]["left_context_size"] == 1 + assert "finished" not in request.additional_information["meta"] + assert "req-tensor" in adapter._finished_load_reqs + + +def test_load_poll_generation_empty_nonterminal_chunk_keeps_polling(build_adapter): + adapter, connector = build_adapter(stage_id=1, model_mode="generation") + request = _req("req-empty-tensor", RequestStatus.WAITING, external_req_id="external-empty") + + empty_payload: OmniPayload = { + "codes": {"audio": torch.empty((4, 0), dtype=torch.long)}, + "meta": { + "left_context_size": 0, + "finished": torch.tensor(False, dtype=torch.bool), + }, + } + ready_payload: OmniPayload = { + "codes": {"audio": torch.tensor([[1, 2]], dtype=torch.long)}, + "meta": { + "left_context_size": 0, + "finished": torch.tensor(False, dtype=torch.bool), + }, + } + connector.get.side_effect = [(empty_payload, 16), (ready_payload, 16)] + + assert adapter._poll_single_request(request) is False + assert request.request_id not in adapter._finished_load_reqs + assert request.request_id not in adapter.requests_with_ready_chunks + assert adapter.get_req_chunk[request.request_id] == 1 + + assert adapter._poll_single_request(request) is True + assert request.request_id in adapter._finished_load_reqs + assert torch.equal(request.additional_information["codes"]["audio"], ready_payload["codes"]["audio"]) + assert adapter.get_req_chunk[request.request_id] == 2 + + def test_save_async(build_adapter): adapter, _ = build_adapter(stage_id=1) request = _req("req-1", RequestStatus.WAITING, external_req_id="external-1") @@ -140,6 +203,18 @@ def test_save_async(build_adapter): assert task["is_finished"] is False +def test_save_async_uses_confirmed_tokens_for_async_scheduler_watermark(build_adapter): + adapter, _ = build_adapter(stage_id=1) + request = _req("req-async", RequestStatus.WAITING, external_req_id="external-async") + request.num_computed_tokens = 10 + request.num_output_placeholders = 2 + + adapter.save_async(pooling_output=None, request=request) + + assert adapter.requests_num_chunks_sent["external-async"] == 8 + assert len(adapter._pending_save_reqs) == 1 + + def test_send_single_request_struct_without_meta_does_not_crash(build_adapter, monkeypatch): """Producer may return a struct with ``meta=None`` (e.g. payload that carries only ``embed`` or ``codes``). The sender's ``meta is not None`` diff --git a/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py b/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py new file mode 100644 index 000000000..41a29ca63 --- /dev/null +++ b/tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +import torch.nn as nn + +from vllm_omni.model_executor.models.fish_speech.configuration_fish_speech import ( + FishSpeechFastARConfig, + FishSpeechSlowARConfig, +) +from vllm_omni.model_executor.models.fish_speech.fish_speech_fast_ar import FishSpeechFastAR +from vllm_omni.model_executor.models.fish_speech.fish_speech_slow_ar import ( + FishSpeechSlowARForConditionalGeneration, +) + + +def test_fast_ar_reuses_dense_position_id_buffer(monkeypatch): + fast_config = FishSpeechFastARConfig( + vocab_size=16, + num_codebooks=4, + dim=8, + n_head=2, + n_local_heads=1, + head_dim=4, + n_layer=1, + intermediate_size=16, + max_seq_len=5, + ) + slow_config = FishSpeechSlowARConfig( + vocab_size=32, + dim=8, + n_head=2, + n_local_heads=1, + head_dim=4, + n_layer=1, + intermediate_size=16, + codebook_size=16, + num_codebooks=4, + semantic_begin_id=4, + semantic_end_id=19, + ) + fast_ar = object.__new__(FishSpeechFastAR) + nn.Module.__init__(fast_ar) + fast_ar.config = fast_config + fast_ar.slow_ar_config = slow_config + fast_ar.fast_project_in = nn.Identity() + fast_ar.fast_embeddings = nn.Embedding(fast_config.vocab_size, fast_config.hidden_size) + fast_ar.fast_output = nn.Linear(fast_config.hidden_size, fast_config.vocab_size, bias=False) + fast_ar.fast_norm = nn.Identity() + fast_ar._num_codebooks = fast_config.num_codebooks + fast_ar._fast_dim = fast_config.hidden_size + fast_ar._embed_buf = None + fast_ar._pos_ids = None + fast_ar._k_cache = None + fast_ar._v_cache = None + fast_ar._compiled_model_fwd = None + fast_ar._compile_attempted = True + fast_ar._compile_failed = False + fast_ar._disable_compile_for_graph = False + + seen_position_ids: list[torch.Tensor] = [] + + def fake_run_model_one(step_input, step_pos_ids, cache_pos): + seen_position_ids.append(step_pos_ids) + return torch.zeros( + step_input.shape[0], + fast_config.hidden_size, + dtype=step_input.dtype, + device=step_input.device, + ) + + monkeypatch.setattr(fast_ar, "_run_model_one", fake_run_model_one) + + hidden = torch.zeros(3, slow_config.hidden_size, dtype=torch.float32) + semantic = torch.full((3,), slow_config.semantic_begin_id, dtype=torch.long) + fast_ar(hidden, semantic, do_sample=False) + + assert fast_ar._pos_ids is not None + assert fast_ar._pos_ids.shape == (3, fast_config.num_codebooks + 1) + assert len(seen_position_ids) == fast_config.num_codebooks + for step, step_pos_ids in enumerate(seen_position_ids): + assert step_pos_ids.shape == (3,) + assert step_pos_ids.untyped_storage().data_ptr() == fast_ar._pos_ids.untyped_storage().data_ptr() + assert step_pos_ids.stride() == (fast_ar._pos_ids.stride(0),) + expected = torch.full((3,), step) + assert torch.equal(step_pos_ids.cpu(), expected) + + +def test_talker_mtp_does_not_mutate_input(): + model = object.__new__(FishSpeechSlowARForConditionalGeneration) + nn.Module.__init__(model) + model._semantic_begin_id = 4 + model._semantic_end_id = 19 + model._codebook_size = 16 + model._num_codebooks = 4 + model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8) + model.fast_ar = lambda **_: torch.tensor( + [ + [0, 1, 2, 3], + [15, 14, 13, 12], + ], + dtype=torch.long, + ) + input_ids = torch.tensor([4, 2], dtype=torch.long) + input_embeds = torch.randn(2, 8) + input_embeds_before = input_embeds.clone() + last_hidden = torch.randn(2, 8, dtype=torch.bfloat16) + text_step = torch.zeros(2, 8, dtype=torch.bfloat16) + + out1, _ = FishSpeechSlowARForConditionalGeneration.talker_mtp( + model, + input_ids, + input_embeds, + last_hidden, + text_step, + ) + + assert torch.equal(input_embeds, input_embeds_before) + assert out1.untyped_storage().data_ptr() != input_embeds.untyped_storage().data_ptr() + + +def test_talker_mtp_forwards_sampling_params(): + model = object.__new__(FishSpeechSlowARForConditionalGeneration) + nn.Module.__init__(model) + model._semantic_begin_id = 4 + model._semantic_end_id = 19 + model._codebook_size = 16 + model._num_codebooks = 4 + model.codebook_embeddings = nn.Embedding(model._num_codebooks * model._codebook_size, 8) + seen = {} + + def fake_fast_ar(**kwargs): + seen.update(kwargs) + return torch.tensor([[0, 1, 2, 3]], dtype=torch.long) + + model.fast_ar = fake_fast_ar + input_ids = torch.tensor([4], dtype=torch.long) + input_embeds = torch.randn(1, 8) + last_hidden = torch.randn(1, 8, dtype=torch.bfloat16) + text_step = torch.zeros(1, 8, dtype=torch.bfloat16) + + FishSpeechSlowARForConditionalGeneration.talker_mtp( + model, + input_ids, + input_embeds, + last_hidden, + text_step, + do_sample=False, + temperature=0.1, + top_k=5, + top_p=0.7, + ) + + assert seen["do_sample"] is False + assert seen["temperature"] == 0.1 + assert seen["top_k"] == 5 + assert seen["top_p"] == 0.7 diff --git a/tests/model_executor/models/fish_speech/test_fish_speech_regressions.py b/tests/model_executor/models/fish_speech/test_fish_speech_regressions.py index dbb0150e8..b21ff3416 100644 --- a/tests/model_executor/models/fish_speech/test_fish_speech_regressions.py +++ b/tests/model_executor/models/fish_speech/test_fish_speech_regressions.py @@ -1,5 +1,6 @@ import functools import math +from types import SimpleNamespace import pytest import torch @@ -26,6 +27,39 @@ class _FakeCodec: return wav, audio_lengths +class _RecordingCodec: + def __init__(self): + self.calls = [] + + def decode(self, codes_bqf: torch.Tensor, feature_lengths: torch.Tensor): + self.calls.append((tuple(codes_bqf.shape), tuple(int(x) for x in feature_lengths.tolist()))) + batch = codes_bqf.shape[0] + wav_len = int(codes_bqf.shape[-1]) * 10 + wav = torch.arange(batch * wav_len, dtype=torch.float32).view(batch, 1, wav_len) + audio_lengths = feature_lengths.to(torch.long) * 10 + return wav, audio_lengths + + +class _FakeOneArgCodec: + def decode(self, codes_bqf: torch.Tensor): + batch = codes_bqf.shape[0] + return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100) + + +class _FakeFromIndicesCodec: + def __init__(self): + self.from_indices_calls = 0 + + def from_indices(self, codes_bqf: torch.Tensor): + self.from_indices_calls += 1 + batch = codes_bqf.shape[0] + return torch.arange(batch * 100, dtype=torch.float32).view(batch, 1, 100) + + def decode(self, codes_bqf: torch.Tensor): + del codes_bqf + raise AssertionError("decode() should not be used for index tensors when from_indices() exists") + + class _FakeTokenizer: def __init__(self, mapping, unk_token_id=-1): self._mapping = mapping @@ -40,6 +74,7 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices(): decoder = object.__new__(FishSpeechDACDecoder) torch.nn.Module.__init__(decoder) decoder._codec = _FakeCodec() + decoder._codec_decode_takes_lengths = True decoder._num_codebooks = 10 decoder._output_sample_rate = 44100 decoder._hop_length = 512 @@ -62,6 +97,132 @@ def test_dac_decoder_mixed_batch_empty_request_does_not_misalign_indices(): assert audios[1].shape[0] == 50 +def test_dac_decoder_uses_tensor_codes_from_runtime_info(): + _, FishSpeechDACDecoder, _ = _fish_speech_regression_modules() + decoder = object.__new__(FishSpeechDACDecoder) + torch.nn.Module.__init__(decoder) + decoder._codec = _FakeCodec() + decoder._codec_decode_takes_lengths = True + decoder._num_codebooks = 2 + decoder._output_sample_rate = 44100 + decoder._hop_length = 512 + decoder._logged_codec_stats = False + decoder._ensure_codec_loaded = lambda: None + + out = decoder.forward( + input_ids=torch.tensor([0], dtype=torch.long), + runtime_additional_information=[ + { + "codes": {"audio": torch.tensor([[1, 2], [3, 4]], dtype=torch.long)}, + "meta": {"left_context_size": 1}, + } + ], + ) + + audios = out.multimodal_outputs["model_outputs"] + assert len(audios) == 1 + # 2 total frames with 1 frame of left context => proportional trim removes half the samples. + assert audios[0].shape[0] == 50 + + +def test_dac_decoder_supports_codec_decode_without_lengths(): + _, FishSpeechDACDecoder, _ = _fish_speech_regression_modules() + decoder = object.__new__(FishSpeechDACDecoder) + torch.nn.Module.__init__(decoder) + decoder._codec = _FakeOneArgCodec() + decoder._codec_decode_takes_lengths = False + decoder._num_codebooks = 10 + decoder._output_sample_rate = 44100 + decoder._hop_length = 512 + decoder._logged_codec_stats = False + decoder._ensure_codec_loaded = lambda: None + decoder._split_request_ids = lambda ids, seq_token_counts=None: [ + torch.arange(10, dtype=torch.long), + torch.arange(20, dtype=torch.long), + ] + + out = decoder.forward( + input_ids=torch.arange(30, dtype=torch.long), + runtime_additional_information=[{}, {}], + ) + + audios = out.multimodal_outputs["model_outputs"] + assert len(audios) == 2 + assert audios[0].shape[0] == 50 + assert audios[1].shape[0] == 100 + + +def test_dac_decoder_prefers_from_indices_for_one_arg_codec(): + _, FishSpeechDACDecoder, _ = _fish_speech_regression_modules() + codec = _FakeFromIndicesCodec() + decoder = object.__new__(FishSpeechDACDecoder) + torch.nn.Module.__init__(decoder) + decoder._codec = codec + decoder._codec_decode_takes_lengths = False + decoder._num_codebooks = 10 + decoder._output_sample_rate = 44100 + decoder._hop_length = 512 + decoder._logged_codec_stats = False + decoder._ensure_codec_loaded = lambda: None + decoder._split_request_ids = lambda ids, seq_token_counts=None: [ + torch.arange(10, dtype=torch.long), + torch.arange(20, dtype=torch.long), + ] + + out = decoder.forward( + input_ids=torch.arange(30, dtype=torch.long), + runtime_additional_information=[{}, {}], + ) + + audios = out.multimodal_outputs["model_outputs"] + assert codec.from_indices_calls == 1 + assert len(audios) == 2 + assert audios[0].shape[0] == 50 + assert audios[1].shape[0] == 100 + + +def test_dac_decoder_splits_groups_by_padded_frame_budget(): + _, FishSpeechDACDecoder, _ = _fish_speech_regression_modules() + codec = _RecordingCodec() + decoder = object.__new__(FishSpeechDACDecoder) + torch.nn.Module.__init__(decoder) + decoder._codec = codec + decoder._codec_decode_takes_lengths = True + decoder._num_codebooks = 2 + decoder._output_sample_rate = 44100 + decoder._hop_length = 512 + decoder._logged_codec_stats = False + decoder._decode_batch_max_padded_frames = 100 + decoder._decode_batch_max_batch = 0 + decoder._ensure_codec_loaded = lambda: None + decoder._split_request_ids = lambda ids, seq_token_counts=None: [ + torch.arange(2 * 46, dtype=torch.long), + torch.arange(2 * 45, dtype=torch.long), + torch.arange(2 * 47, dtype=torch.long), + ] + + decoder.forward(input_ids=torch.arange(2 * (46 + 45 + 47), dtype=torch.long)) + + assert codec.calls == [ + ((2, 2, 46), (45, 46)), + ((1, 2, 47), (47,)), + ] + + +def test_dac_decoder_dtype_can_be_configured(): + _, FishSpeechDACDecoder, _ = _fish_speech_regression_modules() + cfg = SimpleNamespace( + model_config=SimpleNamespace( + model="unused", + stage_connector_config={"extra": {"fish_speech_dac_dtype": "fp16"}}, + ) + ) + + decoder = FishSpeechDACDecoder(vllm_config=cfg) + + assert decoder._dac_dtype == torch.float16 + + def test_structured_voice_clone_prefill_adds_full_codebooks_with_decode_scale(monkeypatch): slow_ar_module, _, FishSpeechSlowARForConditionalGeneration = _fish_speech_regression_modules() model = object.__new__(FishSpeechSlowARForConditionalGeneration) diff --git a/tests/model_executor/stage_input_processors/test_fish_speech_async_chunk.py b/tests/model_executor/stage_input_processors/test_fish_speech_async_chunk.py new file mode 100644 index 000000000..132615b14 --- /dev/null +++ b/tests/model_executor/stage_input_processors/test_fish_speech_async_chunk.py @@ -0,0 +1,229 @@ +from collections import defaultdict +from types import SimpleNamespace + +import pytest +import torch + +from vllm_omni.model_executor.stage_input_processors.fish_speech import ( + slow_ar_to_dac_decoder_async_chunk, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def _make_transfer_manager( + *, + tensor_payload: bool = False, + chunk_frames: int = 2, + left_context: int = 1, + initial_chunk_frames: int = 0, + single_initial_chunk: bool = False, + backlog_chunk_frames: int = 0, + backlog_load_threshold: float = 0.75, + max_num_seqs: int = 1, +): + return SimpleNamespace( + code_prompt_token_ids=defaultdict(list), + put_req_chunk=defaultdict(int), + scheduler_max_num_seqs=max_num_seqs, + connector=SimpleNamespace( + config={ + "extra": { + "codec_chunk_frames": chunk_frames, + "codec_left_context_frames": left_context, + "initial_codec_chunk_frames": initial_chunk_frames, + "fish_speech_tensor_codes": tensor_payload, + "fish_speech_single_initial_chunk": single_initial_chunk, + "fish_speech_backlog_codec_chunk_frames": backlog_chunk_frames, + "fish_speech_backlog_load_threshold": backlog_load_threshold, + } + } + ), + ) + + +def _make_request(req_id="req", *, finished=False): + return SimpleNamespace( + external_req_id=req_id, + additional_information=None, + is_finished=lambda: finished, + ) + + +def _seed_frames(transfer_manager, req_id: str, n_frames: int, n_codebooks: int = 3): + frame = torch.arange(1, n_codebooks + 1, dtype=torch.long) + transfer_manager.code_prompt_token_ids[req_id] = [frame + i * n_codebooks for i in range(n_frames)] + + +def _call_seeded(transfer_manager, req_id: str = "req", *, n_frames: int, finished: bool = False): + _seed_frames(transfer_manager, req_id, n_frames) + return slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + {}, + _make_request(req_id, finished=finished), + is_finished=finished, + ) + + +def test_slow_ar_to_dac_async_chunk_can_emit_tensor_codes(): + transfer_manager = _make_transfer_manager(tensor_payload=True) + request = _make_request() + + assert ( + slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + {"audio_codes": torch.tensor([[1, 2, 3]])}, + request, + ) + is None + ) + payload = slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + {"audio_codes": torch.tensor([[4, 5, 6]])}, + request, + ) + + assert payload is not None + audio_codes = payload.codes.audio + assert isinstance(audio_codes, torch.Tensor) + assert audio_codes.dtype == torch.long + assert torch.equal(audio_codes, torch.tensor([[1, 4], [2, 5], [3, 6]], dtype=torch.long)) + assert payload.meta.left_context_size == 0 + + +def test_slow_ar_to_dac_async_chunk_uses_valid_mask_for_zero_codes(): + transfer_manager = _make_transfer_manager(tensor_payload=True, chunk_frames=1, left_context=0) + request = _make_request() + + assert ( + slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + { + "audio_codes": torch.tensor([[0, 0, 0]]), + "audio_code_valid": torch.tensor([False]), + }, + request, + ) + is None + ) + payload = slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + { + "audio_codes": torch.tensor([[4, 5, 6]]), + "audio_code_valid": torch.tensor([True]), + }, + request, + ) + + assert payload is not None + assert torch.equal(payload.codes.audio, torch.tensor([[4], [5], [6]], dtype=torch.long)) + + +def test_slow_ar_to_dac_async_chunk_keeps_flat_codes_by_default(): + transfer_manager = _make_transfer_manager(tensor_payload=False) + request = _make_request() + + slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + {"audio_codes": torch.tensor([[1, 2]])}, + request, + ) + payload = slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + {"audio_codes": torch.tensor([[3, 4]])}, + request, + ) + + assert payload is not None + assert torch.equal(payload.codes.audio, torch.tensor([1, 3, 2, 4], dtype=torch.long)) + + +def test_qwen3_style_single_initial_chunk_only_emits_first_chunk_early(): + transfer_manager = _make_transfer_manager( + chunk_frames=5, + left_context=5, + initial_chunk_frames=2, + single_initial_chunk=True, + ) + + first = _call_seeded(transfer_manager, n_frames=2) + assert first is not None + assert first.meta.left_context_size == 0 + assert first.codes.audio.numel() == 3 * 2 + + # After the first small chunk, return to normal chunk boundaries: + # initial_coverage=2, so the next emit is at 2 + 5 = 7 frames. + assert _call_seeded(transfer_manager, n_frames=5) is None + + second = _call_seeded(transfer_manager, n_frames=7) + assert second is not None + assert second.meta.left_context_size == 2 + assert second.codes.audio.numel() == 3 * 7 + + +def test_backlog_chunk_size_uses_larger_post_initial_boundary(): + transfer_manager = _make_transfer_manager( + chunk_frames=5, + left_context=5, + initial_chunk_frames=2, + single_initial_chunk=True, + backlog_chunk_frames=8, + backlog_load_threshold=0.5, + max_num_seqs=4, + ) + for idx in range(2): + _seed_frames(transfer_manager, f"other-{idx}", 10) + + first = _call_seeded(transfer_manager, "req", n_frames=2) + assert first is not None + + # Under backlog, steady chunk is 8 instead of 5. The next emit is + # initial_coverage=2 plus 8 frames, so n=7 must still be held. + assert _call_seeded(transfer_manager, "req", n_frames=7) is None + second = _call_seeded(transfer_manager, "req", n_frames=10) + assert second is not None + assert second.meta.left_context_size == 2 + assert second.codes.audio.numel() == 3 * 10 + + +def test_backlog_chunk_size_keeps_multi_initial_boundary_aligned(): + transfer_manager = _make_transfer_manager( + chunk_frames=25, + left_context=25, + initial_chunk_frames=10, + single_initial_chunk=False, + backlog_chunk_frames=50, + backlog_load_threshold=0.5, + max_num_seqs=4, + ) + for idx in range(2): + _seed_frames(transfer_manager, f"other-{idx}", 50) + + first = _call_seeded(transfer_manager, "req", n_frames=10) + assert first is not None + transfer_manager.put_req_chunk["req"] += 1 + second = _call_seeded(transfer_manager, "req", n_frames=20) + assert second is not None + transfer_manager.put_req_chunk["req"] += 1 + + assert _call_seeded(transfer_manager, "req", n_frames=50) is None + third = _call_seeded(transfer_manager, "req", n_frames=70) + assert third is not None + assert third.meta.left_context_size == 20 + assert third.codes.audio.numel() == 3 * 70 + + +def test_invalid_backlog_config_fails_loudly(): + transfer_manager = _make_transfer_manager( + chunk_frames=5, + left_context=5, + backlog_chunk_frames="invalid", + ) + request = _make_request() + + with pytest.raises(ValueError, match="fish_speech_backlog_codec_chunk_frames"): + slow_ar_to_dac_decoder_async_chunk( + transfer_manager, + {"audio_codes": torch.tensor([[1, 2, 3]])}, + request, + ) diff --git a/vllm_omni/attention/__init__.py b/vllm_omni/attention/__init__.py new file mode 100644 index 000000000..208f01a7c --- /dev/null +++ b/vllm_omni/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm_omni/attention/fish_kvcache_attn.py b/vllm_omni/attention/fish_kvcache_attn.py new file mode 100644 index 000000000..9e552e762 --- /dev/null +++ b/vllm_omni/attention/fish_kvcache_attn.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import os +import threading +from typing import Any + +import torch + +FISH_KVCACHE_LONG_SPLIT_TOKENS = 1024 +FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN = 1024 + +_FISH_KVCACHE_ATTN_ENV = "VLLM_OMNI_FISH_KVCACHE_ATTN" +_ENABLED_VALUES = frozenset({"1", "true", "yes", "on", "required"}) +_DISABLED_VALUES = frozenset({"0", "false", "no", "off", "disabled", "disable"}) +_REQUIRED_VALUES = frozenset({"required"}) +_WORKSPACE_CACHE: dict[tuple[Any, ...], tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = {} +_WORKSPACE_CACHE_LOCK = threading.Lock() + + +def _triton_backend() -> Any: + from vllm_omni.attention import fish_kvcache_triton + + return fish_kvcache_triton + + +def is_available() -> bool: + return _triton_backend().is_available() + + +def load_error() -> Exception | None: + return _triton_backend().load_error() + + +def is_fish_kvcache_attn_enabled() -> bool: + value = os.environ.get(_FISH_KVCACHE_ATTN_ENV) + if value is None: + return True + value = value.lower() + if value in _DISABLED_VALUES: + return False + return value in _ENABLED_VALUES or value == "" + + +def is_fish_kvcache_attn_required() -> bool: + return os.environ.get(_FISH_KVCACHE_ATTN_ENV, "").lower() in _REQUIRED_VALUES + + +def _is_sliding_window_disabled(sliding_window: Any) -> bool: + if sliding_window is None: + return True + if isinstance(sliding_window, (list, tuple)) and len(sliding_window) == 2: + return int(sliding_window[0]) == -1 and int(sliding_window[1]) == -1 + return False + + +def can_use_fish_kvcache_attn( + *, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_table: torch.Tensor | None, + seq_lens: torch.Tensor, + max_query_len: int, + max_seq_len: int, + dcp_world_size: int, + use_cascade: bool, + alibi_slopes: Any, + sliding_window: Any, + output_scale: torch.Tensor | None = None, + output_block_scale: torch.Tensor | None = None, +) -> bool: + if not is_fish_kvcache_attn_enabled(): + return False + if not is_available(): + return False + if max_query_len != 1 or use_cascade or dcp_world_size != 1: + return False + if block_table is None or alibi_slopes is not None or not _is_sliding_window_disabled(sliding_window): + return False + if output_scale is not None or output_block_scale is not None: + return False + if query.dim() != 3 or key_cache.dim() != 4 or value_cache.dim() != 4: + return False + if block_table.dim() != 2 or seq_lens.dim() != 1: + return False + if block_table.shape[0] != query.shape[0] or seq_lens.shape[0] != query.shape[0]: + return False + if query.shape[-1] != 128 or key_cache.shape[-1] != 128: + return False + if key_cache.shape[1] != 16: + return False + if query.dtype not in (torch.float16, torch.bfloat16): + return False + if key_cache.dtype != query.dtype or value_cache.dtype != query.dtype: + return False + if block_table.dtype != torch.int32 or seq_lens.dtype != torch.int32: + return False + if max_seq_len <= 0: + return False + if max_seq_len > block_table.shape[1] * key_cache.shape[1]: + return False + if not ( + query.is_contiguous() + and key_cache.is_contiguous() + and value_cache.is_contiguous() + and block_table.is_contiguous() + and seq_lens.is_contiguous() + ): + return False + return True + + +def _is_cuda_graph_capturing() -> bool: + return bool(torch.cuda.is_current_stream_capturing()) + + +def _raise_workspace_capture_miss() -> None: + raise RuntimeError("Fish kvcache attention workspace was not prewarmed before CUDA graph capture") + + +def _get_decode_workspace( + query: torch.Tensor, + max_seq_len: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + batch_size, num_q_heads, head_dim = query.shape + if int(max_seq_len) <= FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN: + key = (query.device.type, query.device.index, "empty") + partial_shapes = ((0,), (0,), (0,)) + else: + num_splits = (int(max_seq_len) + FISH_KVCACHE_LONG_SPLIT_TOKENS - 1) // FISH_KVCACHE_LONG_SPLIT_TOKENS + key = ( + query.device.type, + query.device.index, + int(batch_size), + int(num_q_heads), + int(head_dim), + int(num_splits), + ) + partial_shapes = ( + (num_splits, batch_size, num_q_heads), + (num_splits, batch_size, num_q_heads), + (num_splits, batch_size, num_q_heads, head_dim), + ) + with _WORKSPACE_CACHE_LOCK: + workspace = _WORKSPACE_CACHE.get(key) + if workspace is None: + if _is_cuda_graph_capturing(): + _raise_workspace_capture_miss() + workspace = ( + torch.empty(partial_shapes[0], device=query.device, dtype=torch.float32), + torch.empty(partial_shapes[1], device=query.device, dtype=torch.float32), + torch.empty(partial_shapes[2], device=query.device, dtype=torch.float32), + ) + _WORKSPACE_CACHE[key] = workspace + return workspace + + +def prewarm_fish_kvcache_attn_workspace( + query: torch.Tensor, + max_seq_len: int, +) -> None: + _get_decode_workspace(query, int(max_seq_len)) + + +def fish_decode_kvcache_attn( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_table: torch.Tensor, + seq_lens: torch.Tensor, + out: torch.Tensor, + *, + scale: float, + max_seq_len: int, +) -> torch.Tensor: + partial_m, partial_l, partial_acc = _get_decode_workspace(query, int(max_seq_len)) + return _triton_backend().fish_decode_kvcache_attn_triton( + query, + key_cache, + value_cache, + block_table, + seq_lens, + out, + scale=float(scale), + max_seq_len=int(max_seq_len), + small_path_max_seq_len=FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN, + long_split_tokens=FISH_KVCACHE_LONG_SPLIT_TOKENS, + partial_m=partial_m, + partial_l=partial_l, + partial_acc=partial_acc, + ) diff --git a/vllm_omni/attention/fish_kvcache_backend.py b/vllm_omni/attention/fish_kvcache_backend.py new file mode 100644 index 000000000..3f3e9bdc6 --- /dev/null +++ b/vllm_omni/attention/fish_kvcache_backend.py @@ -0,0 +1,406 @@ +from __future__ import annotations + +import types +from collections import Counter +from collections.abc import Callable +from typing import Any + +import torch +from vllm.logger import init_logger + +from vllm_omni.attention.fish_kvcache_attn import ( + FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN, + can_use_fish_kvcache_attn, + fish_decode_kvcache_attn, + is_available, + is_fish_kvcache_attn_enabled, + is_fish_kvcache_attn_required, + load_error, + prewarm_fish_kvcache_attn_workspace, +) + +logger = init_logger(__name__) + +_FISH_SLOW_AR_ARCH = "FishSpeechSlowARForConditionalGeneration" +_FIRST_SMALL_HIT_LOGGED = False +_FIRST_LONG_HIT_LOGGED = False +_FIRST_MISS_LOGGED = False +_SMALL_HIT_COUNT = 0 +_LONG_HIT_COUNT = 0 +_FALLBACK_COUNTS: Counter[str] = Counter() + + +def _fish_kvcache_enabled() -> bool: + return is_fish_kvcache_attn_enabled() + + +def is_fish_kvcache_attn_active_for_model(model_config: Any) -> bool: + return getattr(model_config, "model_arch", None) == _FISH_SLOW_AR_ARCH and is_fish_kvcache_attn_enabled() + + +def reset_fish_kvcache_attn_stats() -> None: + global _LONG_HIT_COUNT, _SMALL_HIT_COUNT + _SMALL_HIT_COUNT = 0 + _LONG_HIT_COUNT = 0 + _FALLBACK_COUNTS.clear() + + +def get_fish_kvcache_attn_stats() -> dict[str, Any]: + return { + "small_hit_count": _SMALL_HIT_COUNT, + "long_hit_count": _LONG_HIT_COUNT, + "fallback_count_by_reason": dict(_FALLBACK_COUNTS), + } + + +def _record_fallback(reason: str) -> None: + _FALLBACK_COUNTS[reason] += 1 + + +def _small_decode_bucket(max_seq_len: int) -> int: + if max_seq_len <= 64: + return 64 + if max_seq_len <= 128: + return 128 + if max_seq_len <= 256: + return 256 + if max_seq_len <= 512: + return 512 + return FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN + + +def prewarm_fish_kvcache_attn_capture_workspaces( + *, + model_config: Any, + device: torch.device, + dtype: torch.dtype, + capture_sizes: list[int] | tuple[int, ...], +) -> int: + """Preallocate Fish attention workspaces used during CUDA graph capture.""" + if not _fish_kvcache_enabled() or not is_available(): + return 0 + if not is_fish_kvcache_attn_active_for_model(model_config): + return 0 + + text_config = model_config.hf_config.text_config + num_heads = int(text_config.num_attention_heads) + head_dim = int(text_config.head_dim) + max_seq_len = int(model_config.max_model_len) + sizes = sorted({int(size) for size in capture_sizes if int(size) > 0}) + if not sizes: + return 0 + + # The small path uses an empty workspace keyed only by device. Prewarm it + # once, then prewarm every CUDA graph capture batch size for the long path. + small_query = torch.empty((1, num_heads, head_dim), device=device, dtype=dtype) + prewarm_fish_kvcache_attn_workspace(small_query, 1) + prewarmed = 1 + + for batch_size in sizes: + query = torch.empty((batch_size, num_heads, head_dim), device=device, dtype=dtype) + prewarm_fish_kvcache_attn_workspace(query, max_seq_len) + prewarmed += 1 + + logger.info( + "Prewarmed Fish kvcache attention workspaces for capture sizes %s (num_heads=%d head_dim=%d max_seq_len=%d)", + sizes, + num_heads, + head_dim, + max_seq_len, + ) + return prewarmed + + +def _attach_seq_lens_cpu_upper_bound(attn_metadata: Any, seq_lens_cpu_upper_bound: torch.Tensor) -> None: + if attn_metadata is None: + return + if isinstance(attn_metadata, dict): + for value in attn_metadata.values(): + _attach_seq_lens_cpu_upper_bound(value, seq_lens_cpu_upper_bound) + return + if isinstance(attn_metadata, (list, tuple)): + for value in attn_metadata: + _attach_seq_lens_cpu_upper_bound(value, seq_lens_cpu_upper_bound) + return + setattr(attn_metadata, "seq_lens_cpu_upper_bound", seq_lens_cpu_upper_bound) + + +def _graph_capture_seq_len_upper_bound( + *, + model_config: Any, + max_query_len: int, + for_cudagraph_capture: bool, +) -> int | None: + if not for_cudagraph_capture or int(max_query_len) != 1: + return None + max_model_len = int(getattr(model_config, "max_model_len", 0)) + return max(max_model_len, FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN) + + +def maybe_attach_fish_kvcache_seq_lens_upper_bound( + *, + model_config: Any, + attn_metadata: Any, + input_batch: Any, + optimistic_seq_lens_cpu: torch.Tensor, + num_reqs: int, + num_reqs_padded: int, + max_query_len: int, + pad_attn: bool, + for_cudagraph_capture: bool = False, + num_scheduled_tokens_np: Any = None, +) -> None: + """Attach CPU seq-len upper bounds for the Fish-only Triton decode fastpath.""" + if not is_fish_kvcache_attn_active_for_model(model_config) or int(max_query_len) != 1: + return + + upper_bound_rows = int(num_reqs_padded if pad_attn else num_reqs) + capture_upper_bound = _graph_capture_seq_len_upper_bound( + model_config=model_config, + max_query_len=max_query_len, + for_cudagraph_capture=for_cudagraph_capture, + ) + if capture_upper_bound is not None: + optimistic_seq_lens_cpu[:upper_bound_rows].fill_(capture_upper_bound) + else: + if num_scheduled_tokens_np is None: + return + seq_lens_cpu_upper_bound = torch.as_tensor( + input_batch.num_computed_tokens_cpu[:num_reqs] + num_scheduled_tokens_np, + dtype=torch.int32, + device="cpu", + ) + optimistic_seq_lens_cpu[:num_reqs] = seq_lens_cpu_upper_bound + if upper_bound_rows > num_reqs: + optimistic_seq_lens_cpu[num_reqs:upper_bound_rows].fill_(1) + _attach_seq_lens_cpu_upper_bound( + attn_metadata, + optimistic_seq_lens_cpu[:upper_bound_rows], + ) + + +def _dispatch_max_seq_len(attn_metadata: Any, active_batch: int, seq_lens: Any) -> int | None: + max_seq_len = int(attn_metadata.max_seq_len) + if int(attn_metadata.max_query_len) != 1: + return max_seq_len + + upper_bound = getattr(attn_metadata, "seq_lens_cpu_upper_bound", None) + if upper_bound is None: + if is_fish_kvcache_attn_required(): + raise RuntimeError("Fish kvcache attention requires seq_lens_cpu_upper_bound for decode dispatch") + return None + if not isinstance(upper_bound, torch.Tensor) or upper_bound.device.type != "cpu": + if is_fish_kvcache_attn_required(): + raise RuntimeError("Fish kvcache attention seq_lens_cpu_upper_bound must be a CPU tensor") + return None + if int(upper_bound.shape[0]) < active_batch: + if is_fish_kvcache_attn_required(): + raise RuntimeError( + "Fish kvcache attention seq_lens_cpu_upper_bound does not cover the active decode batch: " + f"upper_bound_rows={int(upper_bound.shape[0])}, active_batch={active_batch}" + ) + return None + upper_bound = upper_bound[:active_batch] + real_upper_bound = max(1, int(upper_bound.max().item())) + if is_fish_kvcache_attn_required(): + real_max_seq_len = int(seq_lens.max().item()) + if real_upper_bound < real_max_seq_len: + raise RuntimeError( + "Fish kvcache attention seq_lens_cpu_upper_bound " + f"underestimates real seq_lens: upper_bound={real_upper_bound}, " + f"real={real_max_seq_len}" + ) + if real_upper_bound <= FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN: + return _small_decode_bucket(real_upper_bound) + return real_upper_bound + + +def _forward_with_fish_kvcache( + impl: Any, + original_forward: Callable[..., Any], + layer: Any, + query: Any, + key: Any, + value: Any, + kv_cache: Any, + attn_metadata: Any, + output: Any, + output_scale: Any = None, + output_block_scale: Any = None, +) -> Any: + global _FIRST_LONG_HIT_LOGGED, _FIRST_MISS_LOGGED, _FIRST_SMALL_HIT_LOGGED + global _LONG_HIT_COUNT, _SMALL_HIT_COUNT + + if attn_metadata is not None and not attn_metadata.use_cascade: + num_actual_tokens = attn_metadata.num_actual_tokens + key_cache, value_cache = kv_cache.unbind(0) + q = query[:num_actual_tokens] + out = output[:num_actual_tokens] + active_batch = int(q.shape[0]) + block_table = attn_metadata.block_table[:active_batch] if attn_metadata.block_table is not None else None + seq_lens = attn_metadata.seq_lens[:active_batch] + dispatch_max_seq_len = _dispatch_max_seq_len(attn_metadata, active_batch, seq_lens) + can_use = dispatch_max_seq_len is not None and can_use_fish_kvcache_attn( + query=q, + key_cache=key_cache, + value_cache=value_cache, + block_table=block_table, + seq_lens=seq_lens, + max_query_len=attn_metadata.max_query_len, + max_seq_len=dispatch_max_seq_len, + dcp_world_size=impl.dcp_world_size, + use_cascade=attn_metadata.use_cascade, + alibi_slopes=impl.alibi_slopes, + sliding_window=impl.sliding_window, + output_scale=output_scale, + output_block_scale=output_block_scale, + ) + if can_use: + is_long_path = dispatch_max_seq_len > FISH_KVCACHE_SMALL_PATH_MAX_SEQ_LEN + if is_long_path: + _LONG_HIT_COUNT += 1 + else: + _SMALL_HIT_COUNT += 1 + should_log_hit = (is_long_path and not _FIRST_LONG_HIT_LOGGED) or ( + not is_long_path and not _FIRST_SMALL_HIT_LOGGED + ) + if should_log_hit: + if is_long_path: + _FIRST_LONG_HIT_LOGGED = True + path_name = "long" + else: + _FIRST_SMALL_HIT_LOGGED = True + path_name = "small" + logger.info( + "Fish decode-only kvcache attention fast path hit: " + "query_shape=%s key_cache_shape=%s block_table_shape=%s " + "seq_lens_shape=%s max_seq_len=%s path=%s", + tuple(q.shape), + tuple(key_cache.shape), + tuple(attn_metadata.block_table.shape), + tuple(attn_metadata.seq_lens.shape), + dispatch_max_seq_len, + path_name, + ) + fish_decode_kvcache_attn( + q, + key_cache, + value_cache, + block_table, + seq_lens, + out, + scale=float(impl.scale), + max_seq_len=dispatch_max_seq_len, + ) + return output + + is_decode_request = int(attn_metadata.max_query_len) == 1 + if is_decode_request and dispatch_max_seq_len is None: + _record_fallback("missing_cpu_upper_bound") + elif is_decode_request: + _record_fallback("guard_miss") + else: + _record_fallback("non_decode") + if is_decode_request and is_fish_kvcache_attn_required(): + raise RuntimeError( + "Fish kvcache attention is required but guard rejected the request: " + f"query_shape={tuple(q.shape)} query_dtype={q.dtype} " + f"max_query_len={attn_metadata.max_query_len} max_seq_len={dispatch_max_seq_len} " + f"dcp_world_size={impl.dcp_world_size} use_cascade={attn_metadata.use_cascade}" + ) + + if not _FIRST_MISS_LOGGED: + _FIRST_MISS_LOGGED = True + logger.info( + "Fish decode-only kvcache attention fast path miss: " + "query_shape=%s query_dtype=%s key_cache_shape=%s key_dtype=%s " + "block_table_shape=%s block_dtype=%s seq_lens_shape=%s seq_dtype=%s " + "max_query_len=%s max_seq_len=%s dcp_world_size=%s use_cascade=%s", + tuple(q.shape), + q.dtype, + tuple(key_cache.shape), + key_cache.dtype, + tuple(block_table.shape) if block_table is not None else None, + getattr(block_table, "dtype", None), + tuple(seq_lens.shape), + seq_lens.dtype, + attn_metadata.max_query_len, + dispatch_max_seq_len, + impl.dcp_world_size, + attn_metadata.use_cascade, + ) + + return original_forward( + layer, + query, + key, + value, + kv_cache, + attn_metadata, + output, + output_scale=output_scale, + output_block_scale=output_block_scale, + ) + + +def install_fish_kvcache_attn_backend(model: Any) -> int: + """Install the Fish kvcache fast path on this Fish SlowAR model only.""" + if not _fish_kvcache_enabled(): + return 0 + if not is_available(): + _record_fallback("implementation_unavailable") + if is_fish_kvcache_attn_required(): + raise RuntimeError( + f"VLLM_OMNI_FISH_KVCACHE_ATTN=required but Fish kvcache attention is unavailable: {load_error()!r}" + ) + logger.warning( + "Fish kvcache attention is enabled but unavailable: %r", + load_error(), + ) + return 0 + + installed = 0 + for layer in model.layers: + impl = layer.self_attn.attn.impl + if getattr(impl, "_fish_kvcache_attn_installed", False): + continue + + original_forward = impl.forward + + def fish_forward( + this_impl: Any, + layer: Any, + query: Any, + key: Any, + value: Any, + kv_cache: Any, + attn_metadata: Any, + output: Any, + output_scale: Any = None, + output_block_scale: Any = None, + *, + _original_forward: Callable[..., Any] = original_forward, + ) -> Any: + return _forward_with_fish_kvcache( + this_impl, + _original_forward, + layer, + query, + key, + value, + kv_cache, + attn_metadata, + output, + output_scale=output_scale, + output_block_scale=output_block_scale, + ) + + impl.forward = types.MethodType(fish_forward, impl) + impl._fish_kvcache_attn_installed = True + installed += 1 + + if installed: + logger.info("Installed Fish decode-only kvcache attention backend on %d attention layers", installed) + elif is_fish_kvcache_attn_required(): + raise RuntimeError("Fish kvcache attention is required but installed 0 attention layers") + return installed diff --git a/vllm_omni/attention/fish_kvcache_triton.py b/vllm_omni/attention/fish_kvcache_triton.py new file mode 100644 index 000000000..35d218903 --- /dev/null +++ b/vllm_omni/attention/fish_kvcache_triton.py @@ -0,0 +1,359 @@ +# ruff: noqa: N803 + +from __future__ import annotations + +import torch + +try: + import triton + import triton.language as tl +except Exception as exc: # pragma: no cover - depends on optional Triton install + triton = None + tl = None + _LOAD_ERROR: Exception | None = exc +else: + _LOAD_ERROR = None + +_BLOCK_SIZE = 16 +_HEAD_DIM = 128 +_BLOCK_N = 128 + + +def is_available() -> bool: + return _LOAD_ERROR is None and triton is not None and tl is not None + + +def load_error() -> Exception | None: + return _LOAD_ERROR + + +if is_available(): + + @triton.jit + def _small_decode_kernel( + Q, + K, + V, + BLOCK_TABLE, + SEQ_LENS, + OUT, + SCALE: tl.constexpr, + MAX_SEQ_LEN: tl.constexpr, + MAX_BLOCKS_PER_SEQ: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + KV_GROUP: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + GUARD_MAX_SEQ_LEN: tl.constexpr, + ): + batch_id = tl.program_id(0) + kv_head = tl.program_id(1) + + offs_h = tl.arange(0, BLOCK_H) + offs_d = tl.arange(0, BLOCK_D) + offs_n = tl.arange(0, BLOCK_N) + q_heads = kv_head * KV_GROUP + offs_h + mask_h = offs_h < KV_GROUP + + q_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :] + q = tl.load(Q + q_offsets, mask=mask_h[:, None], other=0.0) + + seq_len = tl.load(SEQ_LENS + batch_id) + if GUARD_MAX_SEQ_LEN and seq_len > GUARD_MAX_SEQ_LEN: + return + + m_i = tl.full((BLOCK_H,), -float("inf"), tl.float32) + l_i = tl.zeros((BLOCK_H,), tl.float32) + acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32) + + start_n = 0 + while start_n < seq_len: + cur_n = start_n + offs_n + mask_n = cur_n < seq_len + logical_block = cur_n // 16 + block_offset = cur_n - logical_block * 16 + physical_block = tl.load( + BLOCK_TABLE + batch_id * MAX_BLOCKS_PER_SEQ + logical_block, + mask=mask_n, + other=0, + ) + + kv_offsets = ( + (physical_block[:, None] * 16 + block_offset[:, None]) * NUM_KV_HEADS + kv_head + ) * BLOCK_D + offs_d[None, :] + k = tl.load(K + kv_offsets, mask=mask_n[:, None], other=0.0) + qk = tl.dot(q, tl.trans(k)) * SCALE + qk = tl.where(mask_h[:, None] & mask_n[None, :], qk, -float("inf")) + + v = tl.load(V + kv_offsets, mask=mask_n[:, None], other=0.0) + m_new = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp(qk - m_new[:, None]) + alpha = tl.exp(m_i - m_new) + acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v) + l_i = l_i * alpha + tl.sum(p, axis=1) + m_i = m_new + start_n += BLOCK_N + + denom = tl.where(l_i > 0.0, l_i, 1.0) + result = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0) + out_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :] + tl.store(OUT + out_offsets, result, mask=mask_h[:, None]) + + @triton.jit + def _split_partial_decode_kernel( + Q, + K, + V, + BLOCK_TABLE, + SEQ_LENS, + PARTIAL_M, + PARTIAL_L, + PARTIAL_ACC, + SCALE: tl.constexpr, + MAX_BLOCKS_PER_SEQ: tl.constexpr, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + KV_GROUP: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + SPLIT_TOKENS: tl.constexpr, + BATCH_SIZE: tl.constexpr, + MIN_SEQ_LEN: tl.constexpr, + ): + batch_id = tl.program_id(0) + kv_head = tl.program_id(1) + split_id = tl.program_id(2) + + offs_h = tl.arange(0, BLOCK_H) + offs_d = tl.arange(0, BLOCK_D) + offs_n = tl.arange(0, BLOCK_N) + q_heads = kv_head * KV_GROUP + offs_h + mask_h = offs_h < KV_GROUP + + partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads + partial_acc_offsets = partial_head_offsets[:, None] * BLOCK_D + offs_d[None, :] + + seq_len = tl.load(SEQ_LENS + batch_id) + if MIN_SEQ_LEN and seq_len < MIN_SEQ_LEN: + return + + m_i = tl.full((BLOCK_H,), -float("inf"), tl.float32) + l_i = tl.zeros((BLOCK_H,), tl.float32) + acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32) + + split_begin = split_id * SPLIT_TOKENS + if split_begin >= seq_len: + tl.store(PARTIAL_M + partial_head_offsets, m_i, mask=mask_h) + tl.store(PARTIAL_L + partial_head_offsets, l_i, mask=mask_h) + tl.store(PARTIAL_ACC + partial_acc_offsets, acc, mask=mask_h[:, None]) + return + + q_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :] + q = tl.load(Q + q_offsets, mask=mask_h[:, None], other=0.0) + + for split_offset in tl.range(0, SPLIT_TOKENS, BLOCK_N): + cur_n = split_begin + split_offset + offs_n + mask_n = cur_n < seq_len + logical_block = cur_n // 16 + block_offset = cur_n - logical_block * 16 + physical_block = tl.load( + BLOCK_TABLE + batch_id * MAX_BLOCKS_PER_SEQ + logical_block, + mask=mask_n, + other=0, + ) + + kv_offsets = ( + (physical_block[:, None] * 16 + block_offset[:, None]) * NUM_KV_HEADS + kv_head + ) * BLOCK_D + offs_d[None, :] + k = tl.load(K + kv_offsets, mask=mask_n[:, None], other=0.0) + qk = tl.dot(q, tl.trans(k)) * SCALE + qk = tl.where(mask_h[:, None] & mask_n[None, :], qk, -float("inf")) + + v = tl.load(V + kv_offsets, mask=mask_n[:, None], other=0.0) + m_new = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp(qk - m_new[:, None]) + alpha = tl.exp(m_i - m_new) + acc = acc * alpha[:, None] + tl.dot(p.to(v.dtype), v) + l_i = l_i * alpha + tl.sum(p, axis=1) + m_i = m_new + + tl.store(PARTIAL_M + partial_head_offsets, m_i, mask=mask_h) + tl.store(PARTIAL_L + partial_head_offsets, l_i, mask=mask_h) + tl.store(PARTIAL_ACC + partial_acc_offsets, acc, mask=mask_h[:, None]) + + @triton.jit + def _split_combine_decode_kernel( + SEQ_LENS, + PARTIAL_M, + PARTIAL_L, + PARTIAL_ACC, + OUT, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + KV_GROUP: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, + BATCH_SIZE: tl.constexpr, + NUM_SPLITS: tl.constexpr, + MIN_SEQ_LEN: tl.constexpr, + ): + batch_id = tl.program_id(0) + kv_head = tl.program_id(1) + + seq_len = tl.load(SEQ_LENS + batch_id) + if MIN_SEQ_LEN and seq_len < MIN_SEQ_LEN: + return + + offs_h = tl.arange(0, BLOCK_H) + offs_d = tl.arange(0, BLOCK_D) + q_heads = kv_head * KV_GROUP + offs_h + mask_h = offs_h < KV_GROUP + + global_m = tl.full((BLOCK_H,), -float("inf"), tl.float32) + for split_id in tl.range(0, NUM_SPLITS): + partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads + split_m = tl.load(PARTIAL_M + partial_head_offsets, mask=mask_h, other=-float("inf")) + global_m = tl.maximum(global_m, split_m) + + global_l = tl.zeros((BLOCK_H,), tl.float32) + acc = tl.zeros((BLOCK_H, BLOCK_D), tl.float32) + for split_id in tl.range(0, NUM_SPLITS): + partial_head_offsets = (split_id * BATCH_SIZE + batch_id) * NUM_Q_HEADS + q_heads + split_m = tl.load(PARTIAL_M + partial_head_offsets, mask=mask_h, other=-float("inf")) + split_l = tl.load(PARTIAL_L + partial_head_offsets, mask=mask_h, other=0.0) + weight = tl.exp(split_m - global_m) + partial_acc_offsets = partial_head_offsets[:, None] * BLOCK_D + offs_d[None, :] + split_acc = tl.load(PARTIAL_ACC + partial_acc_offsets, mask=mask_h[:, None], other=0.0) + global_l += split_l * weight + acc += split_acc * weight[:, None] + + denom = tl.where(global_l > 0.0, global_l, 1.0) + result = tl.where(global_l[:, None] > 0.0, acc / denom[:, None], 0.0) + out_offsets = batch_id * NUM_Q_HEADS * BLOCK_D + q_heads[:, None] * BLOCK_D + offs_d[None, :] + tl.store(OUT + out_offsets, result, mask=mask_h[:, None]) + + +def fish_decode_kvcache_attn_triton( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_table: torch.Tensor, + seq_lens: torch.Tensor, + out: torch.Tensor, + *, + scale: float, + max_seq_len: int, + small_path_max_seq_len: int, + long_split_tokens: int, + partial_m: torch.Tensor, + partial_l: torch.Tensor, + partial_acc: torch.Tensor, +) -> torch.Tensor: + if not is_available(): + raise RuntimeError(f"Fish Triton attention is unavailable: {_LOAD_ERROR!r}") + batch_size, num_q_heads, head_dim = query.shape + block_size = key_cache.shape[1] + num_kv_heads = key_cache.shape[2] + if head_dim != _HEAD_DIM or block_size != _BLOCK_SIZE: + raise RuntimeError("Fish Triton attention only supports head_dim=128 and block_size=16") + if num_q_heads % num_kv_heads != 0: + raise RuntimeError("num_q_heads must be divisible by num_kv_heads") + + kv_group = num_q_heads // num_kv_heads + block_h = triton.next_power_of_2(kv_group) + max_blocks_per_seq = block_table.shape[1] + + if max_seq_len <= small_path_max_seq_len: + _small_decode_kernel[(batch_size, num_kv_heads)]( + query, + key_cache, + value_cache, + block_table, + seq_lens, + out, + SCALE=float(scale), + MAX_SEQ_LEN=int(max_seq_len), + MAX_BLOCKS_PER_SEQ=max_blocks_per_seq, + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + KV_GROUP=kv_group, + BLOCK_H=block_h, + BLOCK_D=head_dim, + BLOCK_N=_BLOCK_N, + GUARD_MAX_SEQ_LEN=0, + num_warps=4, + num_stages=3, + ) + return out + + _small_decode_kernel[(batch_size, num_kv_heads)]( + query, + key_cache, + value_cache, + block_table, + seq_lens, + out, + SCALE=float(scale), + MAX_SEQ_LEN=small_path_max_seq_len, + MAX_BLOCKS_PER_SEQ=max_blocks_per_seq, + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + KV_GROUP=kv_group, + BLOCK_H=block_h, + BLOCK_D=head_dim, + BLOCK_N=_BLOCK_N, + GUARD_MAX_SEQ_LEN=small_path_max_seq_len, + num_warps=4, + num_stages=3, + ) + num_splits = triton.cdiv(int(max_seq_len), long_split_tokens) + expected_partial_acc_shape = (num_splits, batch_size, num_q_heads, head_dim) + if tuple(partial_acc.shape) != expected_partial_acc_shape: + raise RuntimeError( + "Fish Triton long attention workspace shape mismatch: " + f"expected partial_acc={expected_partial_acc_shape}, got {tuple(partial_acc.shape)}" + ) + _split_partial_decode_kernel[(batch_size, num_kv_heads, num_splits)]( + query, + key_cache, + value_cache, + block_table, + seq_lens, + partial_m, + partial_l, + partial_acc, + SCALE=float(scale), + MAX_BLOCKS_PER_SEQ=max_blocks_per_seq, + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + KV_GROUP=kv_group, + BLOCK_H=block_h, + BLOCK_D=head_dim, + BLOCK_N=_BLOCK_N, + SPLIT_TOKENS=long_split_tokens, + BATCH_SIZE=batch_size, + MIN_SEQ_LEN=small_path_max_seq_len + 1, + num_warps=4, + num_stages=3, + ) + _split_combine_decode_kernel[(batch_size, num_kv_heads)]( + seq_lens, + partial_m, + partial_l, + partial_acc, + out, + NUM_Q_HEADS=num_q_heads, + NUM_KV_HEADS=num_kv_heads, + KV_GROUP=kv_group, + BLOCK_H=block_h, + BLOCK_D=head_dim, + BATCH_SIZE=batch_size, + NUM_SPLITS=num_splits, + MIN_SEQ_LEN=small_path_max_seq_len + 1, + num_warps=4, + num_stages=3, + ) + return out diff --git a/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml new file mode 100644 index 000000000..a65609c6c --- /dev/null +++ b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_dual_gpu.yaml @@ -0,0 +1,68 @@ +# Fish Speech S2 Pro high-concurrency dual-GPU deploy config. +# +# Stage0 SlowAR runs on GPU0 and Stage1 DAC runs on GPU1. Use this config +# when the deployment can trade an extra GPU for lower Stage0/Stage1 +# interference between the AR and DAC stages. +async_chunk: true +enable_chunked_prefill: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 + codec_streaming: true + connector_get_sleep_s: 0.001 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + fish_speech_tensor_codes: true + fish_speech_single_initial_chunk: true + fish_speech_backlog_codec_chunk_frames: 50 + fish_speech_backlog_load_threshold: 0.75 + codec_chunk_frames: 25 + codec_left_context_frames: 25 + initial_codec_chunk_frames: 4 + fish_speech_dac_dtype: fp16 + fish_speech_dac_max_padded_frames: 256 + fish_speech_dac_max_batch: 0 + +stages: + - stage_id: 0 + max_num_seqs: 64 + gpu_memory_utilization: 0.85 + enforce_eager: false + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: false + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.8 + top_k: 30 + top_p: 0.9 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.0 + + - stage_id: 1 + max_num_seqs: 64 + gpu_memory_utilization: 0.25 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: false + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "1" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + repetition_penalty: 1.0 diff --git a/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml new file mode 100644 index 000000000..d771e4163 --- /dev/null +++ b/vllm_omni/deploy/fish_qwen3_omni_high_concurrency_single_gpu.yaml @@ -0,0 +1,68 @@ +# Fish Speech S2 Pro high-concurrency single-GPU deploy config. +# +# Stage0 SlowAR and Stage1 DAC share GPU0. This config keeps the optimized +# Fish runtime path explicit instead of changing the default +# fish_qwen3_omni.yaml deployment. +async_chunk: true +enable_chunked_prefill: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 + codec_streaming: true + connector_get_sleep_s: 0.001 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + fish_speech_tensor_codes: true + fish_speech_single_initial_chunk: true + fish_speech_backlog_codec_chunk_frames: 50 + fish_speech_backlog_load_threshold: 0.75 + codec_chunk_frames: 25 + codec_left_context_frames: 25 + initial_codec_chunk_frames: 4 + fish_speech_dac_dtype: fp16 + fish_speech_dac_max_padded_frames: 256 + fish_speech_dac_max_batch: 0 + +stages: + - stage_id: 0 + max_num_seqs: 64 + gpu_memory_utilization: 0.72 + enforce_eager: false + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: false + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + output_connectors: + to_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.8 + top_k: 30 + top_p: 0.9 + max_tokens: 2048 + seed: 42 + repetition_penalty: 1.0 + + - stage_id: 1 + max_num_seqs: 64 + gpu_memory_utilization: 0.18 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + async_scheduling: false + max_num_batched_tokens: 16384 + max_model_len: 16384 + devices: "0" + input_connectors: + from_stage_0: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + repetition_penalty: 1.0 diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index bc840c739..bda2abeea 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -64,6 +64,21 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self.requests_origin_status = {} self.requests_num_chunks_sent: dict[str, int] = defaultdict(int) + @staticmethod + def _is_truthy_scalar(value: Any) -> bool: + if isinstance(value, torch.Tensor): + return value.numel() == 1 and bool(value.item()) + return bool(value) if value is not None else False + + @staticmethod + def _confirmed_num_computed_tokens(request: Request) -> int: + # vLLM async scheduling advances num_computed_tokens with output + # placeholders before the corresponding token is committed. Connector + # chunk send watermarks must use only committed tokens. + num_computed = int(getattr(request, "num_computed_tokens", 0)) + num_placeholders = int(getattr(request, "num_output_placeholders", 0) or 0) + return max(0, num_computed - num_placeholders) + @classmethod def create_connector(cls, model_config: Any): connector_config = getattr(model_config, "stage_connector_config", None) @@ -119,16 +134,19 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): request: Request object """ + confirmed_num_computed_tokens = self._confirmed_num_computed_tokens(request) + # If the request is preempted, skip the already saved chunks. - if request.num_computed_tokens < self.requests_num_chunks_sent.get(request.external_req_id, 0): + if confirmed_num_computed_tokens < self.requests_num_chunks_sent.get(request.external_req_id, 0): logger.warning( f"Enqueue save_async for request {request.external_req_id}, " f"request.num_computed_tokens={request.num_computed_tokens}, " + f"request.num_output_placeholders={getattr(request, 'num_output_placeholders', 0)}, " f"previous_chunks_sent={self.requests_num_chunks_sent.get(request.external_req_id, 0)}" ) return - self.requests_num_chunks_sent[request.external_req_id] = request.num_computed_tokens + self.requests_num_chunks_sent[request.external_req_id] = confirmed_num_computed_tokens task = { "pooling_output": pooling_output, "request": request, @@ -166,24 +184,36 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self.get_req_chunk[req_id] += 1 meta = payload_data.get("meta", {}) + payload_finished = self._is_truthy_scalar(meta.get("finished")) if self.model_mode == "ar": request.additional_information = payload_data - if meta.get("finished"): + if payload_finished: self.finished_requests.add(req_id) else: - if meta.get("finished"): + if payload_finished: self.finished_requests.add(req_id) new_ids = payload_data.get("codes", {}).get("audio") - if isinstance(new_ids, torch.Tensor): + has_tensor_codes = isinstance(new_ids, torch.Tensor) + use_tensor_codes = has_tensor_codes and new_ids.ndim >= 2 + if use_tensor_codes: + request.prompt_token_ids = [0] if new_ids.numel() > 0 else [] + elif has_tensor_codes: new_ids = new_ids.tolist() elif new_ids is None: new_ids = [] - request.prompt_token_ids = new_ids + request.prompt_token_ids = new_ids + if not use_tensor_codes: + request.prompt_token_ids = new_ids prev_info = getattr(request, "additional_information", None) info = dict(prev_info) if isinstance(prev_info, dict) else {} for key, value in payload_data.items(): if key == "codes": + if use_tensor_codes and isinstance(value, dict): + existing_sub = info.get(key) + merged_sub = dict(existing_sub) if isinstance(existing_sub, dict) else {} + merged_sub.update(value) + info[key] = merged_sub continue if isinstance(value, dict): existing_sub = info.get(key) @@ -199,8 +229,13 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): request.num_computed_tokens = 0 # Empty chunk with more data expected: keep polling. - if not new_ids and not meta.get("finished"): - return True + has_new_ids = bool(new_ids.numel()) if use_tensor_codes else bool(new_ids) + if not has_new_ids and not payload_finished: + # The base recv loop treats False as "not ready yet" and + # requeues the request. Do not mark an empty non-terminal + # chunk as ready, otherwise Stage1 can consume before the + # first DAC frame arrives. + return False # Mark as finished for consumption self._finished_load_reqs.add(req_id) diff --git a/vllm_omni/distributed/omni_connectors/utils/serialization.py b/vllm_omni/distributed/omni_connectors/utils/serialization.py index ae9dc4953..c2145dbff 100644 --- a/vllm_omni/distributed/omni_connectors/utils/serialization.py +++ b/vllm_omni/distributed/omni_connectors/utils/serialization.py @@ -82,10 +82,12 @@ class OmniMsgpackEncoder: def _encode_tensor(self, tensor: torch.Tensor) -> dict[str, Any]: """Encode torch.Tensor to dict.""" - t = tensor.detach().contiguous().cpu() + t = tensor.detach().cpu() # Handle 0-dimensional (scalar) tensors by reshaping to 1D first if t.dim() == 0: t = t.reshape(1) + if not t.is_contiguous(): + t = t.contiguous() t = t.view(torch.uint8) return { _TENSOR_MARKER: True, diff --git a/vllm_omni/model_executor/models/fish_speech/fish_speech_dac_decoder.py b/vllm_omni/model_executor/models/fish_speech/fish_speech_dac_decoder.py index daef2be18..c6e9064f1 100644 --- a/vllm_omni/model_executor/models/fish_speech/fish_speech_dac_decoder.py +++ b/vllm_omni/model_executor/models/fish_speech/fish_speech_dac_decoder.py @@ -11,6 +11,7 @@ Install with: ``pip install fish-speech`` from __future__ import annotations +import inspect import os from collections.abc import Iterable from typing import Any @@ -33,6 +34,41 @@ from vllm_omni.model_executor.models.output_templates import OmniOutput logger = init_logger(__name__) +def _connector_extra_config(vllm_config: VllmConfig) -> dict[str, Any]: + model_config = getattr(vllm_config, "model_config", None) + connector_cfg = getattr(model_config, "stage_connector_config", None) + if isinstance(connector_cfg, dict): + return connector_cfg.get("extra", connector_cfg) + extra = getattr(connector_cfg, "extra", None) + return extra if isinstance(extra, dict) else {} + + +def _get_int_config(extra_cfg: dict[str, Any], default: int, *names: str) -> int: + value = None + for name in names: + if name in extra_cfg: + value = extra_cfg[name] + break + if value is None: + return default + try: + return max(0, int(value)) + except (TypeError, ValueError) as exc: + raise ValueError(f"Invalid Fish Speech DAC integer config {names[0]}={value!r}") from exc + + +def _get_dac_dtype(extra_cfg: dict[str, Any]) -> torch.dtype: + value = extra_cfg.get("fish_speech_dac_dtype", "float32") + value = str(value).strip().lower() + if value in {"fp16", "float16", "half"}: + return torch.float16 + if value in {"bf16", "bfloat16"}: + return torch.bfloat16 + if value in {"fp32", "float32", ""}: + return torch.float32 + raise ValueError(f"Invalid Fish Speech DAC dtype: {value!r}") + + class FishSpeechDACDecoder(nn.Module): """Stage-1 DAC decoder for Fish Speech S2 Pro (GenerationModelRunner). @@ -58,6 +94,43 @@ class FishSpeechDACDecoder(nn.Module): self._output_sample_rate: int = DAC_SAMPLE_RATE self._hop_length: int = DAC_HOP_LENGTH self._logged_codec_stats = False + self._codec_decode_takes_lengths: bool | None = None + extra_cfg = _connector_extra_config(vllm_config) + self._dac_dtype = _get_dac_dtype(extra_cfg) + self._decode_batch_max_padded_frames = _get_int_config( + extra_cfg, + 0, + "fish_speech_dac_max_padded_frames", + "dac_max_padded_frames", + ) + self._decode_batch_max_batch = _get_int_config( + extra_cfg, + 0, + "fish_speech_dac_max_batch", + "dac_max_batch", + ) + + def _decode_codes( + self, + codes_bqf: torch.Tensor, + feature_lengths: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert self._codec is not None + if self._codec_decode_takes_lengths: + return self._codec.decode(codes_bqf, feature_lengths) + + if hasattr(self._codec, "from_indices"): + wav_batch = self._codec.from_indices(codes_bqf) + else: + wav_batch = self._codec.decode(codes_bqf) + max_frames = max(int(codes_bqf.shape[-1]), 1) + scale = wav_batch.shape[-1] / max_frames + audio_lengths = torch.clamp( + torch.round(feature_lengths.to(torch.float32) * scale).to(torch.long), + min=0, + max=wav_batch.shape[-1], + ) + return wav_batch, audio_lengths def _bake_weight_norm(self, codec: nn.Module) -> None: baked = 0 @@ -149,14 +222,16 @@ class FishSpeechDACDecoder(nn.Module): codec.quantizer.downsample = None device = self.vllm_config.device_config.device - codec = codec.to(device=device, dtype=torch.float32) + codec = codec.to(device=device, dtype=self._dac_dtype) codec.eval() self._codec = codec + self._codec_decode_takes_lengths = len(inspect.signature(codec.decode).parameters) >= 2 logger.info( - "Fish Speech DAC codec loaded from %s (device=%s, sample_rate=%d)", + "Fish Speech DAC codec loaded from %s (device=%s, dtype=%s, sample_rate=%d)", codec_path, device, + self._dac_dtype, self._output_sample_rate, ) @@ -188,6 +263,28 @@ class FishSpeechDACDecoder(nn.Module): return [ids[boundaries[i] : boundaries[i + 1]] for i in range(len(boundaries) - 1)] return [ids] + def _codes_from_runtime_info( + self, + info: dict[str, Any] | None, + device: torch.device, + ) -> torch.Tensor | None: + if not isinstance(info, dict): + return None + codes = info.get("codes", {}).get("audio") + if not isinstance(codes, torch.Tensor) or codes.numel() == 0: + return None + + q = self._num_codebooks + codes = codes.to(device=device, dtype=torch.long, non_blocking=True).contiguous() + if codes.ndim != 2 or codes.shape[0] != q: + logger.warning( + "DAC tensor codes must have shape [num_codebooks, frames], got %s for num_codebooks=%d.", + tuple(codes.shape), + q, + ) + return None + return codes + @torch.no_grad() def forward( self, @@ -203,14 +300,14 @@ class FishSpeechDACDecoder(nn.Module): input_ids layout per request: flat codes [num_codebooks * num_frames]. Codes are codebook-major: [cb0_f0, cb0_f1, ..., cb0_fN, cb1_f0, ...]. """ - self._ensure_codec_loaded() - assert self._codec is not None - q = self._num_codebooks sr_val = self._output_sample_rate sr_tensor = torch.tensor(sr_val, dtype=torch.int32) empty = torch.zeros((0,), dtype=torch.float32) + self._ensure_codec_loaded() + assert self._codec is not None + if input_ids is None or input_ids.numel() == 0: return OmniOutput( text_hidden_states=None, @@ -235,9 +332,22 @@ class FishSpeechDACDecoder(nn.Module): left_context_size[i] = meta["left_context_size"] for i, req_ids in enumerate(request_ids_list): + ctx_frames = left_context_size[i] + runtime_codes = ( + self._codes_from_runtime_info(runtime_additional_information[i], ids.device) + if runtime_additional_information is not None and i < len(runtime_additional_information) + else None + ) + if runtime_codes is not None: + frames = runtime_codes.shape[1] + parsed_ctx_frames[i] = ctx_frames + parsed_total_frames[i] = frames + valid_codes_qf.append(runtime_codes) + valid_indices.append(i) + continue + if req_ids.numel() < 1: continue - ctx_frames = left_context_size[i] flat = req_ids n = flat.numel() if n == 0 or n % q != 0: @@ -279,25 +389,61 @@ class FishSpeechDACDecoder(nn.Module): except Exception: pass - feature_lengths = torch.tensor( - [codes_qf.shape[1] for codes_qf in valid_codes_qf], - device=valid_codes_qf[0].device, - dtype=torch.long, - ) - max_frames = int(feature_lengths.max().item()) - batch_size = len(valid_codes_qf) + wav_tensors: list[torch.Tensor | None] = [None] * len(valid_codes_qf) - codes_bqf = torch.zeros( - (batch_size, q, max_frames), - device=valid_codes_qf[0].device, - dtype=torch.long, - ) - for i, codes_qf in enumerate(valid_codes_qf): - frame_count = int(feature_lengths[i].item()) - codes_bqf[i, :, :frame_count] = codes_qf + def _iter_bounded_groups(group: list[tuple[int, torch.Tensor]]) -> Iterable[list[tuple[int, torch.Tensor]]]: + max_padded_frames = int(getattr(self, "_decode_batch_max_padded_frames", 0)) + max_batch = int(getattr(self, "_decode_batch_max_batch", 0)) + if max_padded_frames <= 0 and max_batch <= 0: + yield group + return - with torch.amp.autocast("cuda", enabled=False): - wav_batch, audio_lengths = self._codec.decode(codes_bqf, feature_lengths) + current: list[tuple[int, torch.Tensor]] = [] + current_max_frames = 0 + for item in sorted(group, key=lambda pair: int(pair[1].shape[1])): + frames = int(item[1].shape[1]) + next_max_frames = max(current_max_frames, frames) + next_batch_size = len(current) + 1 + exceeds_batch = max_batch > 0 and next_batch_size > max_batch + exceeds_work = max_padded_frames > 0 and next_batch_size * next_max_frames > max_padded_frames + if current and (exceeds_batch or exceeds_work): + yield current + current = [item] + current_max_frames = frames + else: + current.append(item) + current_max_frames = next_max_frames + if current: + yield current + + def _decode_group(group: list[tuple[int, torch.Tensor]]) -> None: + actual_frames = [int(codes_qf.shape[1]) for _, codes_qf in group] + target_frames = max(actual_frames) + batch_size = len(group) + first_codes = group[0][1] + feature_lengths = torch.tensor(actual_frames, device=first_codes.device, dtype=torch.long) + codes_bqf = torch.zeros( + (batch_size, q, target_frames), + device=first_codes.device, + dtype=torch.long, + ) + for row, (_, codes_qf) in enumerate(group): + frame_count = int(codes_qf.shape[1]) + codes_bqf[row, :, :frame_count] = codes_qf + + with torch.amp.autocast("cuda", enabled=False): + wav_batch, audio_lengths = self._decode_codes(codes_bqf, feature_lengths) + audio_lengths_list = ( + audio_lengths.detach().to(device="cpu", dtype=torch.long).reshape(-1).tolist() + if audio_lengths.numel() > 0 + else [] + ) + for row, (j, _) in enumerate(group): + audio_len = int(audio_lengths_list[row]) if len(audio_lengths_list) > row else int(wav_batch.shape[-1]) + wav_tensors[j] = wav_batch[row, 0, :audio_len] + + for bounded_group in _iter_bounded_groups(list(enumerate(valid_codes_qf))): + _decode_group(bounded_group) audios: list[torch.Tensor] = [empty] * num_req srs = [sr_tensor] * num_req @@ -305,8 +451,8 @@ class FishSpeechDACDecoder(nn.Module): for j, idx in enumerate(valid_indices): ctx_frames = parsed_ctx_frames[idx] total_frames = parsed_total_frames[idx] - audio_len = int(audio_lengths[j].item()) if audio_lengths.numel() > j else int(wav_batch.shape[-1]) - wav = wav_batch[j, 0, :audio_len] + wav = wav_tensors[j] + assert wav is not None # Trim context frames (left overlap for streaming). if ctx_frames > 0: # Decode length may deviate from (frames * hop_length) due to model diff --git a/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py b/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py index 633652c92..46ceee498 100644 --- a/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py +++ b/vllm_omni/model_executor/models/fish_speech/fish_speech_fast_ar.py @@ -4,14 +4,13 @@ autoregressively after each Slow AR step. Analogous to Qwen3 TTS's CodePredictor but with its own embedding table, RoPE, and output head. -Uses re-prefill (no KV cache): each AR step forwards the full growing -sequence through the 4-layer transformer. This trades ~O(T^2) attention -FLOPs (negligible for T=10, 4 layers) for zero KV cache management. +Uses a tiny per-call KV cache: each AR step forwards one codebook token while +reusing the previous Fast AR keys/values. Optimisations: - - torch.compile on model forward (kernel fusion) - Pre-allocated projection / embedding buffer - Pre-allocated position_ids + - Pre-allocated per-layer KV cache for the short codebook loop - Inline sampling """ @@ -51,7 +50,8 @@ logger = init_logger(__name__) class _FastARAttention(nn.Module): """Multi-head attention using F.scaled_dot_product_attention (SDPA). - Supports fused QKV, RoPE, optional q/k normalization, and native GQA. + Supports fused QKV, RoPE, optional q/k normalization, native GQA, and a + small one-token decode path for the residual codebook loop. Input: [B, seq_len, hidden_size]. """ @@ -124,6 +124,43 @@ class _FastARAttention(nn.Module): output, _ = self.o_proj(attn_out) return output.view(bsz, seq_len, -1) + def forward_one( + self, + hidden_states: torch.Tensor, + position_ids: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + cache_pos: int, + ) -> torch.Tensor: + bsz = int(hidden_states.shape[0]) + + qkv, _ = self.qkv_proj(hidden_states.reshape(bsz, -1)) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + if self.q_norm is not None: + q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape) + k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape) + + q, k = self.rotary_emb(position_ids.reshape(-1), q, k) + + q = q.view(bsz, self.num_heads, self.head_dim).unsqueeze(2) + k = k.view(bsz, self.num_kv_heads, self.head_dim) + v = v.view(bsz, self.num_kv_heads, self.head_dim) + k_cache[:bsz, :, cache_pos, :] = k + v_cache[:bsz, :, cache_pos, :] = v + + attn_out = F.scaled_dot_product_attention( + q, + k_cache[:bsz, :, : cache_pos + 1, :], + v_cache[:bsz, :, : cache_pos + 1, :], + scale=self.scaling, + is_causal=False, + enable_gqa=self._use_gqa, + ) + attn_out = attn_out.transpose(1, 2).reshape(bsz, -1) + output, _ = self.o_proj(attn_out) + return output + class _FastARMLP(nn.Module): """SiLU-gated MLP, matching Qwen3/LLaMA MLP structure.""" @@ -175,6 +212,25 @@ class _FastARDecoderLayer(nn.Module): hidden_states = residual + hidden_states return hidden_states + def forward_one( + self, + hidden_states: torch.Tensor, + position_ids: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + cache_pos: int, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn.forward_one(hidden_states, position_ids, k_cache, v_cache, cache_pos) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + # =================================================================== # Fast AR Transformer Model @@ -199,6 +255,25 @@ class FishSpeechFastARModel(nn.Module): hidden_states = layer(hidden_states, position_ids) return hidden_states + def forward_one( + self, + input_embed: torch.Tensor, + position_ids: torch.Tensor, + k_caches: torch.Tensor, + v_caches: torch.Tensor, + cache_pos: int, + ) -> torch.Tensor: + hidden_states = input_embed + for layer_idx, layer in enumerate(self.layers): + hidden_states = layer.forward_one( + hidden_states, + position_ids, + k_caches[layer_idx], + v_caches[layer_idx], + cache_pos, + ) + return hidden_states + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ ("qkv_proj", "q_proj", "q"), @@ -259,13 +334,13 @@ class FishSpeechFastARModel(nn.Module): class FishSpeechFastAR(nn.Module): """vLLM-native Fast AR for Fish Speech (residual codebooks). - Re-prefill approach: each AR step forwards the full growing sequence - through the 4-layer transformer. No KV cache needed. + Decode approach: each residual-codebook step forwards one token through + the 4-layer transformer with a per-call KV cache. Optimisations over baseline: - 1. torch.compile on model forward -- kernel fusion. - 2. Pre-allocated embedding buffer [B, max_seq, H]. - 3. Pre-allocated position_ids. + 1. Pre-allocated embedding buffer [B, max_seq, H]. + 2. Pre-allocated position_ids. + 3. Per-call KV cache for the short residual-codebook loop. 4. Inline sampling. """ @@ -307,6 +382,8 @@ class FishSpeechFastAR(nn.Module): # Pre-allocated buffers (lazily initialised on first forward). self._embed_buf: torch.Tensor | None = None self._pos_ids: torch.Tensor | None = None + self._k_cache: torch.Tensor | None = None + self._v_cache: torch.Tensor | None = None self._compiled_model_fwd: object | None = None self._compile_attempted = False self._compile_failed = False @@ -316,13 +393,27 @@ class FishSpeechFastAR(nn.Module): max_seq = self._num_codebooks + 1 # hidden_state + num_codebooks codes if ( self._embed_buf is not None + and self._pos_ids is not None + and self._k_cache is not None + and self._v_cache is not None and self._embed_buf.shape[0] >= bsz and self._embed_buf.device == device and self._embed_buf.dtype == dtype + and self._pos_ids.shape[0] >= bsz + and self._pos_ids.shape[1] == max_seq + and self._pos_ids.device == device + and self._k_cache.shape[1] >= bsz + and self._k_cache.device == device + and self._k_cache.dtype == dtype ): return self._embed_buf = torch.zeros(bsz, max_seq, self._fast_dim, dtype=dtype, device=device) - self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device) + self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device).unsqueeze(0).expand(bsz, -1).contiguous() + num_layers = self.config.num_hidden_layers + num_kv_heads = self.config.num_key_value_heads + head_dim = self.config.head_dim + self._k_cache = torch.empty(num_layers, bsz, num_kv_heads, max_seq, head_dim, dtype=dtype, device=device) + self._v_cache = torch.empty_like(self._k_cache) def _setup_compile(self) -> None: if self._compile_attempted: @@ -375,20 +466,10 @@ class FishSpeechFastAR(nn.Module): torch.accelerator.synchronize(device) @torch.inference_mode() - def _run_model(self, step_input: torch.Tensor, step_pos_ids: torch.Tensor, bsz: int) -> torch.Tensor: - if self._disable_compile_for_graph: - model_fwd = self._compiled_model_fwd or self.model.forward - else: - model_fwd = self._compiled_model_fwd if bsz == 1 else self.model.forward - try: - return model_fwd(step_input, step_pos_ids) - except Exception as exc: - if model_fwd is self.model.forward or self._compile_failed: - raise - self._compile_failed = True - self._compiled_model_fwd = self.model.forward - logger.warning("Fish Speech Fast AR torch.compile fallback to eager after runtime failure: %s", exc) - return self.model.forward(step_input, step_pos_ids) + def _run_model_one(self, input_embed: torch.Tensor, step_pos_ids: torch.Tensor, cache_pos: int) -> torch.Tensor: + assert self._k_cache is not None + assert self._v_cache is not None + return self.model.forward_one(input_embed, step_pos_ids, self._k_cache, self._v_cache, cache_pos) @torch.inference_mode() def forward( @@ -401,6 +482,7 @@ class FishSpeechFastAR(nn.Module): top_k: int = 30, top_p: float = 0.9, seed: int | None = None, + generator: torch.Generator | None = None, ) -> torch.Tensor: """Predict residual codebook codes 0..num_codebooks-1 autoregressively. @@ -428,7 +510,6 @@ class FishSpeechFastAR(nn.Module): all_codes[:, 0] = semantic_code self._ensure_buffers(bsz, device, dtype) - self._setup_compile() embed_buf = self._embed_buf pos_ids = self._pos_ids @@ -445,8 +526,7 @@ class FishSpeechFastAR(nn.Module): inv_temperature = 1.0 / max(temperature, 1e-6) if use_sampling else 0.0 # Create a seeded generator for deterministic residual codebook sampling. - generator = None - if seed is not None and use_sampling: + if generator is None and seed is not None and use_sampling: generator = torch.Generator(device=device) generator.manual_seed(seed) @@ -455,15 +535,11 @@ class FishSpeechFastAR(nn.Module): # codebooks only have 1024 entries. Truncate logits for steps > 0. residual_codebook_size = 1024 - for step in range(1, num_cb): - seq_len = step + 1 - step_input = embed_buf[:bsz, :seq_len, :] - # Use a dense 2D position tensor for every batch size; stride-0 - # views from expand() were fragile under compiled execution. - step_pos_ids = pos_ids[:seq_len].unsqueeze(0).repeat(bsz, 1) + self._run_model_one(embed_buf[:bsz, 0, :], pos_ids[:bsz, 0], 0) - hidden_out = self._run_model(step_input, step_pos_ids, bsz) - logits = self.fast_output(self.fast_norm(hidden_out[:, -1, :])) + for step in range(1, num_cb): + hidden_out = self._run_model_one(embed_buf[:bsz, step, :], pos_ids[:bsz, step], step) + logits = self.fast_output(self.fast_norm(hidden_out)) # Residual codebooks (step >= 1) only have 1024 entries. if step >= 1: diff --git a/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py b/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py index 24182b93e..1317dc044 100644 --- a/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py +++ b/vllm_omni/model_executor/models/fish_speech/fish_speech_slow_ar.py @@ -199,6 +199,12 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module): # Qwen3 transformer backbone. self.model = Qwen3Model(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")) + from vllm_omni.attention.fish_kvcache_attn import is_fish_kvcache_attn_enabled + + if is_fish_kvcache_attn_enabled(): + from vllm_omni.attention.fish_kvcache_backend import install_fish_kvcache_attn_backend + + install_fish_kvcache_attn_backend(self.model) # Fish Speech uses interleaved (GPT-J) RoPE, not NeoX style. # vLLM's Qwen3Attention defaults to NeoX (is_neox_style=True). @@ -672,14 +678,19 @@ class FishSpeechSlowARForConditionalGeneration(nn.Module): past_hidden = last_talker_hidden.reshape(bsz, -1).to(dtype=torch.bfloat16, device=dev) # Run Fast AR to predict all num_codebooks codes. + do_sample = kwargs.get("do_sample") + temperature = kwargs.get("temperature") + top_k = kwargs.get("top_k") + top_p = kwargs.get("top_p") audio_codes = self.fast_ar( slow_ar_hidden=past_hidden, semantic_token_id=input_ids.reshape(bsz), - do_sample=True, - temperature=0.8, - top_k=30, - top_p=0.9, + do_sample=True if do_sample is None else bool(do_sample), + temperature=0.8 if temperature is None else float(temperature), + top_k=30 if top_k is None else int(top_k), + top_p=0.9 if top_p is None else float(top_p), seed=seed, + generator=kwargs.get("generator"), ) # [B, num_codebooks] # Add codebook embeddings to the input embedding (from preprocess). diff --git a/vllm_omni/model_executor/stage_input_processors/fish_speech.py b/vllm_omni/model_executor/stage_input_processors/fish_speech.py index 6eac326a3..ac12e6035 100644 --- a/vllm_omni/model_executor/stage_input_processors/fish_speech.py +++ b/vllm_omni/model_executor/stage_input_processors/fish_speech.py @@ -3,7 +3,6 @@ from typing import Any import torch -from vllm.logger import init_logger from vllm_omni.data_entry_keys import ( CodesStruct, @@ -11,7 +10,85 @@ from vllm_omni.data_entry_keys import ( OmniPayloadStruct, ) -logger = init_logger(__name__) + +def _get_connector_extra(transfer_manager: Any) -> dict[str, Any]: + connector = getattr(transfer_manager, "connector", None) + raw_cfg = getattr(connector, "config", {}) or {} + return raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {} + + +def _use_tensor_code_payload(transfer_manager: Any) -> bool: + cfg = _get_connector_extra(transfer_manager) + return bool(cfg.get("fish_speech_tensor_codes", False)) + + +def _cfg_bool(cfg: dict[str, Any], key: str, default: bool = False) -> bool: + return bool(cfg.get(key, default)) + + +def _cfg_int(cfg: dict[str, Any], key: str, default: int = 0) -> int: + value = cfg.get(key, default) + try: + return int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"Invalid Fish Speech integer config {key}={value!r}") from exc + + +def _cfg_float(cfg: dict[str, Any], key: str, default: float) -> float: + value = cfg.get(key, default) + try: + return float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"Invalid Fish Speech float config {key}={value!r}") from exc + + +def _active_codec_requests(transfer_manager: Any) -> int: + return sum(1 for value in transfer_manager.code_prompt_token_ids.values() if len(value) > 0) + + +def _ready_codec_requests(transfer_manager: Any, min_frames: int) -> int: + return sum(1 for value in transfer_manager.code_prompt_token_ids.values() if len(value) >= min_frames) + + +def _select_backlog_chunk_size( + transfer_manager: Any, + cfg: dict[str, Any], + base_chunk_size: int, + current_length: int, +) -> int: + backlog_chunk_size = _cfg_int( + cfg, + "fish_speech_backlog_codec_chunk_frames", + 0, + ) + if backlog_chunk_size <= base_chunk_size: + return base_chunk_size + + capacity = max(1, int(getattr(transfer_manager, "scheduler_max_num_seqs", 1))) + active = _active_codec_requests(transfer_manager) + load_factor = min(active / capacity, 1.0) + threshold = _cfg_float( + cfg, + "fish_speech_backlog_load_threshold", + 0.75, + ) + if load_factor < threshold: + return base_chunk_size + + ready_min = _cfg_int( + cfg, + "fish_speech_backlog_min_ready", + max(2, min(capacity, int(capacity * threshold))), + ) + # Include the current request even when the caller has not yet committed + # the latest frame into the transfer-manager map. + ready = _ready_codec_requests(transfer_manager, base_chunk_size) + if current_length >= base_chunk_size: + ready = max(ready, 1) + if ready < ready_min: + return base_chunk_size + + return backlog_chunk_size def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None: @@ -21,11 +98,18 @@ def _extract_last_frame(pooling_output: dict[str, Any]) -> torch.Tensor | None: return None if audio_codes.ndim == 2: frame = audio_codes[-1] - if frame.numel() == 0 or not bool(frame.any().item()): + valid = pooling_output.get("audio_code_valid") + if isinstance(valid, torch.Tensor) and valid.numel() > 0: + is_valid = bool(valid.reshape(-1)[-1].item()) + elif valid is not None: + is_valid = bool(valid) + else: + is_valid = bool(frame.any().item()) + if frame.numel() == 0 or not is_valid: return None - return frame.to(torch.long).reshape(-1) + return frame.to(device="cpu", dtype=torch.long).reshape(-1) if audio_codes.ndim == 1: - return audio_codes.to(torch.long).reshape(-1) + return audio_codes.to(device="cpu", dtype=torch.long).reshape(-1) raise ValueError(f"Invalid audio_codes shape for Fish Speech async_chunk: {tuple(audio_codes.shape)}") @@ -74,20 +158,20 @@ def slow_ar_to_dac_decoder_async_chunk( """ request_id = request.external_req_id finished = bool(is_finished or request.is_finished()) + cfg = _get_connector_extra(transfer_manager) if isinstance(pooling_output, dict): frame = _extract_last_frame(pooling_output) if frame is not None: - transfer_manager.code_prompt_token_ids[request_id].append(frame.detach().to(device="cpu", dtype=torch.long)) + transfer_manager.code_prompt_token_ids[request_id].append(frame.detach()) elif not finished: return None - connector = getattr(transfer_manager, "connector", None) - raw_cfg = getattr(connector, "config", {}) or {} - cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {} chunk_size = int(cfg.get("codec_chunk_frames", 25)) left_context_size_config = int(cfg.get("codec_left_context_frames", 25)) - initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0)) + configured_initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0)) + + initial_chunk_size = configured_initial_chunk_size # Per-request override. additional_information = getattr(request, "additional_information", None) @@ -100,7 +184,7 @@ def slow_ar_to_dac_decoder_async_chunk( if entry.list_data is not None and len(entry.list_data) == 1: initial_chunk_size = int(entry.list_data[0]) - if chunk_size <= 0 or left_context_size_config < 0 or initial_chunk_size < 0: + if chunk_size <= 0 or left_context_size_config < 0 or configured_initial_chunk_size < 0 or initial_chunk_size < 0: raise ValueError( f"Invalid codec chunk config: codec_chunk_frames={chunk_size}, " f"codec_left_context_frames={left_context_size_config}, " @@ -110,6 +194,7 @@ def slow_ar_to_dac_decoder_async_chunk( initial_chunk_size = chunk_size length = len(transfer_manager.code_prompt_token_ids[request_id]) + steady_chunk_size = _select_backlog_chunk_size(transfer_manager, cfg, chunk_size, length) if length <= 0: if finished: @@ -119,9 +204,30 @@ def slow_ar_to_dac_decoder_async_chunk( ) return None - in_initial_phase = initial_chunk_size > 0 and length <= chunk_size + single_initial_chunk = _cfg_bool( + cfg, + "fish_speech_single_initial_chunk", + False, + ) + use_first_chunk = initial_chunk_size > 0 and initial_chunk_size < steady_chunk_size - if in_initial_phase: + if single_initial_chunk and use_first_chunk: + if length <= initial_chunk_size: + if not finished and length < initial_chunk_size: + return None + context_length = length if finished and length < initial_chunk_size else initial_chunk_size + else: + adjusted = length - initial_chunk_size + if adjusted <= 0: + return None + if not finished and adjusted % steady_chunk_size != 0: + return None + chunk_length = adjusted % steady_chunk_size + context_length = chunk_length if chunk_length != 0 else steady_chunk_size + end_index = min(length, left_context_size_config + context_length) + left_context_size = max(0, int(end_index - context_length)) + window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:] + elif initial_chunk_size > 0 and length <= chunk_size: already_sent = transfer_manager.put_req_chunk[request_id] * initial_chunk_size pending = length - already_sent if pending <= 0: @@ -134,17 +240,23 @@ def slow_ar_to_dac_decoder_async_chunk( else: initial_coverage = (chunk_size // initial_chunk_size) * initial_chunk_size if initial_chunk_size > 0 else 0 adjusted = length - initial_coverage - chunk_length = adjusted % chunk_size + chunk_length = adjusted % steady_chunk_size if chunk_length != 0 and not finished: return None - context_length = chunk_length if chunk_length != 0 else chunk_size + context_length = chunk_length if chunk_length != 0 else steady_chunk_size end_index = min(length, left_context_size_config + context_length) left_context_size = max(0, int(end_index - context_length)) window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:] - # Pack into codebook-major flat codes. + # Pack into codebook-major codes. The tensor path avoids expanding codec + # indices into Python ints across the connector boundary; Stage1 schedules + # a single placeholder token and consumes the real codes from runtime info. stacked_frames = torch.stack(window_frames, dim=0) - code_predictor_codes = stacked_frames.transpose(0, 1).reshape(-1) + codes_qf = stacked_frames.transpose(0, 1).contiguous() + if _use_tensor_code_payload(transfer_manager): + code_predictor_codes = codes_qf + else: + code_predictor_codes = codes_qf.reshape(-1) return OmniPayloadStruct( codes=CodesStruct(audio=code_predictor_codes), diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index b3fa99fc5..5da4cf9d8 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -509,6 +509,14 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin): cascade_attn_prefix_lens=cascade_attn_prefix_lens, slot_mappings=slot_mappings_by_group, ) + self._maybe_attach_attention_metadata_extensions( + attn_metadata=attn_metadata, + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + max_query_len=max_num_scheduled_tokens, + pad_attn=pad_attn, + num_scheduled_tokens_np=num_scheduled_tokens_np, + ) ( input_ids, diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 5a5175dc0..175547ff3 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -278,6 +278,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin cascade_attn_prefix_lens=cascade_attn_prefix_lens, slot_mappings=slot_mappings_by_group, ) + self._maybe_attach_attention_metadata_extensions( + attn_metadata=attn_metadata, + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + max_query_len=max_num_scheduled_tokens, + pad_attn=pad_attn, + num_scheduled_tokens_np=num_scheduled_tokens_np, + ) ( input_ids, @@ -730,6 +738,14 @@ class GPUGenerationModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin slot_mappings=slot_mappings_by_group, use_spec_decode=self.speculative_config is not None, ) + self._maybe_attach_attention_metadata_extensions( + attn_metadata=attn_metadata, + num_reqs=num_reqs_padded, + num_reqs_padded=num_reqs_padded, + max_query_len=max_query_len, + pad_attn=True, + for_cudagraph_capture=is_graph_capturing, + ) with self.maybe_dummy_run_with_lora( self.lora_config, diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 9ba0b5f7b..0f164fea6 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -96,6 +96,7 @@ class OmniGPUModelRunner(GPUModelRunner): @instrument(span_name="Loading (GPU)") def load_model(self, *args, **kwargs) -> None: super().load_model(*args, **kwargs) + self._prewarm_attention_capture_workspaces() # TODO move this model specific logic to a separate class # TTS model IS the talker (no .talker sub-attr); use getattr to support both Omni and TTS. @@ -122,6 +123,45 @@ class OmniGPUModelRunner(GPUModelRunner): self.last_talker_hidden = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False) self.text_step = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False) + def _prewarm_attention_capture_workspaces(self) -> None: + capture_sizes = getattr(self.compilation_config, "cudagraph_capture_sizes", None) + if not capture_sizes: + return + from vllm_omni.attention.fish_kvcache_backend import prewarm_fish_kvcache_attn_capture_workspaces + + prewarm_fish_kvcache_attn_capture_workspaces( + model_config=self.model_config, + device=self.device, + dtype=self.dtype, + capture_sizes=capture_sizes, + ) + + def _maybe_attach_attention_metadata_extensions( + self, + *, + attn_metadata: Any, + num_reqs: int, + num_reqs_padded: int, + max_query_len: int, + pad_attn: bool, + for_cudagraph_capture: bool = False, + num_scheduled_tokens_np: np.ndarray | None = None, + ) -> None: + from vllm_omni.attention.fish_kvcache_backend import maybe_attach_fish_kvcache_seq_lens_upper_bound + + maybe_attach_fish_kvcache_seq_lens_upper_bound( + model_config=self.model_config, + attn_metadata=attn_metadata, + input_batch=self.input_batch, + optimistic_seq_lens_cpu=self.optimistic_seq_lens_cpu, + num_reqs=num_reqs, + num_reqs_padded=num_reqs_padded, + max_query_len=max_query_len, + pad_attn=pad_attn, + for_cudagraph_capture=for_cudagraph_capture, + num_scheduled_tokens_np=num_scheduled_tokens_np, + ) + def _build_model_sampler_output_token_ids(self) -> list[list[int]]: """Build decoded-token history for ``prefer_model_sampler`` models. @@ -898,6 +938,14 @@ class OmniGPUModelRunner(GPUModelRunner): slot_mappings=slot_mappings_by_group, use_spec_decode=self.speculative_config is not None, ) + self._maybe_attach_attention_metadata_extensions( + attn_metadata=attn_metadata, + num_reqs=num_reqs_padded, + num_reqs_padded=num_reqs_padded, + max_query_len=max_query_len, + pad_attn=True, + for_cudagraph_capture=is_graph_capturing, + ) with self.maybe_dummy_run_with_lora( self.lora_config,