[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
22 files changed
+1473
-38
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Tests for the universal benchmarks/tts/bench_tts.py CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -51,6 +54,73 @@ def test_load_model_configs(model_configs_path: Path) -> None:
|
||||
assert configs["test/ModelA"]["supported_tasks"] == ["voice_clone", "default_voice"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["tencent/AuK", "tencent/AuK-Flash"])
|
||||
@pytest.mark.parametrize("task", ["default_voice", "voice_clone"])
|
||||
def test_auk_benchmark_duration_and_local_bundle(model, task):
|
||||
config = bench_tts.load_model_configs(bench_tts._DEFAULT_MODEL_CONFIGS)[model]
|
||||
cmd = bench_tts.build_bench_args(
|
||||
host="localhost",
|
||||
port=8000,
|
||||
model=model,
|
||||
task=task,
|
||||
model_cfg=config,
|
||||
locale="en",
|
||||
num_prompts=20,
|
||||
concurrency=8,
|
||||
dataset_path="/data/seed-tts",
|
||||
wer_eval=False,
|
||||
output_dir=None,
|
||||
result_filename=None,
|
||||
extra_cli_args=[],
|
||||
served_model_name="/models/auk",
|
||||
duration_seconds=3.5,
|
||||
request_seed=7,
|
||||
)
|
||||
assert cmd[cmd.index("--model") + 1] == "/models/auk"
|
||||
body = json.loads(cmd[cmd.index("--extra-body") + 1])
|
||||
assert body == {
|
||||
"voice": "default",
|
||||
"duration_seconds": 3.5,
|
||||
"seed": 7,
|
||||
"task_type": "CustomVoice" if task == "default_voice" else "Base",
|
||||
}
|
||||
assert config["task_extra_body"][task]["duration_seconds"] == 5.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("task", "expected_task_type"),
|
||||
[
|
||||
("default_voice", "CustomVoice"),
|
||||
("voice_clone", "Base"),
|
||||
],
|
||||
)
|
||||
def test_auk_task_type_survives_served_model_alias(task, expected_task_type):
|
||||
model = "tencent/AuK-Flash"
|
||||
served_model = "models/flash"
|
||||
config = bench_tts.load_model_configs(bench_tts._DEFAULT_MODEL_CONFIGS)[model]
|
||||
cmd = bench_tts.build_bench_args(
|
||||
host="localhost",
|
||||
port=8000,
|
||||
model=model,
|
||||
task=task,
|
||||
model_cfg=config,
|
||||
locale="en",
|
||||
num_prompts=1,
|
||||
concurrency=1,
|
||||
dataset_path="/data/seed-tts",
|
||||
wer_eval=False,
|
||||
output_dir=None,
|
||||
result_filename=None,
|
||||
extra_cli_args=[],
|
||||
served_model_name=served_model,
|
||||
)
|
||||
|
||||
assert cmd[cmd.index("--model") + 1] == served_model
|
||||
extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
|
||||
assert extra_body["task_type"] == expected_task_type
|
||||
assert "_vllm_omni_benchmark_model" not in extra_body
|
||||
|
||||
|
||||
def test_indextts25_is_registered_in_shared_model_configs() -> None:
|
||||
config_path = Path(bench_tts.__file__).with_name("model_configs.yaml")
|
||||
config = bench_tts.load_model_configs(config_path)["IndexTeam/IndexTTS-2.5"]
|
||||
@@ -173,3 +243,20 @@ def test_unsupported_task_exits(model_configs_path: Path, capsys: pytest.Capture
|
||||
)
|
||||
with pytest.raises(SystemExit):
|
||||
bench_tts.main()
|
||||
|
||||
|
||||
def test_summary_table_includes_task_type(capsys: pytest.CaptureFixture) -> None:
|
||||
bench_tts.print_summary_table(
|
||||
[
|
||||
{
|
||||
"_task": "voice_clone",
|
||||
"_task_type": "Base",
|
||||
"_concurrency": 1,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
output = capsys.readouterr().out
|
||||
assert "Task Type" in output
|
||||
assert "voice_clone" in output
|
||||
assert "Base" in output
|
||||
@@ -0,0 +1,225 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Single-request eager and CUDA-graph AuK DiT sampling equivalence."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, sample_latents
|
||||
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_dit(device: str) -> AuKTransformer:
|
||||
torch.manual_seed(12)
|
||||
return (
|
||||
AuKTransformer(
|
||||
dim=32,
|
||||
heads=2,
|
||||
dim_head=16,
|
||||
ff_mult=2,
|
||||
latent_dim=4,
|
||||
text_hidden_dim=8,
|
||||
num_layers=2,
|
||||
num_single_layers=2,
|
||||
)
|
||||
.eval()
|
||||
.to(device)
|
||||
)
|
||||
|
||||
|
||||
def _sample_inputs(device: str, offset: float = 0.0) -> dict[str, torch.Tensor]:
|
||||
generator = torch.Generator(device=device).manual_seed(42)
|
||||
return {
|
||||
"text": torch.randn(1, 7, 8, generator=generator, device=device) + offset,
|
||||
"c_mask": torch.ones(1, 7, dtype=torch.bool, device=device),
|
||||
"ref": torch.randn(1, 4, 4, generator=generator, device=device) - offset,
|
||||
"ref_mask": torch.ones(1, 4, dtype=torch.bool, device=device),
|
||||
}
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
|
||||
def test_single_request_graph_wrapper_cpu_falls_back_to_eager(cfg_strength: float, mocker) -> None:
|
||||
dit = _make_dit("cpu")
|
||||
wrapper = AuKCUDAGraphWrapper(dit)
|
||||
run_spy = mocker.spy(wrapper, "_run")
|
||||
run_cfg_spy = mocker.spy(wrapper, "_run_cfg")
|
||||
capture_spy = mocker.spy(wrapper, "_capture")
|
||||
inputs = _sample_inputs("cpu")
|
||||
common = dict(
|
||||
**inputs,
|
||||
gen_frames=9,
|
||||
t_grid=[0.0, 0.4, 1.0],
|
||||
cfg_strength=cfg_strength,
|
||||
)
|
||||
eager = sample_latents(dit, **common, generator=torch.Generator().manual_seed(7))
|
||||
graph = sample_latents(dit, **common, generator=torch.Generator().manual_seed(7), sampler=wrapper)
|
||||
torch.testing.assert_close(graph, eager)
|
||||
|
||||
if cfg_strength >= 1e-5:
|
||||
run_spy.assert_not_called()
|
||||
assert run_cfg_spy.call_count == 2
|
||||
else:
|
||||
assert run_spy.call_count == 2
|
||||
run_cfg_spy.assert_not_called()
|
||||
|
||||
capture_spy.assert_not_called()
|
||||
assert not wrapper._cache
|
||||
|
||||
|
||||
def test_graph_inputs_use_bounded_length_buckets() -> None:
|
||||
wrapper = AuKCUDAGraphWrapper(_make_dit("cpu"))
|
||||
x = torch.ones(1, 65, 4)
|
||||
text = torch.ones(1, 65, 8)
|
||||
c_mask = torch.ones(1, 65, dtype=torch.bool)
|
||||
ref = torch.ones(1, 51, 4)
|
||||
ref_mask = torch.ones(1, 51, dtype=torch.bool)
|
||||
|
||||
padded = wrapper._bucket_inputs(x, text, c_mask, ref, ref_mask)
|
||||
|
||||
assert padded[0].shape == (1, 128, 4)
|
||||
assert padded[1].shape == (1, 128)
|
||||
assert padded[2].shape == (1, 128, 8)
|
||||
assert padded[3].shape == (1, 128)
|
||||
assert padded[4].shape == (1, 100, 4)
|
||||
assert padded[5].shape == (1, 100)
|
||||
assert [mask.sum().item() for mask in (padded[1], padded[3], padded[5])] == [65, 65, 51]
|
||||
assert wrapper._key(padded[0], padded[2], padded[4], False) == (128, 128, 100, False)
|
||||
assert wrapper.max_graphs == 32
|
||||
|
||||
|
||||
def test_full_graph_cache_retires_as_one_generation() -> None:
|
||||
wrapper = AuKCUDAGraphWrapper(_make_dit("cpu"), max_graphs=2)
|
||||
first = (64, 64, 50, False)
|
||||
second = (128, 64, 50, False)
|
||||
wrapper._cache[first] = object()
|
||||
|
||||
wrapper._retire_graph_generation_if_full()
|
||||
assert list(wrapper._cache) == [first]
|
||||
|
||||
wrapper._cache[second] = object()
|
||||
wrapper._retire_graph_generation_if_full()
|
||||
assert not wrapper._cache
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
|
||||
def test_bucket_padding_preserves_real_frame_outputs(cfg_strength: float) -> None:
|
||||
dit = _make_dit("cpu")
|
||||
wrapper = AuKCUDAGraphWrapper(dit)
|
||||
inputs = _sample_inputs("cpu")
|
||||
x = torch.randn(1, 9, 4)
|
||||
timestep = torch.tensor(0.4)
|
||||
cfg = torch.tensor(cfg_strength)
|
||||
if cfg_strength >= 1e-5:
|
||||
eager = wrapper._run_cfg(
|
||||
x,
|
||||
None,
|
||||
inputs["text"],
|
||||
inputs["c_mask"],
|
||||
inputs["ref"],
|
||||
inputs["ref_mask"],
|
||||
timestep,
|
||||
cfg_strength=cfg,
|
||||
)
|
||||
else:
|
||||
eager = wrapper._run(
|
||||
x,
|
||||
None,
|
||||
inputs["text"],
|
||||
inputs["c_mask"],
|
||||
inputs["ref"],
|
||||
inputs["ref_mask"],
|
||||
timestep,
|
||||
)
|
||||
dit.clear_cache()
|
||||
|
||||
bucketed = wrapper._bucket_inputs(x, inputs["text"], inputs["c_mask"], inputs["ref"], inputs["ref_mask"])
|
||||
if cfg_strength >= 1e-5:
|
||||
padded = wrapper._run_cfg(*bucketed, timestep, cfg_strength=cfg)
|
||||
else:
|
||||
padded = wrapper._run(*bucketed, timestep)
|
||||
dit.clear_cache()
|
||||
|
||||
torch.testing.assert_close(padded[:, : x.shape[1]], eager)
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
|
||||
def test_single_request_graph_replay_matches_eager_and_updates_inputs(cfg_strength: float) -> None:
|
||||
dit = _make_dit("cuda")
|
||||
wrapper = AuKCUDAGraphWrapper(dit)
|
||||
|
||||
for offset in (0.0, 0.25):
|
||||
inputs = _sample_inputs("cuda", offset)
|
||||
common = dict(
|
||||
**inputs,
|
||||
gen_frames=9,
|
||||
t_grid=[0.0, 0.4, 1.0],
|
||||
cfg_strength=cfg_strength,
|
||||
)
|
||||
eager = sample_latents(dit, **common, generator=torch.Generator(device="cuda").manual_seed(7))
|
||||
graph = sample_latents(dit, **common, generator=torch.Generator(device="cuda").manual_seed(7), sampler=wrapper)
|
||||
torch.testing.assert_close(graph, eager, atol=3e-6, rtol=3e-5)
|
||||
|
||||
bucketed = wrapper._bucket_inputs(
|
||||
torch.empty(1, 9, 4, device="cuda"),
|
||||
inputs["text"],
|
||||
inputs["c_mask"],
|
||||
inputs["ref"],
|
||||
inputs["ref_mask"],
|
||||
)
|
||||
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5)
|
||||
assert key in wrapper._cache
|
||||
|
||||
entry = wrapper._cache[key]
|
||||
torch.testing.assert_close(entry.static_x_mask, bucketed[1])
|
||||
torch.testing.assert_close(entry.static_text, bucketed[2])
|
||||
torch.testing.assert_close(entry.static_c_mask, bucketed[3])
|
||||
torch.testing.assert_close(entry.static_ref, bucketed[4])
|
||||
torch.testing.assert_close(entry.static_ref_mask, bucketed[5])
|
||||
torch.testing.assert_close(entry.static_timestep, torch.tensor(0.4, device="cuda"))
|
||||
|
||||
assert len(wrapper._cache) == 1
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
|
||||
@torch.inference_mode()
|
||||
def test_graph_capture_failure_is_propagated(mocker) -> None:
|
||||
dit = _make_dit("cuda")
|
||||
wrapper = AuKCUDAGraphWrapper(dit)
|
||||
capture = mocker.patch.object(wrapper, "_capture", side_effect=RuntimeError("capture failed"))
|
||||
eager = mocker.spy(wrapper, "_run")
|
||||
inputs = _sample_inputs("cuda")
|
||||
common = dict(
|
||||
**inputs,
|
||||
gen_frames=9,
|
||||
t_grid=[0.0, 0.4, 1.0],
|
||||
cfg_strength=0.0,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="capture failed"):
|
||||
sample_latents(
|
||||
dit,
|
||||
**common,
|
||||
generator=torch.Generator(device="cuda").manual_seed(7),
|
||||
sampler=wrapper,
|
||||
)
|
||||
|
||||
result = sample_latents(
|
||||
dit,
|
||||
**common,
|
||||
generator=torch.Generator(device="cuda").manual_seed(7),
|
||||
sampler=wrapper,
|
||||
)
|
||||
|
||||
assert result.shape == (1, 9, 4)
|
||||
assert capture.call_count == 1
|
||||
assert eager.call_count == 2
|
||||
assert not wrapper.enabled
|
||||
assert not wrapper._cache
|
||||
@@ -227,13 +227,16 @@ def build_pipeline(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(pipeline_auk, "AuKTransformer", _StubTransformer)
|
||||
monkeypatch.setattr(pipeline_auk, "AuKVAE", _StubVAE)
|
||||
|
||||
def _build(variant: str = "base") -> tuple[AuKPipeline, list[dict[str, Any]]]:
|
||||
def _build(
|
||||
variant: str = "base", *, model_config: dict[str, Any] | None = None
|
||||
) -> tuple[AuKPipeline, list[dict[str, Any]]]:
|
||||
calls: list[dict[str, Any]] = []
|
||||
monkeypatch.setattr(pipeline_auk, "sample_latents", _stub_sampler(calls))
|
||||
od_config = OmniDiffusionConfig(
|
||||
model=str(_write_checkpoint(tmp_path, variant)),
|
||||
dtype=torch.float32,
|
||||
model_class_name="AuKPipeline",
|
||||
model_config=model_config or {},
|
||||
)
|
||||
return AuKPipeline(od_config=od_config), calls
|
||||
|
||||
@@ -274,6 +277,20 @@ def _silence(seconds: float, sample_rate: int = SAMPLE_RATE) -> tuple[np.ndarray
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
class TestRequestParsing:
|
||||
@pytest.mark.parametrize(
|
||||
"model_config, expected_slots",
|
||||
[({}, 32), ({"max_dit_graphs": 1}, 1), ({"max_dit_graphs": 3}, 3), ({"max_dit_graphs": 64}, 64)],
|
||||
)
|
||||
def test_max_dit_graphs_reaches_lazy_cache(self, build_pipeline, model_config, expected_slots):
|
||||
pipeline, _ = build_pipeline(model_config=model_config)
|
||||
assert pipeline.cudagraph_wrapper.max_graphs == expected_slots
|
||||
assert not pipeline.cudagraph_wrapper._cache
|
||||
|
||||
@pytest.mark.parametrize("max_dit_graphs", [0, -1, True, False, 1.5, "3", None])
|
||||
def test_invalid_max_dit_graphs_is_rejected(self, build_pipeline, max_dit_graphs):
|
||||
with pytest.raises(ValueError, match="AuK max_dit_graphs must be a positive integer"):
|
||||
build_pipeline(model_config={"max_dit_graphs": max_dit_graphs})
|
||||
|
||||
def test_pipeline_declares_audio_output(self, build_pipeline):
|
||||
pipeline, _ = build_pipeline()
|
||||
|
||||
@@ -299,6 +316,7 @@ class TestRequestParsing:
|
||||
assert calls[0]["ref"].shape == (1, 100, LATENT_DIM)
|
||||
assert calls[0]["ref_mask"].shape == (1, 100)
|
||||
assert calls[0]["c_mask"].shape == (1, 8)
|
||||
assert calls[0]["sampler"] is pipeline.cudagraph_wrapper
|
||||
assert outputs[0].output.shape == (300 * HOP,)
|
||||
assert outputs[0].output.dtype is torch.float32
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""AuK Speech API E2E coverage for the encoder -> diffusion deployment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
import soundfile as sf
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import get_asset_path
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
MODEL = os.environ.get("VLLM_OMNI_AUK_MODEL_DIR")
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.tts,
|
||||
pytest.mark.skipif(not MODEL, reason="set VLLM_OMNI_AUK_MODEL_DIR to an assembled AuK checkpoint"),
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("reference", [False, True], ids=["instruct", "voice-clone"])
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["wav", "streaming-pcm"])
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
[
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL or ".",
|
||||
stage_config_path=get_deploy_config_path("auk.yaml"),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="auk",
|
||||
)
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_auk_speech_api_request(omni_server, reference, stream) -> None:
|
||||
"""The adapter accepts Speech API requests and returns bounded 24 kHz WAVs."""
|
||||
|
||||
def generate(duration: float) -> int:
|
||||
extra = {}
|
||||
if reference:
|
||||
extra["ref_audio"] = get_asset_path("cosyvoice3/zero_shot_prompt.wav", as_data_url=True)
|
||||
# Match the shared seed-tts benchmark request envelope.
|
||||
extra["ref_text"] = "Reference transcript is not required by AuK."
|
||||
response = requests.post(
|
||||
f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech",
|
||||
json={
|
||||
"model": omni_server.model,
|
||||
"input": "",
|
||||
"instructions": (
|
||||
"Say the following with the same voice: 'Hello, this is an AuK speech API test.'"
|
||||
if reference
|
||||
else 'Generate speech based on the following description: "A clear, natural voice.". '
|
||||
'The content to speak is: "Hello, this is an AuK speech API test.".'
|
||||
),
|
||||
"voice": "default",
|
||||
"duration_seconds": duration,
|
||||
"seed": 7,
|
||||
"response_format": "pcm" if stream else "wav",
|
||||
"stream": stream,
|
||||
"stream_format": "audio" if stream else None,
|
||||
**extra,
|
||||
},
|
||||
timeout=300,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
if stream:
|
||||
waveform = np.frombuffer(response.content, dtype="<i2").astype(np.float32) / 32768
|
||||
else:
|
||||
waveform, sample_rate = sf.read(io.BytesIO(response.content), dtype="float32")
|
||||
assert sample_rate == 24_000
|
||||
assert np.isfinite(waveform).all()
|
||||
assert np.max(np.abs(waveform)) > 0
|
||||
return int(np.asarray(waveform).reshape(-1).shape[0])
|
||||
|
||||
assert generate(2.0) == 100 * 480
|
||||
@@ -5367,6 +5367,9 @@ class TestTTSAsyncOffloading:
|
||||
adapter_model_type = "adapter_dummy_tts"
|
||||
|
||||
class FakeAdapter:
|
||||
def normalize(self, request):
|
||||
return
|
||||
|
||||
def validate(self, request):
|
||||
return None
|
||||
|
||||
|
||||
@@ -8,11 +8,22 @@ Pure-Python registry/resolution logic; no model or GPU resources are loaded.
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
from vllm_omni.entrypoints.openai.tts_adapters import (
|
||||
TTS_ADAPTER_REGISTRY,
|
||||
ARTTSAdapter,
|
||||
@@ -22,6 +33,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import (
|
||||
detect_tts_model_type,
|
||||
resolve_adapter,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.auk import AuKAdapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import TTSCapabilities, resolve_stage_model_path
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter
|
||||
@@ -42,10 +54,12 @@ from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import (
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.step_audio2 import StepAudio2Adapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.indextts2 import prompt_utils
|
||||
from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import (
|
||||
INDEXTTS25_TOKENIZER_FILE,
|
||||
)
|
||||
from vllm_omni.model_executor.stage_input_processors.auk import encoder2dit
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -70,6 +84,7 @@ EXPECTED_MODEL_TYPES = {
|
||||
"step_audio2",
|
||||
"indextts2",
|
||||
"indextts2_5",
|
||||
"auk",
|
||||
"gepard",
|
||||
}
|
||||
|
||||
@@ -99,6 +114,314 @@ def test_resolve_unknown_returns_none():
|
||||
assert resolve_adapter(None) is None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def auk_adapter(mocker):
|
||||
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
|
||||
server._max_instructions_length = 4096
|
||||
server._validate_ref_audio_format.return_value = None
|
||||
server._resolve_ref_audio = AsyncMock(return_value=([0.1] * 480, 24000, "key"))
|
||||
return AuKAdapter(SpeechServingContext(server=server))
|
||||
|
||||
|
||||
def test_auk_adapter_detection(mocker):
|
||||
assert detect_tts_model_type("encoder", "AuKForConditionalGeneration") == "auk"
|
||||
# ``encoder`` is shared with MiniMax H3, so AuK must not claim the stage key.
|
||||
assert AuKAdapter.stage_keys == frozenset()
|
||||
assert AuKAdapter.arch_identifies_entry_stage
|
||||
assert detect_tts_model_type("encoder", "MiniMaxH3Encoder") is None
|
||||
assert detect_tts_model_type("encoder", None) is None
|
||||
|
||||
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
|
||||
server._diffusion_mode = False
|
||||
server._tts_model_type = detect_tts_model_type("encoder", "AuKForConditionalGeneration")
|
||||
server._adapter = None
|
||||
server.engine_client = SimpleNamespace()
|
||||
|
||||
adapter = OmniOpenAIServingSpeech._get_tts_adapter(server)
|
||||
assert isinstance(adapter, AuKAdapter)
|
||||
server._drop_shadowing_uploads.assert_called_once_with()
|
||||
|
||||
|
||||
def test_auk_source_length_default_and_complete_instruction(auk_adapter):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="",
|
||||
ref_audio="reference.wav",
|
||||
ref_text="Accepted transcript",
|
||||
instructions="Change the pitch by one semitone.",
|
||||
extra_params={"sway": -0.5, "t_grid": [0, 0.25, 1], "vae_sample": True},
|
||||
)
|
||||
assert auk_adapter.validate(request) is None
|
||||
prepared = asyncio.run(auk_adapter.build(request, [], True))
|
||||
assert "Change the pitch by one semitone." in prepared.prompt["prompt"]
|
||||
assert "content to speak" not in prepared.prompt["prompt"]
|
||||
knobs = prepared.prompt["additional_information"]["auk"]
|
||||
assert knobs["gen_seconds"] is None
|
||||
assert knobs["vae_sample"] is True
|
||||
assert knobs["t_grid"] == [0, 0.25, 1]
|
||||
auk_adapter.ctx.server._resolve_ref_audio.assert_awaited_once_with("reference.wav")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"t_grid",
|
||||
[
|
||||
[],
|
||||
[0.0],
|
||||
[0.0, 0.0],
|
||||
[0.5, 0.25],
|
||||
[0.0, float("nan")],
|
||||
[0.0, float("inf")],
|
||||
[0.0, "invalid"],
|
||||
"0,1",
|
||||
],
|
||||
)
|
||||
def test_auk_rejects_invalid_t_grid_before_dispatch(auk_adapter, t_grid):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Say the following: 'target text'",
|
||||
duration_seconds=2,
|
||||
extra_params={"t_grid": t_grid},
|
||||
)
|
||||
|
||||
error = auk_adapter.validate(request)
|
||||
|
||||
assert error == "AuK extra_params.t_grid must contain at least two finite, strictly increasing values"
|
||||
auk_adapter.ctx.server._resolve_ref_audio.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("task_type", "expected_instruction"),
|
||||
[
|
||||
("CustomVoice", 'Say the following: "target text"'),
|
||||
("Base", 'Say the following with the same voice: "target text"'),
|
||||
],
|
||||
)
|
||||
def test_auk_task_type_normalizes_benchmark_text(auk_adapter, task_type, expected_instruction, mocker):
|
||||
warning_once = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning_once")
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="target text",
|
||||
task_type=task_type,
|
||||
duration_seconds=2,
|
||||
ref_audio="reference.wav" if task_type == "Base" else None,
|
||||
)
|
||||
|
||||
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
|
||||
server._validate_speech_sample_rate.return_value = None
|
||||
server._get_tts_adapter.return_value = auk_adapter
|
||||
assert OmniOpenAIServingSpeech._validate_tts_request(server, request) is None
|
||||
|
||||
assert request.input == ""
|
||||
assert request.instructions == expected_instruction
|
||||
assert request.task_type is None
|
||||
warning_once.assert_called_once()
|
||||
assert "prefer a complete `instructions` prompt" in warning_once.call_args.args[0]
|
||||
|
||||
auk_adapter.normalize(request)
|
||||
|
||||
assert request.instructions == expected_instruction
|
||||
warning_once.assert_called_once()
|
||||
|
||||
|
||||
def test_auk_task_type_quotes_embedded_delimiters(auk_adapter):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input='It\'s called "AuK"\\Flash',
|
||||
task_type="CustomVoice",
|
||||
duration_seconds=2,
|
||||
)
|
||||
|
||||
auk_adapter.normalize(request)
|
||||
|
||||
assert request.instructions == 'Say the following: "It\'s called \\"AuK\\"\\\\Flash"'
|
||||
assert request.task_type is None
|
||||
|
||||
|
||||
def test_auk_task_type_preserves_explicit_instructions(auk_adapter, mocker):
|
||||
warning_once = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning_once")
|
||||
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="target text",
|
||||
task_type="Base",
|
||||
instructions="Use this complete AuK instruction.",
|
||||
duration_seconds=2,
|
||||
)
|
||||
|
||||
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
|
||||
server._validate_speech_sample_rate.return_value = None
|
||||
server._get_tts_adapter.return_value = auk_adapter
|
||||
assert OmniOpenAIServingSpeech._validate_tts_request(server, request) is None
|
||||
|
||||
assert request.input == ""
|
||||
assert request.instructions == "Use this complete AuK instruction."
|
||||
assert request.task_type is None
|
||||
warning_once.assert_called_once()
|
||||
warning.assert_called_once()
|
||||
assert "without applying another task template" in warning.call_args.args[0]
|
||||
|
||||
|
||||
def test_auk_task_type_does_not_double_wrap_preformatted_input(auk_adapter, mocker):
|
||||
warning_once = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning_once")
|
||||
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
|
||||
instruction = "Say the following with the same voice: 'target text'"
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input=instruction,
|
||||
task_type="Base",
|
||||
duration_seconds=2,
|
||||
ref_audio="reference.wav",
|
||||
)
|
||||
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
|
||||
server._validate_speech_sample_rate.return_value = None
|
||||
server._get_tts_adapter.return_value = auk_adapter
|
||||
|
||||
assert OmniOpenAIServingSpeech._validate_tts_request(server, request) is None
|
||||
|
||||
assert request.input == ""
|
||||
assert request.instructions == instruction
|
||||
assert request.task_type is None
|
||||
warning_once.assert_called_once()
|
||||
warning.assert_called_once()
|
||||
assert "already contains a complete task instruction" in warning.call_args.args[0]
|
||||
|
||||
|
||||
def test_auk_sampling_overrides_reach_stage1_pipeline(auk_adapter, mocker):
|
||||
"""Exercise one request from the Speech adapter through stage-1 admission."""
|
||||
|
||||
class RecordingDiffusionStage:
|
||||
stage_type = "diffusion"
|
||||
final_output = True
|
||||
engine_input_source = [0]
|
||||
requires_multimodal_data = False
|
||||
custom_process_input_func = staticmethod(encoder2dit)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.request: OmniDiffusionRequest | None = None
|
||||
|
||||
async def add_request_async(self, request_id, prompt, sampling_params, **_kwargs) -> None:
|
||||
# Match the stage-client wire boundary and the receiving diffusion
|
||||
# process, where params are serialized then reconstructed before
|
||||
# creating the request consumed by the pipeline.
|
||||
wire_params = StageDiffusionClient._sampling_params_to_dict(sampling_params)
|
||||
self.request = OmniDiffusionRequest(
|
||||
prompt=prompt,
|
||||
sampling_params=StageDiffusionProc._reconstruct_sampling_params(
|
||||
object.__new__(StageDiffusionProc), wire_params
|
||||
),
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
defaults = [SamplingParams(max_tokens=1), OmniDiffusionSamplingParams(seed=0, num_inference_steps=32)]
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="",
|
||||
instructions='Generate speech based on the following description: "A calm voice". The content to speak is: "Hello".',
|
||||
duration_seconds=2,
|
||||
seed=9,
|
||||
extra_params={
|
||||
"num_inference_steps": 4,
|
||||
"guidance_scale": 1.5,
|
||||
"sway": -0.5,
|
||||
"t_grid": [0.0, 0.25, 1.0],
|
||||
"vae_sample": True,
|
||||
},
|
||||
)
|
||||
prepared = asyncio.run(auk_adapter.build(request, defaults, False))
|
||||
updated = auk_adapter.apply_sampling_overrides(defaults, request)
|
||||
assert defaults[1].num_inference_steps == 32
|
||||
assert defaults[1].seed == 0
|
||||
assert updated[0].max_tokens == 1
|
||||
|
||||
source_stage = SimpleNamespace(
|
||||
stage_type="llm",
|
||||
final_output=False,
|
||||
get_kv_sender_info=lambda: None,
|
||||
)
|
||||
diffusion_stage = RecordingDiffusionStage()
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.stage_pools = [StagePool(0, source_stage), StagePool(1, diffusion_stage)]
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
orchestrator.duplex_control_plane = None
|
||||
orchestrator._running_counter = None
|
||||
|
||||
req_state = OrchestratorRequestState(
|
||||
request_id="auk-sampling-override",
|
||||
prompt=prepared.prompt,
|
||||
sampling_params_list=updated,
|
||||
final_stage_id=1,
|
||||
)
|
||||
encoder_output = SimpleNamespace(
|
||||
request_id=req_state.request_id,
|
||||
finished=True,
|
||||
multimodal_output={"hidden_states": {"output": torch.ones(3, 4)}},
|
||||
outputs=[],
|
||||
)
|
||||
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, req_state.request_id, 0, encoder_output, req_state))
|
||||
|
||||
received = diffusion_stage.request
|
||||
assert received is not None
|
||||
assert received.prompt["prompt_embeds"].shape == (3, 4)
|
||||
assert received.prompt["additional_information"]["auk"] == {
|
||||
"gen_seconds": 2,
|
||||
"sway": -0.5,
|
||||
"t_grid": [0.0, 0.25, 1.0],
|
||||
"vae_sample": True,
|
||||
"has_audio": False,
|
||||
}
|
||||
assert received.sampling_params.num_inference_steps == 4
|
||||
assert received.sampling_params.guidance_scale == 1.5
|
||||
assert received.sampling_params.guidance_scale_provided is True
|
||||
assert received.sampling_params.seed == 9
|
||||
|
||||
runner = mocker.Mock(spec=DiffusionModelRunner)
|
||||
runner.device = torch.device("cpu")
|
||||
DiffusionModelRunner._initialize_generator(runner, received.sampling_params)
|
||||
assert received.sampling_params.generator.initial_seed() == 9
|
||||
|
||||
pipeline_params = DiffusionRequestBatch(requests=[received]).sampling_params
|
||||
assert pipeline_params is received.sampling_params
|
||||
assert pipeline_params.generator.initial_seed() == 9
|
||||
|
||||
|
||||
def test_auk_instructions_take_precedence_over_input(auk_adapter, mocker):
|
||||
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="ignored input",
|
||||
instructions='Generate speech based on the following description: "Speak warmly". The content to speak is: "Hello".',
|
||||
duration_seconds=2,
|
||||
)
|
||||
assert auk_adapter.validate(request) is None
|
||||
prepared = asyncio.run(auk_adapter.build(request, [], False))
|
||||
prompt = prepared.prompt
|
||||
assert 'Generate speech based on the following description: "Speak warmly".' in prompt["prompt"]
|
||||
assert 'The content to speak is: "Hello".' in prompt["prompt"]
|
||||
assert "|<no_prompt_audio>|" in prompt["prompt"]
|
||||
assert prompt["additional_information"]["auk"]["gen_seconds"] == 2
|
||||
warning.assert_called_once()
|
||||
assert "using the complete `instructions`" in warning.call_args.args[0]
|
||||
|
||||
|
||||
def test_auk_input_only_is_treated_as_complete_instruction(auk_adapter, mocker):
|
||||
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Say the following with the same voice: 'Hello world.'",
|
||||
ref_audio="reference.wav",
|
||||
duration_seconds=2,
|
||||
)
|
||||
assert auk_adapter.validate(request) is None
|
||||
prepared = asyncio.run(auk_adapter.build(request, [], True))
|
||||
assert "Say the following with the same voice: 'Hello world.'" in prepared.prompt["prompt"]
|
||||
assert warning.call_count == 1
|
||||
assert 'Prefer `input=""`' in warning.call_args.args[0]
|
||||
|
||||
|
||||
def test_auk_complete_instruction_does_not_require_spoken_text(auk_adapter):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="",
|
||||
instructions="Replace 'morning' with 'evening' in the source recording.",
|
||||
ref_audio="reference.wav",
|
||||
)
|
||||
assert auk_adapter.validate(request) is None
|
||||
prepared = asyncio.run(auk_adapter.build(request, [], True))
|
||||
assert prepared.prompt["prompt"].count("Replace 'morning'") == 1
|
||||
assert "Generate speech based on" not in prepared.prompt["prompt"]
|
||||
assert prepared.prompt["additional_information"]["auk"]["gen_seconds"] is None
|
||||
|
||||
|
||||
def test_stage_model_path_prefers_typed_override():
|
||||
engine_client = SimpleNamespace(
|
||||
stage_configs=[
|
||||
|
||||
@@ -182,6 +182,7 @@ _PIPELINE_STAGES = [
|
||||
"token2text",
|
||||
"token2wav",
|
||||
"tts",
|
||||
"encoder",
|
||||
]
|
||||
|
||||
_STAGES = [*_PIPELINE_STAGES, None, "vae", "not_a_real_stage"]
|
||||
@@ -283,14 +284,22 @@ def test_pipeline_stage_list_is_complete():
|
||||
assert not missing, f"stage keys claimed by adapters but absent from _PIPELINE_STAGES: {sorted(missing)}"
|
||||
|
||||
|
||||
def test_entry_stage_archs_is_ming_only():
|
||||
"""Only Ming dense identifies its entry stage by architecture.
|
||||
def test_entry_stage_archs_are_explicit():
|
||||
"""Only Ming dense and AuK identify their entry stage by architecture.
|
||||
|
||||
Widening this set would change which stage ``_find_tts_stage`` selects in
|
||||
mixed deployments — notably VoxCPM2, which declares ``model_archs`` but is
|
||||
found by its ``latent_generator`` stage key.
|
||||
Both use a generic ``model_stage`` (``llm`` and ``encoder``) that other
|
||||
pipelines share. Widening this set would change which stage
|
||||
``_find_tts_stage`` selects in mixed deployments — notably VoxCPM2, which
|
||||
declares ``model_archs`` but is found by its ``latent_generator`` stage key.
|
||||
"""
|
||||
assert tts_entry_stage_archs() == frozenset({"MingTTSForConditionalGeneration"})
|
||||
assert tts_entry_stage_archs() == frozenset({"MingTTSForConditionalGeneration", "AuKForConditionalGeneration"})
|
||||
|
||||
|
||||
def test_minimax_h3_encoder_is_not_a_tts_stage():
|
||||
"""MiniMax H3 shares AuK's ``encoder`` stage key but is not a speech model."""
|
||||
assert "encoder" not in all_tts_stage_keys()
|
||||
assert detect_tts_model_type("encoder", "MiniMaxH3Encoder") is None
|
||||
assert detect_tts_model_type("encoder", "AuKForConditionalGeneration") == "auk"
|
||||
|
||||
|
||||
def test_every_detected_type_is_adapter_backed():
|
||||
|
||||
@@ -1475,6 +1475,7 @@ class OnlineOmniClient:
|
||||
"max_new_tokens",
|
||||
"seed",
|
||||
"instructions",
|
||||
"duration_seconds",
|
||||
"speed",
|
||||
"sample_rate",
|
||||
"extra_params",
|
||||
|
||||
@@ -214,6 +214,9 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
|
||||
)
|
||||
|
||||
class Adapter:
|
||||
def normalize(self, request):
|
||||
return
|
||||
|
||||
def validate(self, request):
|
||||
return None
|
||||
|
||||
|
||||
Reference in new issue
Block a user