[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469)

Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-30 00:18:38 +00:00
1 parent 3a28882946
commit 35cf6ac355
22 files changed
+1473 -38

No files matched your search

+87
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for the universal benchmarks/tts/bench_tts.py CLI."""
from __future__ import annotations
@@ -51,6 +54,73 @@ def test_load_model_configs(model_configs_path: Path) -> None:
assert configs["test/ModelA"]["supported_tasks"] == ["voice_clone", "default_voice"]
@pytest.mark.parametrize("model", ["tencent/AuK", "tencent/AuK-Flash"])
@pytest.mark.parametrize("task", ["default_voice", "voice_clone"])
def test_auk_benchmark_duration_and_local_bundle(model, task):
config = bench_tts.load_model_configs(bench_tts._DEFAULT_MODEL_CONFIGS)[model]
cmd = bench_tts.build_bench_args(
host="localhost",
port=8000,
model=model,
task=task,
model_cfg=config,
locale="en",
num_prompts=20,
concurrency=8,
dataset_path="/data/seed-tts",
wer_eval=False,
output_dir=None,
result_filename=None,
extra_cli_args=[],
served_model_name="/models/auk",
duration_seconds=3.5,
request_seed=7,
)
assert cmd[cmd.index("--model") + 1] == "/models/auk"
body = json.loads(cmd[cmd.index("--extra-body") + 1])
assert body == {
"voice": "default",
"duration_seconds": 3.5,
"seed": 7,
"task_type": "CustomVoice" if task == "default_voice" else "Base",
}
assert config["task_extra_body"][task]["duration_seconds"] == 5.0
@pytest.mark.parametrize(
("task", "expected_task_type"),
[
("default_voice", "CustomVoice"),
("voice_clone", "Base"),
],
)
def test_auk_task_type_survives_served_model_alias(task, expected_task_type):
model = "tencent/AuK-Flash"
served_model = "models/flash"
config = bench_tts.load_model_configs(bench_tts._DEFAULT_MODEL_CONFIGS)[model]
cmd = bench_tts.build_bench_args(
host="localhost",
port=8000,
model=model,
task=task,
model_cfg=config,
locale="en",
num_prompts=1,
concurrency=1,
dataset_path="/data/seed-tts",
wer_eval=False,
output_dir=None,
result_filename=None,
extra_cli_args=[],
served_model_name=served_model,
)
assert cmd[cmd.index("--model") + 1] == served_model
extra_body = json.loads(cmd[cmd.index("--extra-body") + 1])
assert extra_body["task_type"] == expected_task_type
assert "_vllm_omni_benchmark_model" not in extra_body
def test_indextts25_is_registered_in_shared_model_configs() -> None:
config_path = Path(bench_tts.__file__).with_name("model_configs.yaml")
config = bench_tts.load_model_configs(config_path)["IndexTeam/IndexTTS-2.5"]
@@ -173,3 +243,20 @@ def test_unsupported_task_exits(model_configs_path: Path, capsys: pytest.Capture
)
with pytest.raises(SystemExit):
bench_tts.main()
def test_summary_table_includes_task_type(capsys: pytest.CaptureFixture) -> None:
bench_tts.print_summary_table(
[
{
"_task": "voice_clone",
"_task_type": "Base",
"_concurrency": 1,
}
]
)
output = capsys.readouterr().out
assert "Task Type" in output
assert "voice_clone" in output
assert "Base" in output
@@ -0,0 +1,225 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Single-request eager and CUDA-graph AuK DiT sampling equivalence."""
import pytest
import torch
from vllm_omni.diffusion.models.auk.auk_transformer import AuKTransformer, sample_latents
from vllm_omni.diffusion.models.auk.cudagraph_wrapper import AuKCUDAGraphWrapper
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_dit(device: str) -> AuKTransformer:
torch.manual_seed(12)
return (
AuKTransformer(
dim=32,
heads=2,
dim_head=16,
ff_mult=2,
latent_dim=4,
text_hidden_dim=8,
num_layers=2,
num_single_layers=2,
)
.eval()
.to(device)
)
def _sample_inputs(device: str, offset: float = 0.0) -> dict[str, torch.Tensor]:
generator = torch.Generator(device=device).manual_seed(42)
return {
"text": torch.randn(1, 7, 8, generator=generator, device=device) + offset,
"c_mask": torch.ones(1, 7, dtype=torch.bool, device=device),
"ref": torch.randn(1, 4, 4, generator=generator, device=device) - offset,
"ref_mask": torch.ones(1, 4, dtype=torch.bool, device=device),
}
@torch.inference_mode()
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
def test_single_request_graph_wrapper_cpu_falls_back_to_eager(cfg_strength: float, mocker) -> None:
dit = _make_dit("cpu")
wrapper = AuKCUDAGraphWrapper(dit)
run_spy = mocker.spy(wrapper, "_run")
run_cfg_spy = mocker.spy(wrapper, "_run_cfg")
capture_spy = mocker.spy(wrapper, "_capture")
inputs = _sample_inputs("cpu")
common = dict(
**inputs,
gen_frames=9,
t_grid=[0.0, 0.4, 1.0],
cfg_strength=cfg_strength,
)
eager = sample_latents(dit, **common, generator=torch.Generator().manual_seed(7))
graph = sample_latents(dit, **common, generator=torch.Generator().manual_seed(7), sampler=wrapper)
torch.testing.assert_close(graph, eager)
if cfg_strength >= 1e-5:
run_spy.assert_not_called()
assert run_cfg_spy.call_count == 2
else:
assert run_spy.call_count == 2
run_cfg_spy.assert_not_called()
capture_spy.assert_not_called()
assert not wrapper._cache
def test_graph_inputs_use_bounded_length_buckets() -> None:
wrapper = AuKCUDAGraphWrapper(_make_dit("cpu"))
x = torch.ones(1, 65, 4)
text = torch.ones(1, 65, 8)
c_mask = torch.ones(1, 65, dtype=torch.bool)
ref = torch.ones(1, 51, 4)
ref_mask = torch.ones(1, 51, dtype=torch.bool)
padded = wrapper._bucket_inputs(x, text, c_mask, ref, ref_mask)
assert padded[0].shape == (1, 128, 4)
assert padded[1].shape == (1, 128)
assert padded[2].shape == (1, 128, 8)
assert padded[3].shape == (1, 128)
assert padded[4].shape == (1, 100, 4)
assert padded[5].shape == (1, 100)
assert [mask.sum().item() for mask in (padded[1], padded[3], padded[5])] == [65, 65, 51]
assert wrapper._key(padded[0], padded[2], padded[4], False) == (128, 128, 100, False)
assert wrapper.max_graphs == 32
def test_full_graph_cache_retires_as_one_generation() -> None:
wrapper = AuKCUDAGraphWrapper(_make_dit("cpu"), max_graphs=2)
first = (64, 64, 50, False)
second = (128, 64, 50, False)
wrapper._cache[first] = object()
wrapper._retire_graph_generation_if_full()
assert list(wrapper._cache) == [first]
wrapper._cache[second] = object()
wrapper._retire_graph_generation_if_full()
assert not wrapper._cache
@torch.inference_mode()
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
def test_bucket_padding_preserves_real_frame_outputs(cfg_strength: float) -> None:
dit = _make_dit("cpu")
wrapper = AuKCUDAGraphWrapper(dit)
inputs = _sample_inputs("cpu")
x = torch.randn(1, 9, 4)
timestep = torch.tensor(0.4)
cfg = torch.tensor(cfg_strength)
if cfg_strength >= 1e-5:
eager = wrapper._run_cfg(
x,
None,
inputs["text"],
inputs["c_mask"],
inputs["ref"],
inputs["ref_mask"],
timestep,
cfg_strength=cfg,
)
else:
eager = wrapper._run(
x,
None,
inputs["text"],
inputs["c_mask"],
inputs["ref"],
inputs["ref_mask"],
timestep,
)
dit.clear_cache()
bucketed = wrapper._bucket_inputs(x, inputs["text"], inputs["c_mask"], inputs["ref"], inputs["ref_mask"])
if cfg_strength >= 1e-5:
padded = wrapper._run_cfg(*bucketed, timestep, cfg_strength=cfg)
else:
padded = wrapper._run(*bucketed, timestep)
dit.clear_cache()
torch.testing.assert_close(padded[:, : x.shape[1]], eager)
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
@torch.inference_mode()
@pytest.mark.parametrize("cfg_strength", [0.0, 2.0])
def test_single_request_graph_replay_matches_eager_and_updates_inputs(cfg_strength: float) -> None:
dit = _make_dit("cuda")
wrapper = AuKCUDAGraphWrapper(dit)
for offset in (0.0, 0.25):
inputs = _sample_inputs("cuda", offset)
common = dict(
**inputs,
gen_frames=9,
t_grid=[0.0, 0.4, 1.0],
cfg_strength=cfg_strength,
)
eager = sample_latents(dit, **common, generator=torch.Generator(device="cuda").manual_seed(7))
graph = sample_latents(dit, **common, generator=torch.Generator(device="cuda").manual_seed(7), sampler=wrapper)
torch.testing.assert_close(graph, eager, atol=3e-6, rtol=3e-5)
bucketed = wrapper._bucket_inputs(
torch.empty(1, 9, 4, device="cuda"),
inputs["text"],
inputs["c_mask"],
inputs["ref"],
inputs["ref_mask"],
)
key = wrapper._key(bucketed[0], bucketed[2], bucketed[4], cfg_strength >= 1e-5)
assert key in wrapper._cache
entry = wrapper._cache[key]
torch.testing.assert_close(entry.static_x_mask, bucketed[1])
torch.testing.assert_close(entry.static_text, bucketed[2])
torch.testing.assert_close(entry.static_c_mask, bucketed[3])
torch.testing.assert_close(entry.static_ref, bucketed[4])
torch.testing.assert_close(entry.static_ref_mask, bucketed[5])
torch.testing.assert_close(entry.static_timestep, torch.tensor(0.4, device="cuda"))
assert len(wrapper._cache) == 1
@pytest.mark.cuda
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA graph replay requires CUDA")
@torch.inference_mode()
def test_graph_capture_failure_is_propagated(mocker) -> None:
dit = _make_dit("cuda")
wrapper = AuKCUDAGraphWrapper(dit)
capture = mocker.patch.object(wrapper, "_capture", side_effect=RuntimeError("capture failed"))
eager = mocker.spy(wrapper, "_run")
inputs = _sample_inputs("cuda")
common = dict(
**inputs,
gen_frames=9,
t_grid=[0.0, 0.4, 1.0],
cfg_strength=0.0,
)
with pytest.raises(RuntimeError, match="capture failed"):
sample_latents(
dit,
**common,
generator=torch.Generator(device="cuda").manual_seed(7),
sampler=wrapper,
)
result = sample_latents(
dit,
**common,
generator=torch.Generator(device="cuda").manual_seed(7),
sampler=wrapper,
)
assert result.shape == (1, 9, 4)
assert capture.call_count == 1
assert eager.call_count == 2
assert not wrapper.enabled
assert not wrapper._cache
@@ -227,13 +227,16 @@ def build_pipeline(tmp_path, monkeypatch):
monkeypatch.setattr(pipeline_auk, "AuKTransformer", _StubTransformer)
monkeypatch.setattr(pipeline_auk, "AuKVAE", _StubVAE)
def _build(variant: str = "base") -> tuple[AuKPipeline, list[dict[str, Any]]]:
def _build(
variant: str = "base", *, model_config: dict[str, Any] | None = None
) -> tuple[AuKPipeline, list[dict[str, Any]]]:
calls: list[dict[str, Any]] = []
monkeypatch.setattr(pipeline_auk, "sample_latents", _stub_sampler(calls))
od_config = OmniDiffusionConfig(
model=str(_write_checkpoint(tmp_path, variant)),
dtype=torch.float32,
model_class_name="AuKPipeline",
model_config=model_config or {},
)
return AuKPipeline(od_config=od_config), calls
@@ -274,6 +277,20 @@ def _silence(seconds: float, sample_rate: int = SAMPLE_RATE) -> tuple[np.ndarray
@pytest.mark.core_model
@pytest.mark.cpu
class TestRequestParsing:
@pytest.mark.parametrize(
"model_config, expected_slots",
[({}, 32), ({"max_dit_graphs": 1}, 1), ({"max_dit_graphs": 3}, 3), ({"max_dit_graphs": 64}, 64)],
)
def test_max_dit_graphs_reaches_lazy_cache(self, build_pipeline, model_config, expected_slots):
pipeline, _ = build_pipeline(model_config=model_config)
assert pipeline.cudagraph_wrapper.max_graphs == expected_slots
assert not pipeline.cudagraph_wrapper._cache
@pytest.mark.parametrize("max_dit_graphs", [0, -1, True, False, 1.5, "3", None])
def test_invalid_max_dit_graphs_is_rejected(self, build_pipeline, max_dit_graphs):
with pytest.raises(ValueError, match="AuK max_dit_graphs must be a positive integer"):
build_pipeline(model_config={"max_dit_graphs": max_dit_graphs})
def test_pipeline_declares_audio_output(self, build_pipeline):
pipeline, _ = build_pipeline()
@@ -299,6 +316,7 @@ class TestRequestParsing:
assert calls[0]["ref"].shape == (1, 100, LATENT_DIM)
assert calls[0]["ref_mask"].shape == (1, 100)
assert calls[0]["c_mask"].shape == (1, 8)
assert calls[0]["sampler"] is pipeline.cudagraph_wrapper
assert outputs[0].output.shape == (300 * HOP,)
assert outputs[0].output.dtype is torch.float32
+85
View File
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""AuK Speech API E2E coverage for the encoder -> diffusion deployment."""
from __future__ import annotations
import io
import os
import numpy as np
import pytest
import requests
import soundfile as sf
from tests.helpers.mark import hardware_test
from tests.helpers.media import get_asset_path
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import get_deploy_config_path
MODEL = os.environ.get("VLLM_OMNI_AUK_MODEL_DIR")
pytestmark = [
pytest.mark.slow,
pytest.mark.tts,
pytest.mark.skipif(not MODEL, reason="set VLLM_OMNI_AUK_MODEL_DIR to an assembled AuK checkpoint"),
]
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("reference", [False, True], ids=["instruct", "voice-clone"])
@pytest.mark.parametrize("stream", [False, True], ids=["wav", "streaming-pcm"])
@pytest.mark.parametrize(
"omni_server",
[
pytest.param(
OmniServerParams(
model=MODEL or ".",
stage_config_path=get_deploy_config_path("auk.yaml"),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="auk",
)
],
indirect=True,
)
def test_auk_speech_api_request(omni_server, reference, stream) -> None:
"""The adapter accepts Speech API requests and returns bounded 24 kHz WAVs."""
def generate(duration: float) -> int:
extra = {}
if reference:
extra["ref_audio"] = get_asset_path("cosyvoice3/zero_shot_prompt.wav", as_data_url=True)
# Match the shared seed-tts benchmark request envelope.
extra["ref_text"] = "Reference transcript is not required by AuK."
response = requests.post(
f"http://{omni_server.host}:{omni_server.port}/v1/audio/speech",
json={
"model": omni_server.model,
"input": "",
"instructions": (
"Say the following with the same voice: 'Hello, this is an AuK speech API test.'"
if reference
else 'Generate speech based on the following description: "A clear, natural voice.". '
'The content to speak is: "Hello, this is an AuK speech API test.".'
),
"voice": "default",
"duration_seconds": duration,
"seed": 7,
"response_format": "pcm" if stream else "wav",
"stream": stream,
"stream_format": "audio" if stream else None,
**extra,
},
timeout=300,
)
assert response.status_code == 200, response.text
if stream:
waveform = np.frombuffer(response.content, dtype="<i2").astype(np.float32) / 32768
else:
waveform, sample_rate = sf.read(io.BytesIO(response.content), dtype="float32")
assert sample_rate == 24_000
assert np.isfinite(waveform).all()
assert np.max(np.abs(waveform)) > 0
return int(np.asarray(waveform).reshape(-1).shape[0])
assert generate(2.0) == 100 * 480
@@ -5367,6 +5367,9 @@ class TestTTSAsyncOffloading:
adapter_model_type = "adapter_dummy_tts"
class FakeAdapter:
def normalize(self, request):
return
def validate(self, request):
return None
@@ -8,11 +8,22 @@ Pure-Python registry/resolution logic; no model or GPU resources are loaded.
import asyncio
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
import pytest
import torch
from vllm.sampling_params import SamplingParams
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
from vllm_omni.diffusion.stage_diffusion_proc import StageDiffusionProc
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.tts_adapters import (
TTS_ADAPTER_REGISTRY,
ARTTSAdapter,
@@ -22,6 +33,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import (
detect_tts_model_type,
resolve_adapter,
)
from vllm_omni.entrypoints.openai.tts_adapters.auk import AuKAdapter
from vllm_omni.entrypoints.openai.tts_adapters.base import TTSCapabilities, resolve_stage_model_path
from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter
from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter
@@ -42,10 +54,12 @@ from vllm_omni.entrypoints.openai.tts_adapters.qwen3_tts import (
)
from vllm_omni.entrypoints.openai.tts_adapters.step_audio2 import StepAudio2Adapter
from vllm_omni.entrypoints.openai.tts_adapters.voxtral import VoxtralTTSAdapter
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.indextts2 import prompt_utils
from vllm_omni.model_executor.models.indextts2.tokenizer_v2_5 import (
INDEXTTS25_TOKENIZER_FILE,
)
from vllm_omni.model_executor.stage_input_processors.auk import encoder2dit
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -70,6 +84,7 @@ EXPECTED_MODEL_TYPES = {
"step_audio2",
"indextts2",
"indextts2_5",
"auk",
"gepard",
}
@@ -99,6 +114,314 @@ def test_resolve_unknown_returns_none():
assert resolve_adapter(None) is None
@pytest.fixture
def auk_adapter(mocker):
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
server._max_instructions_length = 4096
server._validate_ref_audio_format.return_value = None
server._resolve_ref_audio = AsyncMock(return_value=([0.1] * 480, 24000, "key"))
return AuKAdapter(SpeechServingContext(server=server))
def test_auk_adapter_detection(mocker):
assert detect_tts_model_type("encoder", "AuKForConditionalGeneration") == "auk"
# ``encoder`` is shared with MiniMax H3, so AuK must not claim the stage key.
assert AuKAdapter.stage_keys == frozenset()
assert AuKAdapter.arch_identifies_entry_stage
assert detect_tts_model_type("encoder", "MiniMaxH3Encoder") is None
assert detect_tts_model_type("encoder", None) is None
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
server._diffusion_mode = False
server._tts_model_type = detect_tts_model_type("encoder", "AuKForConditionalGeneration")
server._adapter = None
server.engine_client = SimpleNamespace()
adapter = OmniOpenAIServingSpeech._get_tts_adapter(server)
assert isinstance(adapter, AuKAdapter)
server._drop_shadowing_uploads.assert_called_once_with()
def test_auk_source_length_default_and_complete_instruction(auk_adapter):
request = OpenAICreateSpeechRequest(
input="",
ref_audio="reference.wav",
ref_text="Accepted transcript",
instructions="Change the pitch by one semitone.",
extra_params={"sway": -0.5, "t_grid": [0, 0.25, 1], "vae_sample": True},
)
assert auk_adapter.validate(request) is None
prepared = asyncio.run(auk_adapter.build(request, [], True))
assert "Change the pitch by one semitone." in prepared.prompt["prompt"]
assert "content to speak" not in prepared.prompt["prompt"]
knobs = prepared.prompt["additional_information"]["auk"]
assert knobs["gen_seconds"] is None
assert knobs["vae_sample"] is True
assert knobs["t_grid"] == [0, 0.25, 1]
auk_adapter.ctx.server._resolve_ref_audio.assert_awaited_once_with("reference.wav")
@pytest.mark.parametrize(
"t_grid",
[
[],
[0.0],
[0.0, 0.0],
[0.5, 0.25],
[0.0, float("nan")],
[0.0, float("inf")],
[0.0, "invalid"],
"0,1",
],
)
def test_auk_rejects_invalid_t_grid_before_dispatch(auk_adapter, t_grid):
request = OpenAICreateSpeechRequest(
input="Say the following: 'target text'",
duration_seconds=2,
extra_params={"t_grid": t_grid},
)
error = auk_adapter.validate(request)
assert error == "AuK extra_params.t_grid must contain at least two finite, strictly increasing values"
auk_adapter.ctx.server._resolve_ref_audio.assert_not_awaited()
@pytest.mark.parametrize(
("task_type", "expected_instruction"),
[
("CustomVoice", 'Say the following: "target text"'),
("Base", 'Say the following with the same voice: "target text"'),
],
)
def test_auk_task_type_normalizes_benchmark_text(auk_adapter, task_type, expected_instruction, mocker):
warning_once = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning_once")
request = OpenAICreateSpeechRequest(
input="target text",
task_type=task_type,
duration_seconds=2,
ref_audio="reference.wav" if task_type == "Base" else None,
)
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
server._validate_speech_sample_rate.return_value = None
server._get_tts_adapter.return_value = auk_adapter
assert OmniOpenAIServingSpeech._validate_tts_request(server, request) is None
assert request.input == ""
assert request.instructions == expected_instruction
assert request.task_type is None
warning_once.assert_called_once()
assert "prefer a complete `instructions` prompt" in warning_once.call_args.args[0]
auk_adapter.normalize(request)
assert request.instructions == expected_instruction
warning_once.assert_called_once()
def test_auk_task_type_quotes_embedded_delimiters(auk_adapter):
request = OpenAICreateSpeechRequest(
input='It\'s called "AuK"\\Flash',
task_type="CustomVoice",
duration_seconds=2,
)
auk_adapter.normalize(request)
assert request.instructions == 'Say the following: "It\'s called \\"AuK\\"\\\\Flash"'
assert request.task_type is None
def test_auk_task_type_preserves_explicit_instructions(auk_adapter, mocker):
warning_once = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning_once")
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
request = OpenAICreateSpeechRequest(
input="target text",
task_type="Base",
instructions="Use this complete AuK instruction.",
duration_seconds=2,
)
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
server._validate_speech_sample_rate.return_value = None
server._get_tts_adapter.return_value = auk_adapter
assert OmniOpenAIServingSpeech._validate_tts_request(server, request) is None
assert request.input == ""
assert request.instructions == "Use this complete AuK instruction."
assert request.task_type is None
warning_once.assert_called_once()
warning.assert_called_once()
assert "without applying another task template" in warning.call_args.args[0]
def test_auk_task_type_does_not_double_wrap_preformatted_input(auk_adapter, mocker):
warning_once = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning_once")
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
instruction = "Say the following with the same voice: 'target text'"
request = OpenAICreateSpeechRequest(
input=instruction,
task_type="Base",
duration_seconds=2,
ref_audio="reference.wav",
)
server = mocker.Mock(spec=OmniOpenAIServingSpeech)
server._validate_speech_sample_rate.return_value = None
server._get_tts_adapter.return_value = auk_adapter
assert OmniOpenAIServingSpeech._validate_tts_request(server, request) is None
assert request.input == ""
assert request.instructions == instruction
assert request.task_type is None
warning_once.assert_called_once()
warning.assert_called_once()
assert "already contains a complete task instruction" in warning.call_args.args[0]
def test_auk_sampling_overrides_reach_stage1_pipeline(auk_adapter, mocker):
"""Exercise one request from the Speech adapter through stage-1 admission."""
class RecordingDiffusionStage:
stage_type = "diffusion"
final_output = True
engine_input_source = [0]
requires_multimodal_data = False
custom_process_input_func = staticmethod(encoder2dit)
def __init__(self) -> None:
self.request: OmniDiffusionRequest | None = None
async def add_request_async(self, request_id, prompt, sampling_params, **_kwargs) -> None:
# Match the stage-client wire boundary and the receiving diffusion
# process, where params are serialized then reconstructed before
# creating the request consumed by the pipeline.
wire_params = StageDiffusionClient._sampling_params_to_dict(sampling_params)
self.request = OmniDiffusionRequest(
prompt=prompt,
sampling_params=StageDiffusionProc._reconstruct_sampling_params(
object.__new__(StageDiffusionProc), wire_params
),
request_id=request_id,
)
defaults = [SamplingParams(max_tokens=1), OmniDiffusionSamplingParams(seed=0, num_inference_steps=32)]
request = OpenAICreateSpeechRequest(
input="",
instructions='Generate speech based on the following description: "A calm voice". The content to speak is: "Hello".',
duration_seconds=2,
seed=9,
extra_params={
"num_inference_steps": 4,
"guidance_scale": 1.5,
"sway": -0.5,
"t_grid": [0.0, 0.25, 1.0],
"vae_sample": True,
},
)
prepared = asyncio.run(auk_adapter.build(request, defaults, False))
updated = auk_adapter.apply_sampling_overrides(defaults, request)
assert defaults[1].num_inference_steps == 32
assert defaults[1].seed == 0
assert updated[0].max_tokens == 1
source_stage = SimpleNamespace(
stage_type="llm",
final_output=False,
get_kv_sender_info=lambda: None,
)
diffusion_stage = RecordingDiffusionStage()
orchestrator = object.__new__(Orchestrator)
orchestrator.stage_pools = [StagePool(0, source_stage), StagePool(1, diffusion_stage)]
orchestrator._cfg_tracker = CfgCompanionTracker()
orchestrator.duplex_control_plane = None
orchestrator._running_counter = None
req_state = OrchestratorRequestState(
request_id="auk-sampling-override",
prompt=prepared.prompt,
sampling_params_list=updated,
final_stage_id=1,
)
encoder_output = SimpleNamespace(
request_id=req_state.request_id,
finished=True,
multimodal_output={"hidden_states": {"output": torch.ones(3, 4)}},
outputs=[],
)
asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, req_state.request_id, 0, encoder_output, req_state))
received = diffusion_stage.request
assert received is not None
assert received.prompt["prompt_embeds"].shape == (3, 4)
assert received.prompt["additional_information"]["auk"] == {
"gen_seconds": 2,
"sway": -0.5,
"t_grid": [0.0, 0.25, 1.0],
"vae_sample": True,
"has_audio": False,
}
assert received.sampling_params.num_inference_steps == 4
assert received.sampling_params.guidance_scale == 1.5
assert received.sampling_params.guidance_scale_provided is True
assert received.sampling_params.seed == 9
runner = mocker.Mock(spec=DiffusionModelRunner)
runner.device = torch.device("cpu")
DiffusionModelRunner._initialize_generator(runner, received.sampling_params)
assert received.sampling_params.generator.initial_seed() == 9
pipeline_params = DiffusionRequestBatch(requests=[received]).sampling_params
assert pipeline_params is received.sampling_params
assert pipeline_params.generator.initial_seed() == 9
def test_auk_instructions_take_precedence_over_input(auk_adapter, mocker):
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
request = OpenAICreateSpeechRequest(
input="ignored input",
instructions='Generate speech based on the following description: "Speak warmly". The content to speak is: "Hello".',
duration_seconds=2,
)
assert auk_adapter.validate(request) is None
prepared = asyncio.run(auk_adapter.build(request, [], False))
prompt = prepared.prompt
assert 'Generate speech based on the following description: "Speak warmly".' in prompt["prompt"]
assert 'The content to speak is: "Hello".' in prompt["prompt"]
assert "|<no_prompt_audio>|" in prompt["prompt"]
assert prompt["additional_information"]["auk"]["gen_seconds"] == 2
warning.assert_called_once()
assert "using the complete `instructions`" in warning.call_args.args[0]
def test_auk_input_only_is_treated_as_complete_instruction(auk_adapter, mocker):
warning = mocker.patch("vllm_omni.entrypoints.openai.tts_adapters.auk.logger.warning")
request = OpenAICreateSpeechRequest(
input="Say the following with the same voice: 'Hello world.'",
ref_audio="reference.wav",
duration_seconds=2,
)
assert auk_adapter.validate(request) is None
prepared = asyncio.run(auk_adapter.build(request, [], True))
assert "Say the following with the same voice: 'Hello world.'" in prepared.prompt["prompt"]
assert warning.call_count == 1
assert 'Prefer `input=""`' in warning.call_args.args[0]
def test_auk_complete_instruction_does_not_require_spoken_text(auk_adapter):
request = OpenAICreateSpeechRequest(
input="",
instructions="Replace 'morning' with 'evening' in the source recording.",
ref_audio="reference.wav",
)
assert auk_adapter.validate(request) is None
prepared = asyncio.run(auk_adapter.build(request, [], True))
assert prepared.prompt["prompt"].count("Replace 'morning'") == 1
assert "Generate speech based on" not in prepared.prompt["prompt"]
assert prepared.prompt["additional_information"]["auk"]["gen_seconds"] is None
def test_stage_model_path_prefers_typed_override():
engine_client = SimpleNamespace(
stage_configs=[
@@ -182,6 +182,7 @@ _PIPELINE_STAGES = [
"token2text",
"token2wav",
"tts",
"encoder",
]
_STAGES = [*_PIPELINE_STAGES, None, "vae", "not_a_real_stage"]
@@ -283,14 +284,22 @@ def test_pipeline_stage_list_is_complete():
assert not missing, f"stage keys claimed by adapters but absent from _PIPELINE_STAGES: {sorted(missing)}"
def test_entry_stage_archs_is_ming_only():
"""Only Ming dense identifies its entry stage by architecture.
def test_entry_stage_archs_are_explicit():
"""Only Ming dense and AuK identify their entry stage by architecture.
Widening this set would change which stage ``_find_tts_stage`` selects in
mixed deployments — notably VoxCPM2, which declares ``model_archs`` but is
found by its ``latent_generator`` stage key.
Both use a generic ``model_stage`` (``llm`` and ``encoder``) that other
pipelines share. Widening this set would change which stage
``_find_tts_stage`` selects in mixed deployments — notably VoxCPM2, which
declares ``model_archs`` but is found by its ``latent_generator`` stage key.
"""
assert tts_entry_stage_archs() == frozenset({"MingTTSForConditionalGeneration"})
assert tts_entry_stage_archs() == frozenset({"MingTTSForConditionalGeneration", "AuKForConditionalGeneration"})
def test_minimax_h3_encoder_is_not_a_tts_stage():
"""MiniMax H3 shares AuK's ``encoder`` stage key but is not a speech model."""
assert "encoder" not in all_tts_stage_keys()
assert detect_tts_model_type("encoder", "MiniMaxH3Encoder") is None
assert detect_tts_model_type("encoder", "AuKForConditionalGeneration") == "auk"
def test_every_detected_type_is_adapter_backed():
+1
View File
@@ -1475,6 +1475,7 @@ class OnlineOmniClient:
"max_new_tokens",
"seed",
"instructions",
"duration_seconds",
"speed",
"sample_rate",
"extra_params",
+3
View File
@@ -214,6 +214,9 @@ def test_speech_extra_params_reach_model_sampler_as_sampling_metadata(monkeypatc
)
class Adapter:
def normalize(self, request):
return
def validate(self, request):
return None