[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469)

Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-30 00:18:38 +00:00
1 parent 3a28882946
commit 35cf6ac355
22 files changed
+1473 -38

No files matched your search

+24 -3
View File
@@ -1,4 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Universal TTS benchmark CLI for vllm-omni.
Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
@@ -87,6 +90,7 @@ def build_bench_args(
served_model_name: str | None = None,
num_warmups: int = 2,
request_seed: int | None = None,
duration_seconds: float | None = None,
) -> list[str]:
"""Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
dataset_name = _TASK_TO_DATASET[task]
@@ -95,7 +99,10 @@ def build_bench_args(
task_extra_body: dict[str, Any] = dict((model_cfg.get("task_extra_body") or {}).get(task) or {})
if request_seed is not None:
task_extra_body["seed"] = request_seed
if duration_seconds is not None:
if not math.isfinite(duration_seconds) or duration_seconds <= 0:
raise ValueError("duration_seconds must be finite and positive")
task_extra_body["duration_seconds"] = duration_seconds
# Resolve dataset path
if dataset_path:
resolved_dataset_path = dataset_path
@@ -194,7 +201,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
if not results:
return
header = (
f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
f"{'Task':<16} {'Task Type':<12} {'Concurrency':>11} {'RTF mean':>10} "
f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
)
print(f"\n{'=' * len(header)}")
@@ -204,6 +211,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
print("-" * len(header))
for r in results:
task = r.get("_task", "?")
task_type = r.get("_task_type", "-")
conc = r.get("_concurrency", "?")
rtf = r.get("mean_audio_rtf", float("nan"))
ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
@@ -216,7 +224,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
return f"{v:.{digits}f}" if not math.isnan(v) else " n/a"
print(
f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
f"{task:<16} {task_type:<12} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
)
print("=" * len(header))
@@ -249,6 +257,12 @@ def main() -> None:
help="Number of prompts per run. If one value, applied to all concurrency levels.",
)
parser.add_argument("--num-warmups", type=int, default=2, help="Warmup requests before each measured run")
parser.add_argument(
"--duration-seconds",
type=float,
default=None,
help="Target audio duration per request (AuK defaults to 5 seconds). Independent of --output-len.",
)
parser.add_argument(
"--request-seed",
type=int,
@@ -277,6 +291,8 @@ def main() -> None:
if args.num_warmups < 0:
parser.error("--num-warmups cannot be negative")
if args.duration_seconds is not None and (not math.isfinite(args.duration_seconds) or args.duration_seconds <= 0):
parser.error("--duration-seconds must be finite and positive")
model_configs = load_model_configs(Path(args.model_configs))
if args.model not in model_configs:
@@ -333,11 +349,16 @@ def main() -> None:
served_model_name=args.served_model_name,
num_warmups=args.num_warmups,
request_seed=args.request_seed,
duration_seconds=args.duration_seconds,
)
result = run_one_benchmark(cmd)
if result is not None:
result["_task"] = task
result["_concurrency"] = concurrency
task_body = model_cfg.get("task_extra_body", {}).get(task, {})
task_type = task_body.get("task_type")
if task_type is not None:
result["_task_type"] = task_type
all_results.append(result)
# Persist the metadata so plot_results.py can pick it up.
if args.output_dir and result_filename:
+16
View File
@@ -7,6 +7,22 @@
# is tracked here.
models:
tencent/AuK: &auk
supported_tasks: [default_voice, voice_clone]
backend: openai-audio-speech
endpoint: /v1/audio/speech
trust_remote_code: true
task_extra_body:
default_voice:
duration_seconds: 5.0
voice: default
task_type: CustomVoice
voice_clone:
duration_seconds: 5.0
voice: default
task_type: Base
tencent/AuK-Flash: *auk
BreezeBlue/Breeze-TTS-2:
supported_tasks: [default_voice, voice_design, voice_clone]
backend: openai-audio-speech