[Feat][Perf][AuK]Support Online Serving and DiT CUDAGraph (#7469)
Signed-off-by: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
22 files changed
+1473
-38
No files matched your search
@@ -1,4 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Universal TTS benchmark CLI for vllm-omni.
|
||||
|
||||
Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
|
||||
@@ -87,6 +90,7 @@ def build_bench_args(
|
||||
served_model_name: str | None = None,
|
||||
num_warmups: int = 2,
|
||||
request_seed: int | None = None,
|
||||
duration_seconds: float | None = None,
|
||||
) -> list[str]:
|
||||
"""Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
|
||||
dataset_name = _TASK_TO_DATASET[task]
|
||||
@@ -95,7 +99,10 @@ def build_bench_args(
|
||||
task_extra_body: dict[str, Any] = dict((model_cfg.get("task_extra_body") or {}).get(task) or {})
|
||||
if request_seed is not None:
|
||||
task_extra_body["seed"] = request_seed
|
||||
|
||||
if duration_seconds is not None:
|
||||
if not math.isfinite(duration_seconds) or duration_seconds <= 0:
|
||||
raise ValueError("duration_seconds must be finite and positive")
|
||||
task_extra_body["duration_seconds"] = duration_seconds
|
||||
# Resolve dataset path
|
||||
if dataset_path:
|
||||
resolved_dataset_path = dataset_path
|
||||
@@ -194,7 +201,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
|
||||
if not results:
|
||||
return
|
||||
header = (
|
||||
f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
|
||||
f"{'Task':<16} {'Task Type':<12} {'Concurrency':>11} {'RTF mean':>10} "
|
||||
f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
|
||||
)
|
||||
print(f"\n{'=' * len(header)}")
|
||||
@@ -204,6 +211,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
|
||||
print("-" * len(header))
|
||||
for r in results:
|
||||
task = r.get("_task", "?")
|
||||
task_type = r.get("_task_type", "-")
|
||||
conc = r.get("_concurrency", "?")
|
||||
rtf = r.get("mean_audio_rtf", float("nan"))
|
||||
ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
|
||||
@@ -216,7 +224,7 @@ def print_summary_table(results: list[dict[str, Any]]) -> None:
|
||||
return f"{v:.{digits}f}" if not math.isnan(v) else " n/a"
|
||||
|
||||
print(
|
||||
f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
|
||||
f"{task:<16} {task_type:<12} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
|
||||
f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
|
||||
)
|
||||
print("=" * len(header))
|
||||
@@ -249,6 +257,12 @@ def main() -> None:
|
||||
help="Number of prompts per run. If one value, applied to all concurrency levels.",
|
||||
)
|
||||
parser.add_argument("--num-warmups", type=int, default=2, help="Warmup requests before each measured run")
|
||||
parser.add_argument(
|
||||
"--duration-seconds",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Target audio duration per request (AuK defaults to 5 seconds). Independent of --output-len.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--request-seed",
|
||||
type=int,
|
||||
@@ -277,6 +291,8 @@ def main() -> None:
|
||||
|
||||
if args.num_warmups < 0:
|
||||
parser.error("--num-warmups cannot be negative")
|
||||
if args.duration_seconds is not None and (not math.isfinite(args.duration_seconds) or args.duration_seconds <= 0):
|
||||
parser.error("--duration-seconds must be finite and positive")
|
||||
|
||||
model_configs = load_model_configs(Path(args.model_configs))
|
||||
if args.model not in model_configs:
|
||||
@@ -333,11 +349,16 @@ def main() -> None:
|
||||
served_model_name=args.served_model_name,
|
||||
num_warmups=args.num_warmups,
|
||||
request_seed=args.request_seed,
|
||||
duration_seconds=args.duration_seconds,
|
||||
)
|
||||
result = run_one_benchmark(cmd)
|
||||
if result is not None:
|
||||
result["_task"] = task
|
||||
result["_concurrency"] = concurrency
|
||||
task_body = model_cfg.get("task_extra_body", {}).get(task, {})
|
||||
task_type = task_body.get("task_type")
|
||||
if task_type is not None:
|
||||
result["_task_type"] = task_type
|
||||
all_results.append(result)
|
||||
# Persist the metadata so plot_results.py can pick it up.
|
||||
if args.output_dir and result_filename:
|
||||
|
||||
@@ -7,6 +7,22 @@
|
||||
# is tracked here.
|
||||
|
||||
models:
|
||||
tencent/AuK: &auk
|
||||
supported_tasks: [default_voice, voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
trust_remote_code: true
|
||||
task_extra_body:
|
||||
default_voice:
|
||||
duration_seconds: 5.0
|
||||
voice: default
|
||||
task_type: CustomVoice
|
||||
voice_clone:
|
||||
duration_seconds: 5.0
|
||||
voice: default
|
||||
task_type: Base
|
||||
tencent/AuK-Flash: *auk
|
||||
|
||||
BreezeBlue/Breeze-TTS-2:
|
||||
supported_tasks: [default_voice, voice_design, voice_clone]
|
||||
backend: openai-audio-speech
|
||||
|
||||
Reference in new issue
Block a user