[Test] Add Qwen3-TTS nightly performance benchmark (#1700)
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed
1 parent
a6b5519616
commit
f144f5effe
4 files changed
+192
-46
No files matched your search
@@ -107,8 +107,14 @@ def omni_server(request):
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_prompt) -> Any:
|
||||
"""Generate synthetic image with random values."""
|
||||
def run_benchmark(
|
||||
args: list,
|
||||
test_name: str,
|
||||
flow,
|
||||
dataset_name: str,
|
||||
num_prompt,
|
||||
) -> Any:
|
||||
"""Run a single benchmark iteration and return the parsed result JSON."""
|
||||
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_{current_dt}.json"
|
||||
if "--result-filename" in args:
|
||||
@@ -117,10 +123,6 @@ def run_benchmark(args: list, test_name: str, flow, dataset_name: str, num_promp
|
||||
["vllm", "bench", "serve", "--omni"]
|
||||
+ args
|
||||
+ [
|
||||
"--backend",
|
||||
"openai-chat-omni",
|
||||
"--endpoint",
|
||||
"/v1/chat/completions",
|
||||
"--save-result",
|
||||
"--result-dir",
|
||||
os.environ.get("BENCHMARK_DIR", "tests"),
|
||||
@@ -196,7 +198,10 @@ def benchmark_params(request, omni_server):
|
||||
total = len(all_params)
|
||||
print(f"\n Running benchmark {current}/{total} for {test_name}")
|
||||
|
||||
return {"test_name": test_name, "params": all_params[param_index]}
|
||||
return {
|
||||
"test_name": test_name,
|
||||
"params": all_params[param_index],
|
||||
}
|
||||
|
||||
|
||||
def assert_result(result, params, num_prompt):
|
||||
@@ -266,7 +271,11 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
for qps, num_prompt in zip(qps_list, num_prompt_list):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
result = run_benchmark(
|
||||
args=args, test_name=test_name, flow=qps, dataset_name=dataset_name, num_prompt=num_prompt
|
||||
args=args,
|
||||
test_name=test_name,
|
||||
flow=qps,
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
)
|
||||
assert_result(result, params, num_prompt=num_prompt)
|
||||
|
||||
@@ -274,6 +283,10 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
|
||||
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
|
||||
result = run_benchmark(
|
||||
args=args, test_name=test_name, flow=concurrency, dataset_name=dataset_name, num_prompt=num_prompt
|
||||
args=args,
|
||||
test_name=test_name,
|
||||
flow=concurrency,
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
)
|
||||
assert_result(result, params, num_prompt=num_prompt)
|
||||
@@ -0,0 +1,99 @@
|
||||
# Stage config for running Qwen3-TTS with 2-stage architecture
|
||||
# Stage 0: Talker (text -> 8-layer RVQ codec codes)
|
||||
# Stage 1: Code2Wav (codec codes -> audio waveform)
|
||||
#
|
||||
# The following config has been verified on 1x H100-80G GPU.
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 4
|
||||
engine_args:
|
||||
model_stage: qwen3_tts
|
||||
model_arch: Qwen3TTSTalkerForConditionalGeneration
|
||||
hf_overrides:
|
||||
architectures: [Qwen3TTSTalkerForConditionalGeneration]
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.3
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 512
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_tts.talker2code2wav_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 4
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3TTSCode2Wav
|
||||
hf_overrides:
|
||||
architectures: [Qwen3TTSCode2Wav]
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.2
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 32768
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
tts_args:
|
||||
max_instructions_length: 500
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1
|
||||
max_inflight: 4
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
window_size: -1
|
||||
@@ -8,6 +8,8 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -28,6 +30,8 @@
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -88,6 +92,8 @@
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -108,6 +114,8 @@
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -140,5 +148,38 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_qwen3_tts",
|
||||
"server_params": {
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"stage_config_name": "qwen3_tts.yaml"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"extra_body": {
|
||||
"voice": "Vivian",
|
||||
"language": "English"
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user