From c167b9d69190299070159e053a0be0a6db7f2cc1 Mon Sep 17 00:00:00 2001 From: zhumingjue138 Date: Wed, 13 May 2026 09:15:20 +0800 Subject: [PATCH] [Bugfix] Fix the issue where the qwen3-omni model long-term stability test sometimes gets stuck without sending requests. (#3468) Signed-off-by: zhumingjue --- tests/dfx/conftest.py | 21 ++++++-- .../dfx/stability/tests/test_qwen3_omni.json | 54 +------------------ tests/dfx/stability/tests/test_qwen3_tts.json | 25 +-------- .../dfx/stability/tests/test_qwen_image.json | 2 +- tests/dfx/stability/tests/test_wan22.json | 2 +- 5 files changed, 21 insertions(+), 83 deletions(-) diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 0c59bed99..9332a24f5 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -2,6 +2,7 @@ import json import os import re import subprocess +import threading import time from datetime import datetime from pathlib import Path @@ -377,11 +378,23 @@ def run_benchmark( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True ) - for line in iter(process.stdout.readline, ""): - print(line, end=" ") + if process.stdout is None or process.stderr is None: + raise RuntimeError("Failed to capture benchmark process output streams") - for line in iter(process.stderr.readline, ""): - print(line, end=" ") + def _forward_stream(stream) -> None: + try: + for line in iter(stream.readline, ""): + print(line, end="") + finally: + stream.close() + + stdout_thread = threading.Thread(target=_forward_stream, args=(process.stdout,)) + stderr_thread = threading.Thread(target=_forward_stream, args=(process.stderr,)) + stdout_thread.start() + stderr_thread.start() + stdout_thread.join() + stderr_thread.join() + process.wait() if "--result-dir" in command: index = command.index("--result-dir") diff --git a/tests/dfx/stability/tests/test_qwen3_omni.json b/tests/dfx/stability/tests/test_qwen3_omni.json index a16ab805c..cbdf25e67 100644 --- a/tests/dfx/stability/tests/test_qwen3_omni.json +++ b/tests/dfx/stability/tests/test_qwen3_omni.json @@ -1,60 +1,8 @@ [ - { - "test_name": "test_qwen3_omni_stability", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "stage_overrides": { - "2": { - "max_num_batched_tokens": 1000000 - } - } - }, - "benchmark_params": [ - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "duration_sec": 86400, - "request_rate": 0.3, - "num_prompts_per_batch": 10, - "random_input_len": { - "min": 0, - "max": 8000 - }, - "random_output_len": { - "min": 0, - "max": 1000 - }, - "random_range_ratio": 0.0, - "random_mm_base_items_per_request": { - "min": 0, - "max": 6 - }, - "random_mm_num_mm_items_range_ratio": 0.0, - "random_mm_limit_mm_per_prompt": { - "image": 2, - "video": 2, - "audio": 2 - }, - "random_mm_bucket_config": { - "(128-1024, 128-1024, 1)": 0.34, - "(256-1080, 256-1920, 2-16)": 0.33, - "(0, 1-60, 1-3)": 0.33 - }, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration" - } - ] - }, { "test_name": "test_qwen3_omni_stability_async_chunk", "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "stage_overrides": { - "2": { - "max_num_batched_tokens": 1000000 - } - }, "extra_cli_args": ["--async-chunk"] }, "benchmark_params": [ @@ -64,7 +12,7 @@ "endpoint": "/v1/chat/completions", "duration_sec": 86400, "max_concurrency": 2, - "num_prompts_per_batch": 10, + "num_prompts_per_batch": 50, "random_input_len": { "min": 0, "max": 8000 diff --git a/tests/dfx/stability/tests/test_qwen3_tts.json b/tests/dfx/stability/tests/test_qwen3_tts.json index fbf30d88a..0208331bd 100644 --- a/tests/dfx/stability/tests/test_qwen3_tts.json +++ b/tests/dfx/stability/tests/test_qwen3_tts.json @@ -5,36 +5,13 @@ "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" }, "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-audio-speech", - "endpoint": "/v1/audio/speech", - "duration_sec": 86400, - "request_rate": 0.3, - "num_prompts_per_batch": 10, - "random_input_len": { - "min": 0, - "max": 1000 - }, - "random_output_len": { - "min": 0, - "max": 1000 - }, - "random_range_ratio": 0.0, - "extra_body": { - "voice": "Vivian", - "language": "English" - }, - "ignore_eos": true, - "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration" - }, { "dataset_name": "random", "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "duration_sec": 86400, "max_concurrency": 2, - "num_prompts_per_batch": 10, + "num_prompts_per_batch": 50, "random_input_len": { "min": 0, "max": 1000 diff --git a/tests/dfx/stability/tests/test_qwen_image.json b/tests/dfx/stability/tests/test_qwen_image.json index f3dd93f6f..e28614973 100644 --- a/tests/dfx/stability/tests/test_qwen_image.json +++ b/tests/dfx/stability/tests/test_qwen_image.json @@ -11,7 +11,7 @@ "backend": "vllm-omni", "duration_sec": 86400, "max_concurrency": 1, - "num_prompts_per_batch": 10, + "num_prompts_per_batch": 50, "width": { "min": 512, "max": 2048 diff --git a/tests/dfx/stability/tests/test_wan22.json b/tests/dfx/stability/tests/test_wan22.json index c787ce96a..7b3b54b65 100644 --- a/tests/dfx/stability/tests/test_wan22.json +++ b/tests/dfx/stability/tests/test_wan22.json @@ -19,7 +19,7 @@ "backend": "v1/videos", "duration_sec": 86400, "max_concurrency": 1, - "num_prompts_per_batch": 20, + "num_prompts_per_batch": 50, "enable_negative_prompt": true, "random_request_config": [ {"width": 832, "height": 480, "num_inference_steps": 4, "num_frames": 81, "fps": 16, "weight": 0.5},