diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 9c311376b..724129cea 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -83,7 +83,7 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - | set +e - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model" EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 0b4ca2740..3bc6c16d9 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -74,6 +74,19 @@ steps: exit $$EXIT mirror_hardwares: h100_2 + - label: ":full_moon: Omni · Perf Test · Async Chunk · Random" + key: weekly-omni-performance-async-chunk-random + timeout_in_minutes: 180 + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow" + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + mirror_hardwares: h100_2 + - label: ":full_moon: Omni · Perf Test · Multi-Replica" key: weekly-omni-performance-multi-replicas timeout_in_minutes: 180 diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index ec1e3d01b..3a12779c3 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -4,8 +4,8 @@ When you want to add L4-level ***performance test*** cases, add entries to JSON | Model type | Runner | Example JSON files | | ---------- | ------ | ------------------ | -| Omni (nightly) | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json` (`full_model` in `mark`) | -| Omni (weekly) | `run_benchmark.py` | `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` (`slow` in `mark`; **Perf Test** in `test-weekly.yml`) | +| Omni (nightly) | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json` (`full_model` without `slow` in `mark`) | +| Omni (weekly) | `run_benchmark.py` | `test_qwen3_omni_async_chunk.json` (CUDA only), `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` (`slow` in `mark`; **Perf Test** in `test-weekly.yml`) | | TTS | `run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | | Diffusion | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, … | @@ -20,7 +20,7 @@ Diffusion cases are detected when the JSON has `server_type` (typically `"vllm-o #### Running perf cases -Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bulk scan above and filter with pytest **`-m`** on each case's JSON `mark` (for example `pytest … run_benchmark.py -m "full_model and tts and H100"` or `-m "slow and omni and H100"` for weekly Omni configs). +Pass **`--test-config-file`** to load one JSON file, or omit it for the bulk scan above. In either mode, pytest **`-m`** filters each case using its JSON `mark`. A single file can contain nightly, weekly, and platform-specific case objects. For example, the nightly object in `test_qwen3_omni_async_chunk.json` declares both H100 and A3, while its `slow` weekly object is CUDA-only; there is no NPU weekly counterpart. Use `-m "H100 and full_model and not slow"`, `-m "H100 and full_model and slow"`, or `-m "npu"` to select the intended schedule and platform. ```JSON { @@ -75,7 +75,7 @@ Omit `mark` only for configs not meant to be filtered by `-m`. `server_type` is Optional top-level field on each perf JSON **case object** (one per `test_name`). `run_benchmark.py` and `run_diffusion_benchmark.py` read it via `tests.dfx.conftest.resolve_pytest_marks` and attach the marks to the corresponding `pytest.param`, so you can filter locally with `-m`. -When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same shape as `@hardware_test` / `hardware_marks()` in `tests/helpers/mark.py`), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, `diffusion`, or `local_model`. +When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same shape as `@hardware_test` / `hardware_marks()` in `tests/helpers/mark.py`), followed by registered pytest marker name strings such as `full_model`, `slow`, `omni`, `tts`, `diffusion`, or `local_model`. Supported form: @@ -142,11 +142,18 @@ Examples: pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and H100 and diffusion" pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" -# Single file (same as nightly CI Perf steps) +# Single file (same selectors as the CI Perf steps) pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ - --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \ + -m "H100 and full_model and not slow" +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \ + -m "H100 and full_model and slow" +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json \ + -m "npu" ``` diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 666e993bb..5995599b0 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -21,47 +21,6 @@ ] }, "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "H100": { - "mean_e2el_ms": [ - 11701.3818, - 27510.3956, - 31907.9613, - 42173.9832, - 63970.0751 - ], - "mean_audio_rtf": [ - 0.1175, - 0.1647, - 0.2003, - 0.2726, - 0.4195 - ] - } - } - }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", @@ -170,45 +129,6 @@ } } }, - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "H100": { - "mean_e2el_ms": [ - 4354.8945, - 7672.4248, - 9861.1687, - 13086.8865, - 18716.709 - ] - } - } - }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", @@ -324,5 +244,61 @@ } } ] + }, + { + "test_name": "test_qwen3_omni_chunk_cuda", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "omni", + "slow" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 32, + 128 + ], + "max_concurrency": [ + 1, + 8, + 32 + ], + "random_input_len": 100, + "random_output_len": 100, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "H100": { + "mean_e2el_ms": [ + 4354.8945, + 9861.1687, + 18716.709 + ] + } + } + } + ] } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index d077188ad..ef8d97a94 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -29,16 +29,12 @@ "endpoint": "/v1/completions", "num_prompts": [ 4, - 16, 32, - 64, 128 ], "max_concurrency": [ 1, - 4, 8, - 16, 32 ], "random_input_len": 2500, @@ -54,9 +50,7 @@ "H100": { "mean_e2el_ms": [ 4619.0068, - 7868.3523, 9760.6175, - 13422.3491, 18007.6918 ] } @@ -99,45 +93,6 @@ } } }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "H100": { - "mean_e2el_ms": [ - 524.5321 - ] - } - } - }, { "dataset_name": "random-mm", "backend": "vllm", diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_omni_chat.py b/tests/dfx/reliability/invalid_param_test/test_invalid_omni_chat.py index 954d8e67e..0d026a37b 100644 --- a/tests/dfx/reliability/invalid_param_test/test_invalid_omni_chat.py +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_omni_chat.py @@ -72,6 +72,9 @@ def _chat_completions_request_without_expectations(omni_server: OmniServer, case body["top_logprobs"] = 5 elif case_id == "speaker_unknown": body["speaker"] = "zz_invalid_qwen3_omni_chat_speaker_xyz" + elif case_id == "audio_format_unsupported": + body["modalities"] = ["text", "audio"] + body["audio"] = {"voice": "alloy", "format": "aac"} else: raise AssertionError(f"unknown chat completions invalid case_id {case_id!r}") return {"json": body, "timeout": 120} @@ -135,6 +138,12 @@ def _chat_completions_request_without_expectations(omni_server: OmniServer, case ("Invalid speaker", "Supported"), id="speaker_unknown_preset", ), + pytest.param( + "audio_format_unsupported", + 400, + ("Invalid audio format", "aac", "Supported formats"), + id="unsupported_audio_format", + ), ], ) @pytest.mark.parametrize("omni_server", _QWEN3_OMNI_SERVER, indirect=True) diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 12a65670e..10c5dd8cb 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -7,7 +7,6 @@ E2E Online tests for Qwen3-Omni model. import os import pytest -from openai import BadRequestError from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video @@ -89,7 +88,7 @@ test_params = [ ), ] -test_token_params = [ +reduced_token_params = [ pytest.param( OmniServerParams( model=model, @@ -97,7 +96,7 @@ test_token_params = [ use_stage_cli=True, server_args=["--async-chunk"], ), - id="batch_token_64", + id="batch_token_2048", ) ] @@ -138,29 +137,7 @@ def get_max_batch_size(size_type="few"): @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_text_to_audio_001(omni_server, online_client) -> None: - """ - Input Modal: text - Output Modal: audio - Input Setting: stream=True - Datasets: single request - """ - messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) - - request_config = { - "model": omni_server.model, - "messages": messages, - "modalities": ["audio"], - "stream": True, - "key_words": {"text": ["beijing"]}, - } - - online_client.send_omni_request(request_config) - - -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) +@pytest.mark.parametrize("omni_server", test_params + reduced_token_params, indirect=True) def test_text_to_text_audio_001(omni_server, online_client) -> None: """ Input Modal: text @@ -204,36 +181,13 @@ def test_text_video_to_text_001(omni_server, online_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) -def test_text_audio_to_text_audio_001(omni_server, online_client) -> None: - """ - Input Modal: text, audio - Output Modal: text, audio - Input Setting: stream=False - Datasets: single request - """ - audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" - messages = dummy_messages_from_mix_data( - audio_data_url=audio_data_url, system_prompt=get_system_prompt(), content_text=get_prompt("text_audio") - ) - - request_config = { - "model": omni_server.model, - "messages": messages, - "key_words": {"audio": AUDIO_KEY}, - } - - online_client.send_omni_request(request_config) - - -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) +@pytest.mark.parametrize("omni_server", test_params + reduced_token_params, indirect=True) def test_text_audio_to_text_audio_002(omni_server, online_client) -> None: """ Input Modal: text, long-duration audio (~LONG_AUDIO_DURATION_SEC s WAV) Output Modal: text, audio Input Setting: stream=False - Datasets: single request + Datasets: few requests """ audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(LONG_AUDIO_DURATION_SEC, 1)['base64']}" messages = dummy_messages_from_mix_data( @@ -252,37 +206,13 @@ def test_text_audio_to_text_audio_002(omni_server, online_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) -def test_text_image_to_text_audio_001(omni_server, online_client) -> None: - """ - Input Modal: text, image - Output Modal: text, audio - Input Setting: stream=False - Datasets: single request - """ - image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" - - messages = dummy_messages_from_mix_data( - image_data_url=image_data_url, system_prompt=get_system_prompt(), content_text=get_prompt("text_image") - ) - - request_config = { - "model": omni_server.model, - "messages": messages, - "key_words": {"image": IMAGE_KEY}, - } - - online_client.send_omni_request(request_config) - - -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_large_image_to_text_audio_001(omni_server, online_client) -> None: """ Input Modal: text, high-resolution image (1080p-class JPEG) Output Modal: text, audio Input Setting: stream=False - Datasets: single request + Datasets: few requests """ image_data_url = ( f"data:image/jpeg;base64,{generate_synthetic_image(LARGE_IMAGE_WIDTH, LARGE_IMAGE_HEIGHT)['base64']}" @@ -304,7 +234,7 @@ def test_large_image_to_text_audio_001(omni_server, online_client) -> None: @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_audio_001(omni_server, online_client) -> None: """ Input Modal: text, video @@ -328,36 +258,6 @@ def test_text_video_to_text_audio_001(omni_server, online_client) -> None: online_client.send_omni_request(request_config) -@pytest.mark.skip(reason="There is a known issue with shape mismatch error.") -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params + test_token_params, indirect=True) -def test_mix_to_text_audio_001(omni_server, online_client) -> None: - """ - Input Modal: text, audio, image, video - Output Modal: text, audio - Input Setting: stream=True - Datasets: few requests - """ - video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" - image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" - audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), - video_data_url=video_data_url, - image_data_url=image_data_url, - audio_data_url=audio_data_url, - content_text=get_prompt("mix"), - ) - - request_config = { - "model": omni_server.model, - "messages": messages, - "stream": True, - "key_words": {"audio": AUDIO_KEY, "image": IMAGE_KEY, "video": VIDEO_KEY}, - } - online_client.send_omni_request(request_config, request_num=get_max_batch_size()) - - @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_audio_in_video_001(omni_server, online_client) -> None: @@ -507,22 +407,22 @@ def test_one_word_prompt_001(omni_server, online_client) -> None: @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_speaker_001(omni_server, online_client) -> None: """ - Input Modal: text only (one-word answer constraint). - Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text. + Input Modal: Chinese text. + Output Modal: Chinese text and audio using an uppercase preset speaker name. Input Setting: stream=True Datasets: single request """ messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), - content_text=get_prompt("text"), + content_text=get_prompt("text_chinese"), ) request_config = { "model": omni_server.model, "messages": messages, "stream": True, - "speaker": "Chelsie", - "key_words": {"text": ["beijing"]}, + "speaker": "CHELSIE", + "key_words": {"text": ["北京"]}, } online_client.send_omni_request(request_config) @@ -563,55 +463,6 @@ def test_speaker_002(omni_server, online_client) -> None: print(f"Gender assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}") -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_speaker_003(omni_server, online_client) -> None: - """ - Input Modal: text only (one-word answer constraint). - Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text. - Input Setting: stream=True - Datasets: single request - """ - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), - content_text=get_prompt("text"), - ) - - request_config = { - "model": omni_server.model, - "messages": messages, - "stream": True, - "speaker": "CHELSIE", - "key_words": {"text": ["beijing"]}, - } - - online_client.send_omni_request(request_config) - - -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_language_001(omni_server, online_client) -> None: - """ - Input Modal: text only (one-word answer constraint). - Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text. - Input Setting: stream=True - Datasets: single request - """ - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), - content_text=get_prompt("text_chinese"), - ) - - request_config = { - "model": omni_server.model, - "messages": messages, - "stream": True, - "key_words": {"text": ["北京"]}, - } - - online_client.send_omni_request(request_config) - - @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_long_output_001(omni_server, online_client) -> None: @@ -619,7 +470,7 @@ def test_text_to_audio_long_output_001(omni_server, online_client) -> None: Input Modal: text only (long-form generation prompt). Output Modal: text, audio (default ``modalities``); Input Setting: stream=True - Datasets: single request + Datasets: few requests """ messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), @@ -631,21 +482,3 @@ def test_text_to_audio_long_output_001(omni_server, online_client) -> None: text = responses[0].text_content if responses else "" word_count = len(text.split()) assert word_count >= 200, f"Expected at least 200 words in long output, got {word_count}" - - -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) -@pytest.mark.parametrize("omni_server", test_params[:1], indirect=True) -def test_invalid_audio_format_rejected(omni_server, openai_client) -> None: - """Ensure invalid audio format is rejected with 400 before streaming starts.""" - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), - content_text=get_prompt(), - ) - with pytest.raises(BadRequestError, match="audio format"): - openai_client.client.chat.completions.create( - model=omni_server.model, - messages=messages, - modalities=["text", "audio"], - audio={"voice": "alloy", "format": "aac"}, - stream=True, - )