diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index fa83fe0f7..f416631ff 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -227,6 +227,18 @@ steps: " mirror_hardwares: h100_1 + - label: "Omni · MiniCPM-o 4.5 Perf Test" + timeout_in_minutes: 60 + source_file_dependencies: + - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json + - vllm_omni/deploy/minicpmo_4_5.yaml + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda' + mirror_hardwares: h100_1 + - label: "TTS · VoxCPM2 Test" source_file_dependencies: - tests/e2e/online_serving/test_voxcpm2_tts.py diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index fd7f6f546..f23ba7120 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -17,6 +17,15 @@ steps: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" commands: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model' + - label: "Omni · MiniCPM-o 4.5 Perf Test" + timeout_in_minutes: 60 + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni' - group: ":card_index_dividers: TTS Model Test" key: ready-tts-group diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json new file mode 100644 index 000000000..e327be510 --- /dev/null +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json @@ -0,0 +1,80 @@ +[ + { + "test_name": "test_minicpmo_4_5_ready", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100", + "npu": "A3" + }, + "num_cards": 1 + } + }, + "core_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10, + 40 + ], + "max_concurrency": [ + 1, + 4 + ], + "no_oversample": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", + "baseline": { + "H100": { + "request_throughput": [ + 0.7803, + 1.1008 + ], + "mean_ttft_ms": [ + 166.9764, + 307.7133 + ], + "mean_e2el_ms": [ + 1361.1682, + 3800.6332 + ], + "mean_audio_ttfp_ms": [ + 635.8099, + 1615.6591 + ], + "mean_audio_rtf": [ + 0.3151, + 0.8568 + ] + } + } + } + ] + } +]