[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)
Signed-off-by: R2-Y <ruiruyang2@gmail.com> Signed-off-by: psv666 <2693925048@qq.com> Co-authored-by: psv666 <2693925048@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
59 files changed
+1908
-155
No files matched your search
@@ -8,7 +8,7 @@
|
||||
# NPU naming: {queue}_{resource_class}
|
||||
# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
|
||||
# a2b3_npu_4 — ascend-a2b3, resource_class npu-4
|
||||
# a3_npu_2 — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
|
||||
# a3_npu_2 — ascend-a3, resource_class npu-2
|
||||
|
||||
x-npu-k8s-plugins: &npu_k8s_plugins
|
||||
- kubernetes:
|
||||
|
||||
@@ -139,7 +139,7 @@ steps:
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
source_file_dependencies:
|
||||
@@ -148,10 +148,10 @@ steps:
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||
- tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||
- tests/helpers/minicpmo_4_5_duplex.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
@@ -164,7 +164,7 @@ steps:
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -22,7 +22,7 @@ steps:
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with L4"
|
||||
timeout_in_minutes: 90
|
||||
@@ -50,6 +50,19 @@ steps:
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export SEED_TTS_WER_EVAL=1
|
||||
- export SEED_TTS_EVAL_DEVICE=cuda:1
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
||||
key: nightly-omni-performance-no-async-chunk
|
||||
timeout_in_minutes: 300
|
||||
@@ -102,6 +115,32 @@ steps:
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_3
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
|
||||
timeout_in_minutes: 45
|
||||
commands:
|
||||
|
||||
@@ -172,7 +172,7 @@ steps:
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Online Test"
|
||||
source_file_dependencies:
|
||||
@@ -183,7 +183,7 @@ steps:
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
source_file_dependencies:
|
||||
@@ -192,10 +192,10 @@ steps:
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||
- tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||
- tests/helpers/minicpmo_4_5_duplex.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
@@ -210,7 +210,7 @@ steps:
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -39,7 +39,7 @@ steps:
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
@@ -47,6 +47,34 @@ steps:
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
-m "npu and A3 and omni"
|
||||
--run-level full_model
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: nightly-tts-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
|
||||
@@ -3,22 +3,20 @@ steps:
|
||||
key: ready-omni-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-2"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpecPatch:
|
||||
imagePullSecrets:
|
||||
- name: swr-secret
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: ready-tts-group
|
||||
|
||||
Reference in new issue
Block a user