From d0f073686afc4e740373ef709df6e2de26cd6698 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:08:24 +0800 Subject: [PATCH] [CI/Build] Split long weekly TTS jobs and route slow full-model cases to nightly (#8239) Signed-off-by: wangyu <410167048@qq.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com> --- .buildkite/cuda/test-nightly.yml | 16 +-- .buildkite/cuda/test-weekly.yml | 124 +++++++++++++++++- .buildkite/npu/test-npu-nightly.yml | 5 + .../test_hunyuan_image3_pixel_accuracy.py | 1 - .../test_minicpmo_4_5_window.py | 7 +- .../e2e/online_serving/test_pi05_expansion.py | 4 +- 6 files changed, 136 insertions(+), 21 deletions(-) diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 4f6a88f52..376e322ed 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -21,7 +21,7 @@ steps: - omni_minicpmo_4_5_function timeout_in_minutes: 120 commands: - - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py + - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py - label: ":full_moon: Omni · Function Test with H100 · 2-GPU" source_file_dependencies: @@ -33,9 +33,10 @@ steps: - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test" source_file_dependencies: omni_minicpmo_4_5_duplex_function - timeout_in_minutes: 50 + timeout_in_minutes: 120 commands: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model" + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model" - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU" source_file_dependencies: omni_qwen2_5_omni_doc @@ -229,17 +230,6 @@ steps: -m "full_model and diffusion and H100 and B200 and cards_1" --run-level "full_model" - - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU" - timeout_in_minutes: 120 - commands: - - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client" - - >- - pytest -sv - tests/e2e/online_serving/test_pi05_expansion.py - -m "full_model and diffusion and H100 and cards_1" - --run-level "full_model" - mirror_hardwares: h100_1 - - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU" source_file_dependencies: diffusion_qwen_image_function timeout_in_minutes: 120 diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 1d864c176..928270723 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -152,15 +152,122 @@ steps: --run-level "full_model" mirror_hardwares: h100_4 - - label: "TTS · L4" - timeout_in_minutes: 120 + - label: "TTS · Moss · L4" + timeout_in_minutes: 90 commands: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_USE_DEEP_GEMM="0" - export VLLM_MOE_USE_DEEP_GEMM="0" - | failed=0 - for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do + for f in \ + tests/e2e/offline_inference/test_moss_tts_expansion.py \ + tests/e2e/offline_inference/test_moss_tts_nano_expansion.py \ + tests/e2e/offline_inference/test_moss_tts_realtime.py \ + tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py \ + tests/e2e/online_serving/test_moss_tts_expansion.py \ + tests/e2e/online_serving/test_moss_tts_nano_expansion.py + do + pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 + done + exit $$failed + mirror_hardwares: l4_1 + + - label: "TTS · VoxCPM2 · Voxtral · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + failed=0 + for f in \ + tests/e2e/offline_inference/test_voxcpm2_tts.py \ + tests/e2e/online_serving/test_voxcpm2_tts.py \ + tests/e2e/online_serving/test_voxcpm2_tts_expansion.py \ + tests/e2e/online_serving/test_voxcpm2_tts_lora.py \ + tests/e2e/offline_inference/test_voxtral_tts_expansion.py \ + tests/e2e/online_serving/test_voxtral_tts_expansion.py + do + pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 + done + exit $$failed + mirror_hardwares: l4_1 + + - label: "TTS · Higgs · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + failed=0 + for f in \ + tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py \ + tests/e2e/online_serving/test_higgs_audio_v2_expansion.py \ + tests/e2e/online_serving/test_higgs_audio_v3.py + do + pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 + done + exit $$failed + mirror_hardwares: l4_1 + + - label: "TTS · OmniVoice · Step-Audio · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + failed=0 + for f in \ + tests/e2e/offline_inference/test_omnivoice_expansion.py \ + tests/e2e/online_serving/test_omnivoice_expansion.py \ + tests/e2e/online_serving/test_omnivoice_parity.py \ + tests/e2e/offline_inference/test_step_audio2_expansion.py + do + pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 + done + exit $$failed + mirror_hardwares: l4_1 + + - label: "TTS · Ming · GLM · IndexTTS · Breeze · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + failed=0 + for f in \ + tests/e2e/offline_inference/test_ming_tts_expansion.py \ + tests/e2e/online_serving/test_ming_tts_expansion.py \ + tests/e2e/offline_inference/test_glm_tts_expansion.py \ + tests/e2e/online_serving/test_glm_tts_expansion.py \ + tests/e2e/offline_inference/test_indextts2_expansion.py \ + tests/e2e/online_serving/test_indextts2_expansion.py \ + tests/e2e/online_serving/test_breeze_tts_2.py + do + pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 + done + exit $$failed + mirror_hardwares: l4_1 + + - label: "TTS · Audex · CosyVoice · Qwen3-TTS · L4" + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - | + failed=0 + for f in \ + tests/e2e/offline_inference/test_audex_tts.py \ + tests/e2e/offline_inference/test_audex_s2t.py \ + tests/e2e/offline_inference/test_audex_tta.py \ + tests/e2e/offline_inference/test_cosyvoice3_expansion.py \ + tests/e2e/online_serving/test_qwen3_tts_word_timestamps.py + do pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1 done exit $$failed @@ -224,6 +331,17 @@ steps: # Split the old shared H100 Single-GPU sweep: one pytest process collected # every slow+cards_1 diffusion file and exceeded 150 minutes. + - label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU" + timeout_in_minutes: 120 + commands: + - pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client" + - >- + pytest -sv + tests/e2e/online_serving/test_pi05_expansion.py + -m "slow and diffusion and H100 and cards_1" + --run-level "full_model" + mirror_hardwares: h100_1 + - label: "Diffusion · Qwen-Image · H100 · Single-GPU" timeout_in_minutes: 120 commands: diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index d500de157..cc2fed0fe 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -64,6 +64,11 @@ steps: tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "npu and A3 and omni" --run-level full_model + - >- + pytest -s -v + tests/e2e/online_serving/test_minicpmo_4_5_window.py + -m "full_model and npu and A3 and omni" + --run-level full_model - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test" source_file_dependencies: omni_minicpmo_4_5_accuracy timeout_in_minutes: 180 diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 15aa4d739..6399425e8 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -89,7 +89,6 @@ _DEPLOY_CONFIG: dict[str, Any] = { "trust_remote_code": True, "devices": "0,1,2,3", # set dynamically by _write_deploy_config "vae_use_slicing": False, - "moe_backend": "flashinfer_cutlass", "vae_use_tiling": False, "parallel_config": { "pipeline_parallel_size": 1, diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_window.py b/tests/e2e/online_serving/test_minicpmo_4_5_window.py index 300205eb0..3a62ae276 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_window.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_window.py @@ -16,7 +16,7 @@ from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import ( from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn from tests.helpers.mark import hardware_test -pytestmark = [pytest.mark.omni, pytest.mark.advanced_model] +pytestmark = [pytest.mark.omni] def _assert_complete(result, *, require_audio=False): @@ -30,6 +30,7 @@ def _assert_complete(result, *, require_audio=False): @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.advanced_model @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) @pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)]) def test_window_rebuild_and_next_session(omni_server, mode, reference): @@ -47,7 +48,8 @@ def test_window_rebuild_and_next_session(omni_server, mode, reference): _assert_complete(result, require_audio=reference) -@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1) +@pytest.mark.full_model @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) @pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)]) def test_window_continuous_input(omni_server, tmp_path, mode, camera): @@ -71,6 +73,7 @@ def test_window_continuous_input(omni_server, tmp_path, mode, camera): @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.advanced_model @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) @pytest.mark.parametrize("mode", ["off", "basic", "context"]) def test_window_buffered_flush(omni_server, mode): diff --git a/tests/e2e/online_serving/test_pi05_expansion.py b/tests/e2e/online_serving/test_pi05_expansion.py index 0c6fbfd34..dab64dd4c 100644 --- a/tests/e2e/online_serving/test_pi05_expansion.py +++ b/tests/e2e/online_serving/test_pi05_expansion.py @@ -11,7 +11,7 @@ websocket (``/v1/realtime/robot/openpi``) — the same wire path a robot uses The ``pi0_openpi_*`` helpers are protocol-level (handshake, send obs, receive chunk) and not π0-specific, so π0.5 reuses them; only the deploy config and the -expected metadata differ. This file is selected explicitly by the nightly H100 +expected metadata differ. This file is selected explicitly by the weekly H100 robot-policy job. The in-process LeRobot parity oracle lives separately in @@ -59,7 +59,7 @@ test_params = [ ] -@pytest.mark.full_model +@pytest.mark.slow @pytest.mark.diffusion @hardware_test(res={"cuda": "H100"}) @pytest.mark.parametrize("omni_server", test_params, indirect=True)