[CI/Build] Split long weekly TTS jobs and route slow full-model cases to nightly (#8239)
Signed-off-by: wangyu <410167048@qq.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com>
This commit is contained in:
wangyuandAlicia authored and GitHub committed
1 parent
004a5711dc
commit
d0f073686a
6 files changed
+136
-21
No files matched your search
@@ -21,7 +21,7 @@ steps:
|
||||
- omni_minicpmo_4_5_function
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py
|
||||
|
||||
- label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
|
||||
source_file_dependencies:
|
||||
@@ -33,9 +33,10 @@ steps:
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
source_file_dependencies: omni_minicpmo_4_5_duplex_function
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model"
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with L4 · 4-GPU"
|
||||
source_file_dependencies: omni_qwen2_5_omni_doc
|
||||
@@ -229,17 +230,6 @@ steps:
|
||||
-m "full_model and diffusion and H100 and B200 and cards_1"
|
||||
--run-level "full_model"
|
||||
|
||||
- label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_pi05_expansion.py
|
||||
-m "full_model and diffusion and H100 and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
|
||||
source_file_dependencies: diffusion_qwen_image_function
|
||||
timeout_in_minutes: 120
|
||||
|
||||
@@ -152,15 +152,122 @@ steps:
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: "TTS · L4"
|
||||
timeout_in_minutes: 120
|
||||
- label: "TTS · Moss · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
|
||||
for f in \
|
||||
tests/e2e/offline_inference/test_moss_tts_expansion.py \
|
||||
tests/e2e/offline_inference/test_moss_tts_nano_expansion.py \
|
||||
tests/e2e/offline_inference/test_moss_tts_realtime.py \
|
||||
tests/e2e/offline_inference/test_moss_tts_realtime_expansion.py \
|
||||
tests/e2e/online_serving/test_moss_tts_expansion.py \
|
||||
tests/e2e/online_serving/test_moss_tts_nano_expansion.py
|
||||
do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · VoxCPM2 · Voxtral · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in \
|
||||
tests/e2e/offline_inference/test_voxcpm2_tts.py \
|
||||
tests/e2e/online_serving/test_voxcpm2_tts.py \
|
||||
tests/e2e/online_serving/test_voxcpm2_tts_expansion.py \
|
||||
tests/e2e/online_serving/test_voxcpm2_tts_lora.py \
|
||||
tests/e2e/offline_inference/test_voxtral_tts_expansion.py \
|
||||
tests/e2e/online_serving/test_voxtral_tts_expansion.py
|
||||
do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Higgs · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in \
|
||||
tests/e2e/offline_inference/test_higgs_audio_v2_expansion.py \
|
||||
tests/e2e/online_serving/test_higgs_audio_v2_expansion.py \
|
||||
tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · OmniVoice · Step-Audio · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in \
|
||||
tests/e2e/offline_inference/test_omnivoice_expansion.py \
|
||||
tests/e2e/online_serving/test_omnivoice_expansion.py \
|
||||
tests/e2e/online_serving/test_omnivoice_parity.py \
|
||||
tests/e2e/offline_inference/test_step_audio2_expansion.py
|
||||
do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Ming · GLM · IndexTTS · Breeze · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in \
|
||||
tests/e2e/offline_inference/test_ming_tts_expansion.py \
|
||||
tests/e2e/online_serving/test_ming_tts_expansion.py \
|
||||
tests/e2e/offline_inference/test_glm_tts_expansion.py \
|
||||
tests/e2e/online_serving/test_glm_tts_expansion.py \
|
||||
tests/e2e/offline_inference/test_indextts2_expansion.py \
|
||||
tests/e2e/online_serving/test_indextts2_expansion.py \
|
||||
tests/e2e/online_serving/test_breeze_tts_2.py
|
||||
do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Audex · CosyVoice · Qwen3-TTS · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in \
|
||||
tests/e2e/offline_inference/test_audex_tts.py \
|
||||
tests/e2e/offline_inference/test_audex_s2t.py \
|
||||
tests/e2e/offline_inference/test_audex_tta.py \
|
||||
tests/e2e/offline_inference/test_cosyvoice3_expansion.py \
|
||||
tests/e2e/online_serving/test_qwen3_tts_word_timestamps.py
|
||||
do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
@@ -224,6 +331,17 @@ steps:
|
||||
|
||||
# Split the old shared H100 Single-GPU sweep: one pytest process collected
|
||||
# every slow+cards_1 diffusion file and exceeded 150 minutes.
|
||||
- label: ":robot_face: Robot Policy OpenPI · π0.5 · H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pip install --no-deps "openpi-client @ git+https://github.com/Physical-Intelligence/openpi.git@215abfb217dbac7d5f1273282331b9b1866c0479#subdirectory=packages/openpi-client"
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_pi05_expansion.py
|
||||
-m "slow and diffusion and H100 and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · Qwen-Image · H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
|
||||
@@ -64,6 +64,11 @@ steps:
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
-m "npu and A3 and omni"
|
||||
--run-level full_model
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_window.py
|
||||
-m "full_model and npu and A3 and omni"
|
||||
--run-level full_model
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
source_file_dependencies: omni_minicpmo_4_5_accuracy
|
||||
timeout_in_minutes: 180
|
||||
|
||||
@@ -89,7 +89,6 @@ _DEPLOY_CONFIG: dict[str, Any] = {
|
||||
"trust_remote_code": True,
|
||||
"devices": "0,1,2,3", # set dynamically by _write_deploy_config
|
||||
"vae_use_slicing": False,
|
||||
"moe_backend": "flashinfer_cutlass",
|
||||
"vae_use_tiling": False,
|
||||
"parallel_config": {
|
||||
"pipeline_parallel_size": 1,
|
||||
|
||||
@@ -16,7 +16,7 @@ from tests.e2e.online_serving.helpers.minicpmo_4_5_duplex import (
|
||||
from tests.e2e.online_serving.helpers.minicpmo_window_e2e import run_window_turn
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.omni, pytest.mark.advanced_model]
|
||||
pytestmark = [pytest.mark.omni]
|
||||
|
||||
|
||||
def _assert_complete(result, *, require_audio=False):
|
||||
@@ -30,6 +30,7 @@ def _assert_complete(result, *, require_audio=False):
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,reference", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_rebuild_and_next_session(omni_server, mode, reference):
|
||||
@@ -47,7 +48,8 @@ def test_window_rebuild_and_next_session(omni_server, mode, reference):
|
||||
_assert_complete(result, require_audio=reference)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode,camera", [("basic", False), ("context", False), ("context", True)])
|
||||
def test_window_continuous_input(omni_server, tmp_path, mode, camera):
|
||||
@@ -71,6 +73,7 @@ def test_window_continuous_input(omni_server, tmp_path, mode, camera):
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize("mode", ["off", "basic", "context"])
|
||||
def test_window_buffered_flush(omni_server, mode):
|
||||
|
||||
@@ -11,7 +11,7 @@ websocket (``/v1/realtime/robot/openpi``) — the same wire path a robot uses
|
||||
|
||||
The ``pi0_openpi_*`` helpers are protocol-level (handshake, send obs, receive
|
||||
chunk) and not π0-specific, so π0.5 reuses them; only the deploy config and the
|
||||
expected metadata differ. This file is selected explicitly by the nightly H100
|
||||
expected metadata differ. This file is selected explicitly by the weekly H100
|
||||
robot-policy job.
|
||||
|
||||
The in-process LeRobot parity oracle lives separately in
|
||||
@@ -59,7 +59,7 @@ test_params = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
|
||||
Reference in new issue
Block a user