[CI] Expand E2E source_file_dependencies for shared model code (#5994)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
This commit is contained in:
1 parent
b95c73742d
commit
d671e6b0db
14 files changed
+135
-228
No files matched your search
@@ -39,8 +39,6 @@ def _render(changed_files: list[str]) -> str:
|
||||
|
||||
|
||||
def test_bootstrap_if_injected_by_step_key() -> None:
|
||||
import yaml
|
||||
|
||||
rendered = _render_bootstrap_pipeline(
|
||||
BOOTSTRAP_STEPS_TEMPLATE,
|
||||
decision=resolve_ci_decision([]),
|
||||
@@ -142,12 +140,7 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None:
|
||||
]
|
||||
|
||||
|
||||
COVERAGE_PILOT_LABELS = ("Diffusion · Bagel Test", "TTS · Qwen3-TTS Base Test")
|
||||
MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml")
|
||||
|
||||
|
||||
def _surviving_labels(changed_files: list[str]) -> set[str]:
|
||||
doc = yaml.safe_load(MERGE_PIPELINE.read_text(encoding="utf-8"))
|
||||
def _surviving_labels(doc: dict, changed_files: list[str]) -> set[str]:
|
||||
rendered = _render_test_pipeline(doc, changed_files=changed_files)
|
||||
labels: set[str] = set()
|
||||
|
||||
@@ -163,6 +156,29 @@ def _surviving_labels(changed_files: list[str]) -> set[str]:
|
||||
return labels
|
||||
|
||||
|
||||
# Synthetic coverage-style job: shared inputs that change what the split measures.
|
||||
_COVERAGE_SHARED_INPUTS_DOC = {
|
||||
"steps": [
|
||||
{
|
||||
"label": "Coverage Pilot",
|
||||
"source_file_dependencies": [
|
||||
"tests/e2e/online_serving/test_example.py",
|
||||
".buildkite/common/scripts/run_cov_split.sh",
|
||||
"pyproject.toml",
|
||||
],
|
||||
"commands": [".buildkite/common/scripts/run_cov_split.sh --model-id example"],
|
||||
},
|
||||
{
|
||||
"label": "Unrelated Model Test",
|
||||
"source_file_dependencies": [
|
||||
"tests/e2e/online_serving/test_other.py",
|
||||
],
|
||||
"commands": ["pytest -sv tests/e2e/online_serving/test_other.py"],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changed_file",
|
||||
[
|
||||
@@ -170,18 +186,17 @@ def _surviving_labels(changed_files: list[str]) -> set[str]:
|
||||
"pyproject.toml",
|
||||
],
|
||||
)
|
||||
def test_coverage_pilots_selected_by_their_shared_inputs(changed_file: str) -> None:
|
||||
"""The split helper and the coverage config both change what these jobs measure.
|
||||
|
||||
Without them in source_file_dependencies the jobs are filtered out of normal
|
||||
PR builds, so a regression would only surface in a later nightly.
|
||||
"""
|
||||
labels = _surviving_labels([changed_file])
|
||||
for pilot in COVERAGE_PILOT_LABELS:
|
||||
assert pilot in labels, f"{pilot} dropped when {changed_file} changed"
|
||||
def test_coverage_shared_inputs_select_dependent_job(changed_file: str) -> None:
|
||||
"""Jobs that list coverage shared inputs must stay selected when those files change."""
|
||||
labels = _surviving_labels(_COVERAGE_SHARED_INPUTS_DOC, [changed_file])
|
||||
assert "Coverage Pilot" in labels
|
||||
assert "Unrelated Model Test" not in labels
|
||||
|
||||
|
||||
def test_coverage_pilots_not_selected_by_unrelated_change() -> None:
|
||||
labels = _surviving_labels(["vllm_omni/entrypoints/openai/serving_chat.py"])
|
||||
for pilot in COVERAGE_PILOT_LABELS:
|
||||
assert pilot not in labels
|
||||
def test_coverage_shared_inputs_ignored_for_unrelated_change() -> None:
|
||||
labels = _surviving_labels(
|
||||
_COVERAGE_SHARED_INPUTS_DOC,
|
||||
["vllm_omni/entrypoints/openai/serving_chat.py"],
|
||||
)
|
||||
assert "Coverage Pilot" not in labels
|
||||
assert "Unrelated Model Test" not in labels
|
||||
@@ -46,7 +46,7 @@ REFERENCE_TEXT_IMG2TEXT = (
|
||||
_OMNI_RUNNER_PARAM = (MODEL_NAME, STAGE_CONFIG)
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.advanced_model,
|
||||
pytest.mark.slow,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
|
||||
@@ -72,7 +72,7 @@ def _send_inpaint_with_generator(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandler):
|
||||
@@ -93,7 +93,7 @@ def test_flux2_klein_inpaint_deterministic(omni_runner_handler: OmniRunnerHandle
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_flux2_klein_inpaint_different_seeds_different_output(omni_runner_handler: OmniRunnerHandler):
|
||||
|
||||
@@ -39,8 +39,7 @@ def _extract_audio(multimodal_output: dict) -> torch.Tensor:
|
||||
return audio
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None:
|
||||
@@ -53,8 +52,7 @@ def test_voxcpm2_zero_shot_001(omni_runner: OmniRunner) -> None:
|
||||
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None:
|
||||
@@ -95,8 +93,7 @@ def test_voxcpm2_voice_clone_002(omni_runner: OmniRunner) -> None:
|
||||
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm2_prefill_decode_mixed_batch_003(omni_runner: OmniRunner) -> None:
|
||||
|
||||
@@ -11,7 +11,7 @@ MODEL = "Tongyi-MAI/Z-Image-Turbo"
|
||||
PROMPT = "A high-detail studio photo of an orange tabby cat sitting on a laptop keyboard."
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 1, "rocm": 1, "xpu": 2})
|
||||
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
|
||||
|
||||
@@ -75,8 +75,7 @@ def _build_img2img_messages(prompt: str, image_b64: str) -> list[dict]:
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
@@ -98,8 +97,7 @@ def test_bagel_text2img_online(omni_server, openai_client) -> None:
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
|
||||
@@ -59,8 +59,7 @@ _REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know w
|
||||
class TestHiggsAudioV3OnlineHappyPath:
|
||||
"""Plain-text -> audio happy paths against the live HTTP server."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_plain_text_wav(self, omni_server, openai_client) -> None:
|
||||
@@ -79,7 +78,7 @@ class TestHiggsAudioV3OnlineHappyPath:
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_plain_text_with_max_new_tokens(self, omni_server, openai_client) -> None:
|
||||
@@ -97,8 +96,7 @@ class TestHiggsAudioV3OnlineHappyPath:
|
||||
)
|
||||
|
||||
@pytest.mark.skip(reason="issue#4411")
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_concurrent_plain_text(self, omni_server, openai_client) -> None:
|
||||
@@ -115,7 +113,7 @@ class TestHiggsAudioV3OnlineHappyPath:
|
||||
)
|
||||
|
||||
@pytest.mark.skip(reason="issue#4411")
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_plain_text_pcm_streaming(self, omni_server, openai_client) -> None:
|
||||
@@ -147,7 +145,7 @@ class TestHiggsAudioV3OnlineHappyPath:
|
||||
)
|
||||
|
||||
@pytest.mark.skip(reason="issue#4411")
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_concurrent_pcm_streaming(self, omni_server, openai_client) -> None:
|
||||
@@ -189,7 +187,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||
They do not assert audio quality (out of scope for CI).
|
||||
"""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_inline_emotion_and_expressive(self, omni_server, openai_client) -> None:
|
||||
@@ -205,7 +203,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_inline_style_whispering(self, omni_server, openai_client) -> None:
|
||||
@@ -221,7 +219,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_inline_prosody_speed_and_pitch(self, omni_server, openai_client) -> None:
|
||||
@@ -239,7 +237,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_inline_pause_mid_text(self, omni_server, openai_client) -> None:
|
||||
@@ -255,7 +253,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_inline_sfx_with_onomatopoeia(self, omni_server, openai_client) -> None:
|
||||
@@ -279,8 +277,7 @@ class TestHiggsAudioV3OnlineInlineControlTokens:
|
||||
class TestHiggsAudioV3OnlineVoiceClone:
|
||||
"""Voice clone via the two payload shapes the model serves."""
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_ref_audio_ref_text(self, omni_server, openai_client) -> None:
|
||||
@@ -297,7 +294,7 @@ class TestHiggsAudioV3OnlineVoiceClone:
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voice_clone_references_alias(self, omni_server, openai_client) -> None:
|
||||
|
||||
@@ -6,7 +6,7 @@ Online serving tests for the Qwen-Image-Edit family (image-to-image via chat com
|
||||
|
||||
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
|
||||
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512.
|
||||
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input
|
||||
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``slow`` — mixed input
|
||||
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
|
||||
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
|
||||
|
||||
@@ -52,7 +52,7 @@ def _get_diffusion_feature_cases(model: str):
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
@@ -82,8 +82,7 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
@@ -114,7 +113,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
|
||||
@@ -44,8 +44,7 @@ def _get_diffusion_feature_cases(model: str):
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
|
||||
def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
|
||||
@@ -46,8 +46,7 @@ tts_server_params = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
@@ -72,7 +71,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_audio_speech_request(request_config, request_num=MAX_CONCURRENT)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.tts
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
|
||||
@@ -32,7 +32,7 @@ def _get_diffusion_feature_cases():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
@@ -60,7 +60,7 @@ def test_basic_001(
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
|
||||
Reference in new issue
Block a user