[CI/Build] Add experimental AMD MI300 nightly lane (#6978)
Signed-off-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
1 parent
e8f45fcd89
commit
58adeec05f
10 files changed
+147
-30
No files matched your search
@@ -136,7 +136,7 @@ def test_label_lookup_selects_tiers(tmp_path, labels, expected_levels):
|
||||
] == expected_levels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("debug_test_yaml", ["ready,ready", "nightly", ", ,", " \t"])
|
||||
@pytest.mark.parametrize("debug_test_yaml", ["ready,ready", "nightly,nightly", ", ,", " \t"])
|
||||
def test_invalid_debug_override_fails_without_label_lookup(tmp_path, debug_test_yaml):
|
||||
result = _run_bootstrap(tmp_path, debug_test_yaml=debug_test_yaml, curl_status=22)
|
||||
|
||||
|
||||
@@ -10,10 +10,11 @@ import yaml
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml")
|
||||
AMD_NIGHTLY_PIPELINE = Path(".buildkite/amd/test-amd-nightly.yml")
|
||||
|
||||
|
||||
def _find_step(label: str) -> dict:
|
||||
pipeline = yaml.safe_load(AMD_MERGE_PIPELINE.read_text(encoding="utf-8"))
|
||||
def _find_step(label: str, pipeline_path: Path = AMD_MERGE_PIPELINE) -> dict:
|
||||
pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8"))
|
||||
|
||||
def walk(steps: list[dict]) -> dict | None:
|
||||
for step in steps:
|
||||
@@ -40,3 +41,15 @@ def test_qwen3_tts_base_preserves_advanced_model_arguments() -> None:
|
||||
run_level_index = argv.index("--run-level")
|
||||
assert argv[marker_index + 1] == "advanced_model and cuda"
|
||||
assert argv[run_level_index + 1] == "advanced_model"
|
||||
|
||||
|
||||
def test_qwen3_accuracy_defers_artifact_path_expansion() -> None:
|
||||
step = _find_step("Qwen3-Omni Accuracy", AMD_NIGHTLY_PIPELINE)
|
||||
staging_command = next(command for command in step["commands"] if "artifact_dir=" in command)
|
||||
|
||||
# Dynamic pipelines are interpolated once during upload. Double dollars
|
||||
# preserve these variables for the GPU job's runtime shell.
|
||||
assert '"$$PWD"' in staging_command
|
||||
assert '"$${BUILDKITE_BUILD_CHECKOUT_PATH:?}"' in staging_command
|
||||
assert '"$$artifact_dir"' in staging_command
|
||||
assert step["artifact_paths"] == ["tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"]
|
||||
@@ -24,10 +24,12 @@ SPEC.loader.exec_module(SELECTOR)
|
||||
("feature", ("ready",), ("ready",)),
|
||||
("feature", ("merge-test",), ("merge",)),
|
||||
("feature", ("merge-test", "ready"), ("ready", "merge")),
|
||||
("feature", ("nightly-test",), ("nightly",)),
|
||||
("feature", ("ready", "merge-test", "nightly-test"), ("ready", "merge", "nightly")),
|
||||
("feature", ("amd-test",), ("ready",)),
|
||||
("feature", ("amd-test", "merge-test"), ("merge",)),
|
||||
("feature", ("amd-test", "ready", "merge-test"), ("ready", "merge")),
|
||||
("feature", ("nightly-test",), ("ready",)),
|
||||
("feature", ("amd-test", "nightly-test"), ("nightly",)),
|
||||
("feature", ("not-ready", "merge-test-extra"), ("ready",)),
|
||||
],
|
||||
)
|
||||
@@ -39,8 +41,17 @@ def test_debug_override_takes_precedence_and_normalizes_input():
|
||||
assert SELECTOR.select_amd_test_suites(
|
||||
branch="main",
|
||||
labels=("ready",),
|
||||
debug_test_yaml=" MERGE, ready,",
|
||||
) == ("merge", "ready")
|
||||
debug_test_yaml=" NIGHTLY, merge, ready,",
|
||||
nightly=False,
|
||||
) == ("nightly", "merge", "ready")
|
||||
|
||||
|
||||
def test_scheduled_main_selects_only_nightly():
|
||||
assert SELECTOR.select_amd_test_suites(
|
||||
branch="main",
|
||||
labels=(),
|
||||
nightly=True,
|
||||
) == ("nightly",)
|
||||
|
||||
|
||||
def test_empty_debug_override_uses_normal_selection():
|
||||
@@ -51,7 +62,7 @@ def test_empty_debug_override_uses_normal_selection():
|
||||
) == ("merge",)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["ready,ready", "nightly", ", ,", " \t"])
|
||||
@pytest.mark.parametrize("value", ["ready,ready", "nightly,nightly", "weekly", ", ,", " \t"])
|
||||
def test_invalid_debug_override(value):
|
||||
with pytest.raises(ValueError):
|
||||
SELECTOR.select_amd_test_suites(
|
||||
|
||||
@@ -144,6 +144,14 @@ def test_npu_nightly_only_skips_all_l23() -> None:
|
||||
assert not decision.is_run("npu", "l3")
|
||||
|
||||
|
||||
def test_amd_nightly_only_skips_all_l23() -> None:
|
||||
decision = _decision([".buildkite/amd/test-amd-nightly.yml"])
|
||||
assert _is_yaml_gated(decision)
|
||||
assert _all_l23_skipped(decision)
|
||||
assert not decision.is_run("amd", "l2")
|
||||
assert not decision.is_run("amd", "l3")
|
||||
|
||||
|
||||
def test_nightly_and_weekly_skips_all_l23() -> None:
|
||||
decision = _decision(
|
||||
[
|
||||
|
||||
Reference in new issue
Block a user