[CI/Build] Add experimental AMD MI300 nightly lane (#6978)

Signed-off-by: andyluo7 <andy.luo@amd.com>
This commit is contained in:
andyluo7 authored and GitHub committed 2026-09-14 12:02:28 +08:00
1 parent e8f45fcd89
commit 58adeec05f
10 files changed
+147 -30

No files matched your search

+1 -1
View File
@@ -136,7 +136,7 @@ def test_label_lookup_selects_tiers(tmp_path, labels, expected_levels):
] == expected_levels
@pytest.mark.parametrize("debug_test_yaml", ["ready,ready", "nightly", ", ,", " \t"])
@pytest.mark.parametrize("debug_test_yaml", ["ready,ready", "nightly,nightly", ", ,", " \t"])
def test_invalid_debug_override_fails_without_label_lookup(tmp_path, debug_test_yaml):
result = _run_bootstrap(tmp_path, debug_test_yaml=debug_test_yaml, curl_status=22)
+15 -2
View File
@@ -10,10 +10,11 @@ import yaml
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml")
AMD_NIGHTLY_PIPELINE = Path(".buildkite/amd/test-amd-nightly.yml")
def _find_step(label: str) -> dict:
pipeline = yaml.safe_load(AMD_MERGE_PIPELINE.read_text(encoding="utf-8"))
def _find_step(label: str, pipeline_path: Path = AMD_MERGE_PIPELINE) -> dict:
pipeline = yaml.safe_load(pipeline_path.read_text(encoding="utf-8"))
def walk(steps: list[dict]) -> dict | None:
for step in steps:
@@ -40,3 +41,15 @@ def test_qwen3_tts_base_preserves_advanced_model_arguments() -> None:
run_level_index = argv.index("--run-level")
assert argv[marker_index + 1] == "advanced_model and cuda"
assert argv[run_level_index + 1] == "advanced_model"
def test_qwen3_accuracy_defers_artifact_path_expansion() -> None:
step = _find_step("Qwen3-Omni Accuracy", AMD_NIGHTLY_PIPELINE)
staging_command = next(command for command in step["commands"] if "artifact_dir=" in command)
# Dynamic pipelines are interpolated once during upload. Double dollars
# preserve these variables for the GPU job's runtime shell.
assert '"$$PWD"' in staging_command
assert '"$${BUILDKITE_BUILD_CHECKOUT_PATH:?}"' in staging_command
assert '"$$artifact_dir"' in staging_command
assert step["artifact_paths"] == ["tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"]
+15 -4
View File
@@ -24,10 +24,12 @@ SPEC.loader.exec_module(SELECTOR)
("feature", ("ready",), ("ready",)),
("feature", ("merge-test",), ("merge",)),
("feature", ("merge-test", "ready"), ("ready", "merge")),
("feature", ("nightly-test",), ("nightly",)),
("feature", ("ready", "merge-test", "nightly-test"), ("ready", "merge", "nightly")),
("feature", ("amd-test",), ("ready",)),
("feature", ("amd-test", "merge-test"), ("merge",)),
("feature", ("amd-test", "ready", "merge-test"), ("ready", "merge")),
("feature", ("nightly-test",), ("ready",)),
("feature", ("amd-test", "nightly-test"), ("nightly",)),
("feature", ("not-ready", "merge-test-extra"), ("ready",)),
],
)
@@ -39,8 +41,17 @@ def test_debug_override_takes_precedence_and_normalizes_input():
assert SELECTOR.select_amd_test_suites(
branch="main",
labels=("ready",),
debug_test_yaml=" MERGE, ready,",
) == ("merge", "ready")
debug_test_yaml=" NIGHTLY, merge, ready,",
nightly=False,
) == ("nightly", "merge", "ready")
def test_scheduled_main_selects_only_nightly():
assert SELECTOR.select_amd_test_suites(
branch="main",
labels=(),
nightly=True,
) == ("nightly",)
def test_empty_debug_override_uses_normal_selection():
@@ -51,7 +62,7 @@ def test_empty_debug_override_uses_normal_selection():
) == ("merge",)
@pytest.mark.parametrize("value", ["ready,ready", "nightly", ", ,", " \t"])
@pytest.mark.parametrize("value", ["ready,ready", "nightly,nightly", "weekly", ", ,", " \t"])
def test_invalid_debug_override(value):
with pytest.raises(ValueError):
SELECTOR.select_amd_test_suites(
+8
View File
@@ -144,6 +144,14 @@ def test_npu_nightly_only_skips_all_l23() -> None:
assert not decision.is_run("npu", "l3")
def test_amd_nightly_only_skips_all_l23() -> None:
decision = _decision([".buildkite/amd/test-amd-nightly.yml"])
assert _is_yaml_gated(decision)
assert _all_l23_skipped(decision)
assert not decision.is_run("amd", "l2")
assert not decision.is_run("amd", "l3")
def test_nightly_and_weekly_skips_all_l23() -> None:
decision = _decision(
[