[CI][ROCm] Add and stabilize HunyuanImage3 nightly coverage (#7934)

Signed-off-by: haic0 <149741444+haic0@users.noreply.github.com>
Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
Signed-off-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: akshatvishu <akshatnayak197@gmail.com>
Co-authored-by: andyluo7 <andy.luo@amd.com>

.buildkite/amd/test-amd-nightly.yml

@@ -132,3 +132,25 @@ steps:
             artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-cosmos3-nightly"
             mkdir -p "$$artifact_dir"
             cp -v /tmp/cosmos3-pytest.xml "$$artifact_dir/"
+
+  - group: ":full_moon: HunyuanImage3 MI300 Nightly"
+    steps:
+      - label: "HunyuanImage3 Offline Pixel Accuracy"
+        agent_pool: mi300_4
+        mirror_hardwares: [amdproduction]
+        grade: NonBlocking
+        timeout_in_minutes: 180
+        env:
+          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
+          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        artifact_paths:
+          - artifacts/rocm-hunyuanimage3/**/*
+        commands:
+          - pytest --collect-only -q tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py::test_hunyuan_image3_pixel_accuracy_offline -m "full_model and rocm and MI325 and cards_4" --run-level full_model --junitxml=/tmp/hunyuanimage3-pytest.xml
+          - |
+            artifact_dir="$${BUILDKITE_BUILD_CHECKOUT_PATH:?}/artifacts/rocm-hunyuanimage3"
+            mkdir -p "$$artifact_dir"
+            cp -v tests/e2e/accuracy/artifacts/HunyuanImage-3_0-Instruct/*.png "$$artifact_dir/"
+            cp -v /tmp/hunyuanimage3-pytest.xml "$$artifact_dir/"

tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py

@@ -42,6 +42,7 @@ WIDTH = 1024
 PROMPT = "A brown and white dog is running on the grass."
 MEAN_THRESHOLD = 3e-2
 P99_THRESHOLD = 3e-1
+PSNR_THRESHOLD_ROCM = 29.0
 PSNR_THRESHOLD_NPU = 26.0
 
 # Per-device SSIM/PSNR for online/offline vs baseline. Unlisted devices use ``default``.
@@ -56,6 +57,8 @@ def _psnr_threshold(thresholds: SimilarityThresholds | None = None) -> float:
 
     if current_omni_platform.is_npu():
         return PSNR_THRESHOLD_NPU
+    if current_omni_platform.is_rocm():
+        return PSNR_THRESHOLD_ROCM
     if thresholds is None:
         thresholds = resolve_similarity_thresholds(SIMILARITY_THRESHOLDS_BY_DEVICE)
     return thresholds.psnr
@@ -111,6 +114,14 @@ _DEPLOY_CONFIG: dict[str, Any] = {
         },
     ],
     "platforms": {
+        "rocm": {
+            "stages": [
+                {
+                    "stage_id": 0,
+                    "moe_backend": "auto",
+                },
+            ],
+        },
         "npu": {
             "stages": [
                 {
@@ -310,7 +321,7 @@ def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> N
 
 
 @pytest.mark.full_model
-@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=4)
+@hardware_test(res={"cuda": ["H100", "B200"], "rocm": "MI325", "npu": "A3"}, num_cards=4)
 def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
     model = _model_name()
     output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)

tests/platforms/test_rocm_groupnorm_patch.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch.nn as nn
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+_PATCH_PATH = Path(__file__).parents[2] / "vllm_omni" / "platforms" / "rocm" / "patch" / "worker" / "patch_groupnorm.py"
+_SPEC = importlib.util.spec_from_file_location("test_patch_groupnorm", _PATCH_PATH)
+assert _SPEC is not None and _SPEC.loader is not None
+patch_groupnorm = importlib.util.module_from_spec(_SPEC)
+sys.modules[_SPEC.name] = patch_groupnorm
+_SPEC.loader.exec_module(patch_groupnorm)
+patch_groupnorm._registry_mod.initialize_model = patch_groupnorm._original_initialize_model
+
+
+def test_hunyuan_image3_keeps_pytorch_groupnorm(monkeypatch):
+    model = types.SimpleNamespace(vae=nn.Sequential(nn.GroupNorm(4, 8)))
+    monkeypatch.setattr(patch_groupnorm, "_original_initialize_model", lambda _: model)
+    monkeypatch.setattr(
+        patch_groupnorm,
+        "_replace_groupnorm_with_aiter",
+        lambda _: pytest.fail("Hunyuan Image 3.0 must keep PyTorch GroupNorm"),
+    )
+
+    od_config = types.SimpleNamespace(model_class_name="HunyuanImage3ForCausalMM")
+    assert patch_groupnorm._patched_initialize_model(od_config) is model
+    assert type(model.vae[0]) is nn.GroupNorm
+
+
+def test_other_models_keep_aiter_groupnorm_patch(monkeypatch):
+    model = types.SimpleNamespace(vae=nn.Sequential(nn.GroupNorm(4, 8)))
+    monkeypatch.setattr(patch_groupnorm, "_original_initialize_model", lambda _: model)
+
+    replaced = False
+
+    def replace_groupnorm(_vae):
+        nonlocal replaced
+        replaced = True
+        return True
+
+    monkeypatch.setattr(patch_groupnorm, "_replace_groupnorm_with_aiter", replace_groupnorm)
+    aiter_ops = types.ModuleType("vllm._aiter_ops")
+    setattr(aiter_ops, "is_aiter_found_and_supported", lambda: True)
+    monkeypatch.setitem(sys.modules, "vllm._aiter_ops", aiter_ops)
+
+    od_config = types.SimpleNamespace(model_class_name="OtherDiffusionModel")
+    assert patch_groupnorm._patched_initialize_model(od_config) is model
+    assert replaced

vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 """Patch ``initialize_model`` to replace VAE GroupNorm with AITER GroupNorm on ROCm."""
 
@@ -42,6 +43,13 @@ def _patched_initialize_model(od_config):
     model = _original_initialize_model(od_config)
 
     if hasattr(model, "vae"):
+        # AITER GroupNorm does not preserve the PyTorch autocast behavior required
+        # by Hunyuan Image 3.0. Keep PyTorch GroupNorm until the fix is released:
+        # https://github.com/ROCm/aiter/issues/4780
+        # https://github.com/ROCm/aiter/pull/4779
+        if od_config.model_class_name == "HunyuanImage3ForCausalMM":
+            return model
+
         try:
             from vllm._aiter_ops import is_aiter_found_and_supported