[CI/Build] Split diffusion distributed nightly tests by L4 and H100 (#5695)

Signed-off-by: wangyu <410167048@qq.com>

.buildkite/cuda/test-nightly.yml

@@ -457,14 +457,20 @@ steps:
   - group: ":card_index_dividers: Diffusion Test"
     key: nightly-diffusion-test-group
     depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     steps:
-      - label: ":full_moon: Diffusion · Distributed Test"
+      - label: ":full_moon: Diffusion · Distributed Test with L4"
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda' --run-level "full_model"
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
         mirror_hardwares: l4_4
 
+      - label: ":full_moon: Diffusion · Distributed Test with H100"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_4
+
       - label: ":full_moon: Diffusion · Offloader Test"
         timeout_in_minutes: 60
         commands:

tests/diffusion/distributed/test_vae_decode_parallelism.py

@@ -39,7 +39,7 @@ MODEL_CASES = [
             "needs_image": False,
             "is_moe": False,
             "mean_threshold": 3e-2,
-            "p99_threshold": 3e-2,
+            "p99_threshold": 4e-2,
         },
         id="wan22_t2v_a14b",
     ),
@@ -146,7 +146,7 @@ def _run_generate(
 @pytest.mark.full_model
 @pytest.mark.diffusion
 @pytest.mark.parallel
-@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
+@hardware_test(res={"cuda": "H100"}, num_cards={"cuda": 4})
 @pytest.mark.parametrize("model_case", MODEL_CASES)
 def test_vae_patch_parallel_tp2(model_case: dict[str, Any], tmp_path: Path):
     if current_omni_platform.is_npu():