[Test] add marker for stability and add Wan2.2 npu test case (#6343)

Signed-off-by: zhumingjue <zhumingjue@huawei.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>

pyproject.toml

@@ -268,6 +268,7 @@ markers = [
     # specified computation resources marks (auto-added)
     # Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
     "H100: [hardware-resource] Tests that require H100 GPU",
+    "H800: [hardware-resource] Tests that require H800 GPU",
     "H200: [hardware-resource] Tests that require H200 GPU",
     "L4: [hardware-resource] Tests that require L4 GPU",
     "B200: [hardware-resource] Tests that require B200 GPU",

tests/dfx/conftest.py

@@ -105,7 +105,7 @@ def create_unique_server_pytest_params(
     configs: list[dict[str, Any]],
     stage_configs_dir: Path,
 ) -> list[Any]:
-    """Like :func:`create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks."""
+    """Like :func:`_create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks."""
     marks_by_name = _marks_by_test_name(configs)
     return [
         pytest.param(
@@ -113,7 +113,7 @@ def create_unique_server_pytest_params(
             marks=marks_by_name.get(row[0], []),
             id=row[0],
         )
-        for row in create_unique_server_params(configs, stage_configs_dir)
+        for row in _create_unique_server_params(configs, stage_configs_dir)
     ]
 
 
@@ -150,7 +150,7 @@ def create_paired_omni_benchmark_pytest_params(
     """Paired params for ``run_benchmark.py`` (omni/tts)."""
     mapping = create_test_parameter_mapping(configs)
     marks_by_name = _marks_by_test_name(configs)
-    server_entries = [(row, row[0]) for row in create_unique_server_params(configs, stage_configs_dir)]
+    server_entries = [(row, row[0]) for row in _create_unique_server_params(configs, stage_configs_dir)]
     params_by_test_name = {
         test_name: get_benchmark_params_for_server(test_name, mapping) for _, test_name in server_entries
     }
@@ -232,7 +232,7 @@ def _build_serve_args(serve_args: Any) -> list[str]:
     return args
 
 
-def create_unique_server_params(
+def _create_unique_server_params(
     configs: list[dict[str, Any]],
     stage_configs_dir: Path,
 ) -> list[tuple[str, str, str | None, str | None, tuple[str, ...], bool]]:
@@ -311,7 +311,7 @@ def create_reliability_omni_server_params(
     configs: list[dict[str, Any]], stage_configs_dir: Path
 ) -> list[OmniServerParams]:
     adjusted_configs = configs_with_platform_stage_configs(configs)
-    unique_params = create_unique_server_params(adjusted_configs, stage_configs_dir)
+    unique_params = _create_unique_server_params(adjusted_configs, stage_configs_dir)
     server_args_by_name = extract_server_args_by_test_name(adjusted_configs)
     return [
         OmniServerParams(

tests/dfx/stability/conftest.py

@@ -37,7 +37,7 @@ def omni_server(request: pytest.FixtureRequest):
     with _omni_server_lock:
         # Same tuple and CLI composition as ``tests/dfx/perf/scripts/run_benchmark.py``;
         # ``serve_args`` from JSON are folded into ``extra_cli_args`` inside
-        # ``create_unique_server_params``.
+        # ``create_unique_server_pytest_params`` (JSON ``mark`` → pytest marks).
         test_name, model, deploy_path, stage_overrides, extra_cli_args, use_omni = request.param
 
         print(f"Starting OmniServer with test: {test_name}, model: {model}")

tests/dfx/stability/scripts/test_stability_hunyuan_image.py

@@ -13,7 +13,7 @@ import pytest
 from tests.dfx.conftest import (
     create_benchmark_indices,
     create_test_parameter_mapping,
-    create_unique_server_params,
+    create_unique_server_pytest_params,
     load_configs,
 )
 from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop
@@ -29,7 +29,7 @@ try:
 except FileNotFoundError:
     BENCHMARK_CONFIGS = []
 
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 

tests/dfx/stability/scripts/test_stability_qwen3_omni.py

@@ -13,7 +13,7 @@ import pytest
 from tests.dfx.conftest import (
     create_benchmark_indices,
     create_test_parameter_mapping,
-    create_unique_server_params,
+    create_unique_server_pytest_params,
     load_configs,
 )
 from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop
@@ -29,7 +29,7 @@ try:
 except FileNotFoundError:
     BENCHMARK_CONFIGS = []
 
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 

tests/dfx/stability/scripts/test_stability_qwen3_tts.py

@@ -13,7 +13,7 @@ import pytest
 from tests.dfx.conftest import (
     create_benchmark_indices,
     create_test_parameter_mapping,
-    create_unique_server_params,
+    create_unique_server_pytest_params,
     load_configs,
 )
 from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop
@@ -29,7 +29,7 @@ try:
 except FileNotFoundError:
     BENCHMARK_CONFIGS = []
 
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 

tests/dfx/stability/scripts/test_stability_qwen_image.py

@@ -13,7 +13,7 @@ import pytest
 from tests.dfx.conftest import (
     create_benchmark_indices,
     create_test_parameter_mapping,
-    create_unique_server_params,
+    create_unique_server_pytest_params,
     load_configs,
 )
 from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop
@@ -29,7 +29,7 @@ try:
 except FileNotFoundError:
     BENCHMARK_CONFIGS = []
 
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 

tests/dfx/stability/scripts/test_stability_wan22.py

@@ -13,7 +13,7 @@ import pytest
 from tests.dfx.conftest import (
     create_benchmark_indices,
     create_test_parameter_mapping,
-    create_unique_server_params,
+    create_unique_server_pytest_params,
     load_configs,
 )
 from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop
@@ -29,7 +29,7 @@ try:
 except FileNotFoundError:
     BENCHMARK_CONFIGS = []
 
-test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
 server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 

tests/dfx/stability/tests/test_hunyuan_image.json

@@ -1,6 +1,18 @@
 [
     {
         "test_name": "test_hunyuan_image_stability_tp2_fp8_sp2",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800"
+                    },
+                    "num_cards": 2
+                }
+            },
+            "local_model",
+            "diffusion"
+        ],
         "server_params": {
             "model": "tencent/HunyuanImage-3.0-Instruct",
             "serve_args": {

tests/dfx/stability/tests/test_qwen3_omni.json

@@ -1,6 +1,18 @@
 [
     {
         "test_name": "test_qwen3_omni_stability_async_chunk",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800"
+                    },
+                    "num_cards": 2
+                }
+            },
+            "local_model",
+            "omni"
+        ],
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
             "extra_cli_args": ["--async-chunk"]

tests/dfx/stability/tests/test_qwen3_tts.json

@@ -1,6 +1,19 @@
 [
     {
         "test_name": "test_qwen3_tts_stability",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800",
+                        "npu": "A3"
+                    },
+                    "num_cards": 2
+                }
+            },
+            "local_model",
+            "tts"
+        ],
         "server_params": {
             "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
         },

tests/dfx/stability/tests/test_qwen_image.json

@@ -1,6 +1,18 @@
 [
     {
         "test_name": "test_qwen_image_stability",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800"
+                    },
+                    "num_cards": 1
+                }
+            },
+            "local_model",
+            "diffusion"
+        ],
         "server_params": {
             "model": "Qwen/Qwen-Image"
         },

tests/dfx/stability/tests/test_voxcpm2.json

@@ -1,35 +0,0 @@
-[
-    {
-        "test_name": "test_voxcpm2_stability",
-        "server_params": {
-            "model": "openbmb/VoxCPM2",
-            "trust_remote_code": true
-        },
-        "benchmark_params": [
-            {
-                "dataset_name": "random",
-                "backend": "openai-audio-speech",
-                "endpoint": "/v1/audio/speech",
-                "trust_remote_code": true,
-                "duration_sec": 86400,
-                "max_concurrency": 2,
-                "num_prompts_per_batch": 50,
-                "random_input_len": {
-                    "min": 0,
-                    "max": 1000
-                },
-                "random_output_len": {
-                    "min": 0,
-                    "max": 1000
-                },
-                "random_range_ratio": 0.0,
-                "extra_body": {
-                    "voice": "default",
-                    "language": "English"
-                },
-                "ignore_eos": true,
-                "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration"
-            }
-        ]
-    }
-]

tests/dfx/stability/tests/test_wan22.json

@@ -1,6 +1,19 @@
 [
     {
-        "test_name": "test_wan22_i2v_stability_v1_videos",
+        "test_name": "test_wan22_i2v_stability_ulysses2_vae_patch2_hsdp_slicing_tiling",
+        "description": "Ulysses=2 + VAE patch parallel=2 + HSDP + VAE slicing + tiling",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "cuda": "H800"
+                    },
+                    "num_cards": 2
+                }
+            },
+            "local_model",
+            "diffusion"
+        ],
         "server_params": {
             "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
             "serve_args": {
@@ -27,5 +40,45 @@
                 ]
             }
         ]
+    },
+    {
+        "test_name": "test_wan22_i2v_stability_usp2_vae_patch2_hsdp_slicing",
+        "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing",
+        "mark": [
+            {
+                "hardware_marks": {
+                    "res": {
+                        "npu": "A3"
+                    },
+                    "num_cards": 2
+                }
+            },
+            "local_model",
+            "diffusion"
+        ],
+        "server_params": {
+            "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+            "serve_args": {
+                "usp": 2,
+                "vae-patch-parallel-size": 2,
+                "use-hsdp": true,
+                "vae-use-slicing": true
+            }
+        },
+        "benchmark_params": [
+            {
+                "dataset": "random",
+                "task": "i2v",
+                "backend": "v1/videos",
+                "duration_sec": 86400,
+                "max_concurrency": 1,
+                "num_prompts_per_batch": 50,
+                "enable_negative_prompt": true,
+                "random_request_config": [
+                    {"width": 832, "height": 480, "num_inference_steps": 4, "num_frames": 81, "fps": 16, "weight": 0.5},
+                    {"width": 1280, "height": 720, "num_inference_steps": 4, "num_frames": 121, "fps": 16, "weight": 0.5}
+                ]
+            }
+        ]
     }
 ]

tests/helpers/mark.py

@@ -73,8 +73,10 @@ def cuda_marks(*, res: str, num_cards: int):
         test_resource = pytest.mark.L4
     elif res == "H100":
         test_resource = pytest.mark.H100
+    elif res == "H800":
+        test_resource = pytest.mark.H800
     else:
-        raise ValueError(f"Invalid CUDA resource type: {res}. Supported: L4, H100")
+        raise ValueError(f"Invalid CUDA resource type: {res}. Supported: L4, H100, H800")
     marks = [test_resource, test_platform_detail]
     if num_cards == 1:
         return marks
@@ -137,7 +139,7 @@ def musa_marks(*, res: str, num_cards: int):
 
 def gpu_marks(*, res: str, num_cards: int):
     test_platform = pytest.mark.gpu
-    if res in ("L4", "H100"):
+    if res in ("L4", "H100", "H800"):
         return [test_platform] + cuda_marks(res=res, num_cards=num_cards)
     if res == "MI325":
         return [test_platform] + rocm_marks(res=res, num_cards=num_cards)
@@ -145,7 +147,7 @@ def gpu_marks(*, res: str, num_cards: int):
         return [test_platform] + xpu_marks(res=res, num_cards=num_cards)
     if res == "S5000":
         return [test_platform] + musa_marks(res=res, num_cards=num_cards)
-    raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325, B60, S5000")
+    raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, H800, MI325, B60, S5000")
 
 
 def npu_marks(*, res: str, num_cards: int):

tools/nightly/run_nightly_jobs.sh

@@ -20,6 +20,8 @@
 #   acc      — label contains "Accuracy Test" (incl. GEBench / GEdit-Bench style)
 #   function — label has neither "Perf Test" nor "Accuracy Test" (incl. Doc, Multi-Replica, etc.)
 #   stability— fixed dfx stability scripts under tests/dfx/stability/scripts/ (see below)
+#              Uses --pytest-mark / PYTEST_MARK for pytest -m (hardware via JSON marks).
+#              When unset, -m defaults to H800 (CUDA stability fleet; use --pytest-mark A3 on NPU).
 #   local    — pytest -sv -m "<MODEL_TYPE markers> and local_model" from repo root (no YAML step extract)
 #              When LABEL_SUBSTR is set, also runs matching tests/**/test_*.py and perf JSON configs under
 #              tests/dfx/perf/tests/*.json via run_benchmark.py / run_diffusion_benchmark.py.
@@ -42,7 +44,9 @@
 #                   whose filename contains the substring (benchmark runner chosen by JSON family).
 #
 #   stability (when included in TEST_TYPE):
-#     From repo root: pytest -s -v --run-level full_model tests/dfx/stability/scripts/test_stability_*.py
+#     From repo root: pytest -s -v --run-level full_model -m "<mark>" tests/dfx/stability/scripts/...
+#     -m comes from --pytest-mark / PYTEST_MARK when set (e.g. H800 or A3).
+#     Otherwise -m defaults to H800. PYTEST_MARK is rejected unless stability is enabled.
 #     model_type: omni → qwen3_omni; tts → qwen3_tts; diffusion → qwen_image + wan22 + hunyuan_image; all → all five
 #     LABEL_SUBSTR: if set, script path / job key / filename must contain it
 #
@@ -73,6 +77,7 @@
 #   MODEL_TYPE    - comma-separated and/or repeated flags (default: all); see above
 #   LABEL_SUBSTR  - YAML mode: substring of Buildkite step label; stability: substring of path/key/filename;
 #                   local: substring of test_*.py basename or tests/dfx/perf/tests/*.json filename
+#   PYTEST_MARK   - pytest -m expression for stability jobs only (also --pytest-mark)
 #   DRY_RUN=1     - print extracted commands only; do not write scripts or run pytest
 #   RUN_JOB_TIMEOUT_KILL_AFTER - seconds after SIGTERM before SIGKILL on inline pytest
 #                              timeout (default: 60; baked into generated job scripts)
@@ -82,6 +87,7 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 LABEL_SUBSTR="${LABEL_SUBSTR:-}"
+PYTEST_MARK="${PYTEST_MARK:-}"
 TEST_TYPE_ENV="${TEST_TYPE:-all}"
 MODEL_TYPE_ENV="${MODEL_TYPE:-all}"
 TEST_TYPE_CLI_PARTS=()
@@ -204,6 +210,10 @@ while [[ $# -gt 0 ]]; do
       LABEL_SUBSTR="$2"
       shift 2
       ;;
+    --pytest-mark)
+      PYTEST_MARK="$2"
+      shift 2
+      ;;
     --model-type)
       MODEL_TYPE_FROM_CLI=1
       _split_append_csv_array MODEL_TYPE_CLI_PARTS "$2"
@@ -339,7 +349,7 @@ fi
 
 mkdir -p "${LOG_DIR}/jobs"
 LOG_DIR="$(cd "${LOG_DIR}" && pwd)"
-export REPO_ROOT LOG_DIR YML LABEL_SUBSTR TEST_TYPE MODEL_TYPE DRY_RUN
+export REPO_ROOT LOG_DIR YML LABEL_SUBSTR TEST_TYPE MODEL_TYPE DRY_RUN PYTEST_MARK
 
 # Drop stale wrapper scripts, old tee logs, and perf manifest so a filtered run does not
 # mix with the previous outputs.
@@ -600,6 +610,26 @@ def _stability_model_matches(model_types: list[str], families: tuple[str, ...])
     return bool(set(model_types) & set(families))
 
 
+STABILITY_DEFAULT_PYTEST_MARK = "H800"
+
+
+def _shell_quote_pytest_mark(marker_expr: str) -> str:
+    """Shell-safe pytest ``-m`` value for generated bash job scripts."""
+    expr = (marker_expr or "").strip()
+    if not expr:
+        raise ValueError("pytest -m expression must not be empty")
+    return shlex.quote(expr)
+
+
+def _stability_pytest_mark(model_types: list[str]) -> str:
+    """Return explicit --pytest-mark or the default CUDA stability hardware marker."""
+    del model_types  # MODEL_TYPE filters scripts; default -m is hardware-specific.
+    override = (os.environ.get("PYTEST_MARK") or "").strip()
+    if override:
+        return override
+    return STABILITY_DEFAULT_PYTEST_MARK
+
+
 def _write_job_script(key: str, script_lines: list[str], jobs_dir: Path) -> None:
     body = "\n".join(script_lines) + "\n"
     if DRY_RUN:
@@ -712,7 +742,7 @@ def run_local_mode(
 
         if rel_paths:
             paths_arg = " ".join(shlex.quote(p) for p in rel_paths)
-            pytest_line = f'pytest -sv -m "{marker_expr}" {paths_arg}'
+            pytest_line = f"pytest -sv -m {_shell_quote_pytest_mark(marker_expr)} {paths_arg}"
             slug = _local_job_slug(LABEL_SUBSTR)
             key = f"local_pytest_{slug}"
             header = (
@@ -732,7 +762,7 @@ def run_local_mode(
         for json_rel, runner_rel in perf_pairs:
             json_name = Path(json_rel).name
             pytest_line = (
-                f'pytest -s -v -m "{marker_expr}" '
+                f"pytest -s -v -m {_shell_quote_pytest_mark(marker_expr)} "
                 f"{shlex.quote(runner_rel)} "
                 f"--test-config-file {shlex.quote(json_rel)}"
             )
@@ -763,7 +793,7 @@ def run_local_mode(
             )
         return matched
 
-    pytest_line = f'pytest -sv -m "{marker_expr}"'
+    pytest_line = f"pytest -sv -m {_shell_quote_pytest_mark(marker_expr)}"
     key = "local_pytest"
     header = f"# Local marker tests — MODEL_TYPE={','.join(model_types)}"
     script_lines = [
@@ -779,6 +809,8 @@ def run_local_mode(
 
 def run_stability_mode(jobs_dir: Path, model_types: list[str]) -> int:
     matched = 0
+    marker_expr = _stability_pytest_mark(model_types)
+    mark_from_cli = bool((os.environ.get("PYTEST_MARK") or "").strip())
     for key, rel_posix, families in STABILITY_CASES:
         if not _stability_model_matches(model_types, families):
             continue
@@ -794,11 +826,17 @@ def run_stability_mode(jobs_dir: Path, model_types: list[str]) -> int:
         if not rel_path.is_file():
             print(f"# skip (missing file): {rel_path}", file=sys.stderr)
             continue
-        pytest_line = f"pytest -s -v --run-level full_model {rel_posix}"
+        pytest_line = (
+            f"pytest -s -v --run-level full_model "
+            f"-m {_shell_quote_pytest_mark(marker_expr)} "
+            f"{shlex.quote(rel_posix)}"
+        )
         matched += 1
         script_lines = [
             "#!/usr/bin/env bash",
             f"# Stability: {key} — {rel_posix}",
+            f"# pytest -m {marker_expr!r}"
+            + (" (--pytest-mark)" if mark_from_cli else f" (default {STABILITY_DEFAULT_PYTEST_MARK})"),
             "set -euo pipefail",
             f'cd "{REPO_ROOT}"',
             pytest_line,
@@ -816,6 +854,14 @@ def main() -> None:
     yaml_test_types = [t for t in test_types if t not in ("stability", "local")]
     needs_yaml = bool(yaml_test_types)
 
+    pytest_mark = (os.environ.get("PYTEST_MARK") or "").strip()
+    if pytest_mark and not want_stability:
+        print(
+            "PYTEST_MARK / --pytest-mark is only supported when --test-type includes stability",
+            file=sys.stderr,
+        )
+        sys.exit(2)
+
     jobs_dir = LOG_DIR / "jobs"
     if not DRY_RUN:
         jobs_dir.mkdir(parents=True, exist_ok=True)