diff --git a/pyproject.toml b/pyproject.toml index 5135661ce..9b9493730 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -268,6 +268,7 @@ markers = [ # specified computation resources marks (auto-added) # Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list(). "H100: [hardware-resource] Tests that require H100 GPU", + "H800: [hardware-resource] Tests that require H800 GPU", "H200: [hardware-resource] Tests that require H200 GPU", "L4: [hardware-resource] Tests that require L4 GPU", "B200: [hardware-resource] Tests that require B200 GPU", diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 2e22da531..864fb7ab3 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -105,7 +105,7 @@ def create_unique_server_pytest_params( configs: list[dict[str, Any]], stage_configs_dir: Path, ) -> list[Any]: - """Like :func:`create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks.""" + """Like :func:`_create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks.""" marks_by_name = _marks_by_test_name(configs) return [ pytest.param( @@ -113,7 +113,7 @@ def create_unique_server_pytest_params( marks=marks_by_name.get(row[0], []), id=row[0], ) - for row in create_unique_server_params(configs, stage_configs_dir) + for row in _create_unique_server_params(configs, stage_configs_dir) ] @@ -150,7 +150,7 @@ def create_paired_omni_benchmark_pytest_params( """Paired params for ``run_benchmark.py`` (omni/tts).""" mapping = create_test_parameter_mapping(configs) marks_by_name = _marks_by_test_name(configs) - server_entries = [(row, row[0]) for row in create_unique_server_params(configs, stage_configs_dir)] + server_entries = [(row, row[0]) for row in _create_unique_server_params(configs, stage_configs_dir)] params_by_test_name = { test_name: get_benchmark_params_for_server(test_name, mapping) for _, test_name in server_entries } @@ -232,7 +232,7 @@ def _build_serve_args(serve_args: Any) -> list[str]: return args -def create_unique_server_params( +def _create_unique_server_params( configs: list[dict[str, Any]], stage_configs_dir: Path, ) -> list[tuple[str, str, str | None, str | None, tuple[str, ...], bool]]: @@ -311,7 +311,7 @@ def create_reliability_omni_server_params( configs: list[dict[str, Any]], stage_configs_dir: Path ) -> list[OmniServerParams]: adjusted_configs = configs_with_platform_stage_configs(configs) - unique_params = create_unique_server_params(adjusted_configs, stage_configs_dir) + unique_params = _create_unique_server_params(adjusted_configs, stage_configs_dir) server_args_by_name = extract_server_args_by_test_name(adjusted_configs) return [ OmniServerParams( diff --git a/tests/dfx/stability/conftest.py b/tests/dfx/stability/conftest.py index eaa755d45..0889db885 100644 --- a/tests/dfx/stability/conftest.py +++ b/tests/dfx/stability/conftest.py @@ -37,7 +37,7 @@ def omni_server(request: pytest.FixtureRequest): with _omni_server_lock: # Same tuple and CLI composition as ``tests/dfx/perf/scripts/run_benchmark.py``; # ``serve_args`` from JSON are folded into ``extra_cli_args`` inside - # ``create_unique_server_params``. + # ``create_unique_server_pytest_params`` (JSON ``mark`` → pytest marks). test_name, model, deploy_path, stage_overrides, extra_cli_args, use_omni = request.param print(f"Starting OmniServer with test: {test_name}, model: {model}") diff --git a/tests/dfx/stability/scripts/test_stability_hunyuan_image.py b/tests/dfx/stability/scripts/test_stability_hunyuan_image.py index 38858047f..be2aec805 100644 --- a/tests/dfx/stability/scripts/test_stability_hunyuan_image.py +++ b/tests/dfx/stability/scripts/test_stability_hunyuan_image.py @@ -13,7 +13,7 @@ import pytest from tests.dfx.conftest import ( create_benchmark_indices, create_test_parameter_mapping, - create_unique_server_params, + create_unique_server_pytest_params, load_configs, ) from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop @@ -29,7 +29,7 @@ try: except FileNotFoundError: BENCHMARK_CONFIGS = [] -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py b/tests/dfx/stability/scripts/test_stability_qwen3_omni.py index 5f487a27c..98ab5ee89 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py +++ b/tests/dfx/stability/scripts/test_stability_qwen3_omni.py @@ -13,7 +13,7 @@ import pytest from tests.dfx.conftest import ( create_benchmark_indices, create_test_parameter_mapping, - create_unique_server_params, + create_unique_server_pytest_params, load_configs, ) from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop @@ -29,7 +29,7 @@ try: except FileNotFoundError: BENCHMARK_CONFIGS = [] -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py b/tests/dfx/stability/scripts/test_stability_qwen3_tts.py index 0c8059ed9..293b723ca 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py +++ b/tests/dfx/stability/scripts/test_stability_qwen3_tts.py @@ -13,7 +13,7 @@ import pytest from tests.dfx.conftest import ( create_benchmark_indices, create_test_parameter_mapping, - create_unique_server_params, + create_unique_server_pytest_params, load_configs, ) from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop @@ -29,7 +29,7 @@ try: except FileNotFoundError: BENCHMARK_CONFIGS = [] -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) diff --git a/tests/dfx/stability/scripts/test_stability_qwen_image.py b/tests/dfx/stability/scripts/test_stability_qwen_image.py index 7ff39b5ca..62b0cd86c 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen_image.py +++ b/tests/dfx/stability/scripts/test_stability_qwen_image.py @@ -13,7 +13,7 @@ import pytest from tests.dfx.conftest import ( create_benchmark_indices, create_test_parameter_mapping, - create_unique_server_params, + create_unique_server_pytest_params, load_configs, ) from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop @@ -29,7 +29,7 @@ try: except FileNotFoundError: BENCHMARK_CONFIGS = [] -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) diff --git a/tests/dfx/stability/scripts/test_stability_wan22.py b/tests/dfx/stability/scripts/test_stability_wan22.py index a363133f5..aae0ae429 100644 --- a/tests/dfx/stability/scripts/test_stability_wan22.py +++ b/tests/dfx/stability/scripts/test_stability_wan22.py @@ -13,7 +13,7 @@ import pytest from tests.dfx.conftest import ( create_benchmark_indices, create_test_parameter_mapping, - create_unique_server_params, + create_unique_server_pytest_params, load_configs, ) from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop @@ -29,7 +29,7 @@ try: except FileNotFoundError: BENCHMARK_CONFIGS = [] -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) diff --git a/tests/dfx/stability/tests/test_hunyuan_image.json b/tests/dfx/stability/tests/test_hunyuan_image.json index b78fa812c..b468ceec3 100644 --- a/tests/dfx/stability/tests/test_hunyuan_image.json +++ b/tests/dfx/stability/tests/test_hunyuan_image.json @@ -1,6 +1,18 @@ [ { "test_name": "test_hunyuan_image_stability_tp2_fp8_sp2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 2 + } + }, + "local_model", + "diffusion" + ], "server_params": { "model": "tencent/HunyuanImage-3.0-Instruct", "serve_args": { diff --git a/tests/dfx/stability/tests/test_qwen3_omni.json b/tests/dfx/stability/tests/test_qwen3_omni.json index cbdf25e67..816e981b6 100644 --- a/tests/dfx/stability/tests/test_qwen3_omni.json +++ b/tests/dfx/stability/tests/test_qwen3_omni.json @@ -1,6 +1,18 @@ [ { "test_name": "test_qwen3_omni_stability_async_chunk", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 2 + } + }, + "local_model", + "omni" + ], "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "extra_cli_args": ["--async-chunk"] diff --git a/tests/dfx/stability/tests/test_qwen3_tts.json b/tests/dfx/stability/tests/test_qwen3_tts.json index 0208331bd..1c9356574 100644 --- a/tests/dfx/stability/tests/test_qwen3_tts.json +++ b/tests/dfx/stability/tests/test_qwen3_tts.json @@ -1,6 +1,19 @@ [ { "test_name": "test_qwen3_tts_stability", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800", + "npu": "A3" + }, + "num_cards": 2 + } + }, + "local_model", + "tts" + ], "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" }, diff --git a/tests/dfx/stability/tests/test_qwen_image.json b/tests/dfx/stability/tests/test_qwen_image.json index e28614973..0c21d0ac1 100644 --- a/tests/dfx/stability/tests/test_qwen_image.json +++ b/tests/dfx/stability/tests/test_qwen_image.json @@ -1,6 +1,18 @@ [ { "test_name": "test_qwen_image_stability", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 1 + } + }, + "local_model", + "diffusion" + ], "server_params": { "model": "Qwen/Qwen-Image" }, diff --git a/tests/dfx/stability/tests/test_voxcpm2.json b/tests/dfx/stability/tests/test_voxcpm2.json deleted file mode 100644 index dd912da36..000000000 --- a/tests/dfx/stability/tests/test_voxcpm2.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "test_name": "test_voxcpm2_stability", - "server_params": { - "model": "openbmb/VoxCPM2", - "trust_remote_code": true - }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-audio-speech", - "endpoint": "/v1/audio/speech", - "trust_remote_code": true, - "duration_sec": 86400, - "max_concurrency": 2, - "num_prompts_per_batch": 50, - "random_input_len": { - "min": 0, - "max": 1000 - }, - "random_output_len": { - "min": 0, - "max": 1000 - }, - "random_range_ratio": 0.0, - "extra_body": { - "voice": "default", - "language": "English" - }, - "ignore_eos": true, - "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration" - } - ] - } -] diff --git a/tests/dfx/stability/tests/test_wan22.json b/tests/dfx/stability/tests/test_wan22.json index 7b3b54b65..e6ab0b5cf 100644 --- a/tests/dfx/stability/tests/test_wan22.json +++ b/tests/dfx/stability/tests/test_wan22.json @@ -1,6 +1,19 @@ [ { - "test_name": "test_wan22_i2v_stability_v1_videos", + "test_name": "test_wan22_i2v_stability_ulysses2_vae_patch2_hsdp_slicing_tiling", + "description": "Ulysses=2 + VAE patch parallel=2 + HSDP + VAE slicing + tiling", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 2 + } + }, + "local_model", + "diffusion" + ], "server_params": { "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", "serve_args": { @@ -27,5 +40,45 @@ ] } ] + }, + { + "test_name": "test_wan22_i2v_stability_usp2_vae_patch2_hsdp_slicing", + "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 2 + } + }, + "local_model", + "diffusion" + ], + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "usp": 2, + "vae-patch-parallel-size": 2, + "use-hsdp": true, + "vae-use-slicing": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "i2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "enable_negative_prompt": true, + "random_request_config": [ + {"width": 832, "height": 480, "num_inference_steps": 4, "num_frames": 81, "fps": 16, "weight": 0.5}, + {"width": 1280, "height": 720, "num_inference_steps": 4, "num_frames": 121, "fps": 16, "weight": 0.5} + ] + } + ] } ] diff --git a/tests/helpers/mark.py b/tests/helpers/mark.py index 94e7db758..6f75773ff 100644 --- a/tests/helpers/mark.py +++ b/tests/helpers/mark.py @@ -73,8 +73,10 @@ def cuda_marks(*, res: str, num_cards: int): test_resource = pytest.mark.L4 elif res == "H100": test_resource = pytest.mark.H100 + elif res == "H800": + test_resource = pytest.mark.H800 else: - raise ValueError(f"Invalid CUDA resource type: {res}. Supported: L4, H100") + raise ValueError(f"Invalid CUDA resource type: {res}. Supported: L4, H100, H800") marks = [test_resource, test_platform_detail] if num_cards == 1: return marks @@ -137,7 +139,7 @@ def musa_marks(*, res: str, num_cards: int): def gpu_marks(*, res: str, num_cards: int): test_platform = pytest.mark.gpu - if res in ("L4", "H100"): + if res in ("L4", "H100", "H800"): return [test_platform] + cuda_marks(res=res, num_cards=num_cards) if res == "MI325": return [test_platform] + rocm_marks(res=res, num_cards=num_cards) @@ -145,7 +147,7 @@ def gpu_marks(*, res: str, num_cards: int): return [test_platform] + xpu_marks(res=res, num_cards=num_cards) if res == "S5000": return [test_platform] + musa_marks(res=res, num_cards=num_cards) - raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325, B60, S5000") + raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, H800, MI325, B60, S5000") def npu_marks(*, res: str, num_cards: int): diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index 501f6a605..b5b6b92ce 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -20,6 +20,8 @@ # acc — label contains "Accuracy Test" (incl. GEBench / GEdit-Bench style) # function — label has neither "Perf Test" nor "Accuracy Test" (incl. Doc, Multi-Replica, etc.) # stability— fixed dfx stability scripts under tests/dfx/stability/scripts/ (see below) +# Uses --pytest-mark / PYTEST_MARK for pytest -m (hardware via JSON marks). +# When unset, -m defaults to H800 (CUDA stability fleet; use --pytest-mark A3 on NPU). # local — pytest -sv -m " and local_model" from repo root (no YAML step extract) # When LABEL_SUBSTR is set, also runs matching tests/**/test_*.py and perf JSON configs under # tests/dfx/perf/tests/*.json via run_benchmark.py / run_diffusion_benchmark.py. @@ -42,7 +44,9 @@ # whose filename contains the substring (benchmark runner chosen by JSON family). # # stability (when included in TEST_TYPE): -# From repo root: pytest -s -v --run-level full_model tests/dfx/stability/scripts/test_stability_*.py +# From repo root: pytest -s -v --run-level full_model -m "" tests/dfx/stability/scripts/... +# -m comes from --pytest-mark / PYTEST_MARK when set (e.g. H800 or A3). +# Otherwise -m defaults to H800. PYTEST_MARK is rejected unless stability is enabled. # model_type: omni → qwen3_omni; tts → qwen3_tts; diffusion → qwen_image + wan22 + hunyuan_image; all → all five # LABEL_SUBSTR: if set, script path / job key / filename must contain it # @@ -73,6 +77,7 @@ # MODEL_TYPE - comma-separated and/or repeated flags (default: all); see above # LABEL_SUBSTR - YAML mode: substring of Buildkite step label; stability: substring of path/key/filename; # local: substring of test_*.py basename or tests/dfx/perf/tests/*.json filename +# PYTEST_MARK - pytest -m expression for stability jobs only (also --pytest-mark) # DRY_RUN=1 - print extracted commands only; do not write scripts or run pytest # RUN_JOB_TIMEOUT_KILL_AFTER - seconds after SIGTERM before SIGKILL on inline pytest # timeout (default: 60; baked into generated job scripts) @@ -82,6 +87,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LABEL_SUBSTR="${LABEL_SUBSTR:-}" +PYTEST_MARK="${PYTEST_MARK:-}" TEST_TYPE_ENV="${TEST_TYPE:-all}" MODEL_TYPE_ENV="${MODEL_TYPE:-all}" TEST_TYPE_CLI_PARTS=() @@ -204,6 +210,10 @@ while [[ $# -gt 0 ]]; do LABEL_SUBSTR="$2" shift 2 ;; + --pytest-mark) + PYTEST_MARK="$2" + shift 2 + ;; --model-type) MODEL_TYPE_FROM_CLI=1 _split_append_csv_array MODEL_TYPE_CLI_PARTS "$2" @@ -339,7 +349,7 @@ fi mkdir -p "${LOG_DIR}/jobs" LOG_DIR="$(cd "${LOG_DIR}" && pwd)" -export REPO_ROOT LOG_DIR YML LABEL_SUBSTR TEST_TYPE MODEL_TYPE DRY_RUN +export REPO_ROOT LOG_DIR YML LABEL_SUBSTR TEST_TYPE MODEL_TYPE DRY_RUN PYTEST_MARK # Drop stale wrapper scripts, old tee logs, and perf manifest so a filtered run does not # mix with the previous outputs. @@ -600,6 +610,26 @@ def _stability_model_matches(model_types: list[str], families: tuple[str, ...]) return bool(set(model_types) & set(families)) +STABILITY_DEFAULT_PYTEST_MARK = "H800" + + +def _shell_quote_pytest_mark(marker_expr: str) -> str: + """Shell-safe pytest ``-m`` value for generated bash job scripts.""" + expr = (marker_expr or "").strip() + if not expr: + raise ValueError("pytest -m expression must not be empty") + return shlex.quote(expr) + + +def _stability_pytest_mark(model_types: list[str]) -> str: + """Return explicit --pytest-mark or the default CUDA stability hardware marker.""" + del model_types # MODEL_TYPE filters scripts; default -m is hardware-specific. + override = (os.environ.get("PYTEST_MARK") or "").strip() + if override: + return override + return STABILITY_DEFAULT_PYTEST_MARK + + def _write_job_script(key: str, script_lines: list[str], jobs_dir: Path) -> None: body = "\n".join(script_lines) + "\n" if DRY_RUN: @@ -712,7 +742,7 @@ def run_local_mode( if rel_paths: paths_arg = " ".join(shlex.quote(p) for p in rel_paths) - pytest_line = f'pytest -sv -m "{marker_expr}" {paths_arg}' + pytest_line = f"pytest -sv -m {_shell_quote_pytest_mark(marker_expr)} {paths_arg}" slug = _local_job_slug(LABEL_SUBSTR) key = f"local_pytest_{slug}" header = ( @@ -732,7 +762,7 @@ def run_local_mode( for json_rel, runner_rel in perf_pairs: json_name = Path(json_rel).name pytest_line = ( - f'pytest -s -v -m "{marker_expr}" ' + f"pytest -s -v -m {_shell_quote_pytest_mark(marker_expr)} " f"{shlex.quote(runner_rel)} " f"--test-config-file {shlex.quote(json_rel)}" ) @@ -763,7 +793,7 @@ def run_local_mode( ) return matched - pytest_line = f'pytest -sv -m "{marker_expr}"' + pytest_line = f"pytest -sv -m {_shell_quote_pytest_mark(marker_expr)}" key = "local_pytest" header = f"# Local marker tests — MODEL_TYPE={','.join(model_types)}" script_lines = [ @@ -779,6 +809,8 @@ def run_local_mode( def run_stability_mode(jobs_dir: Path, model_types: list[str]) -> int: matched = 0 + marker_expr = _stability_pytest_mark(model_types) + mark_from_cli = bool((os.environ.get("PYTEST_MARK") or "").strip()) for key, rel_posix, families in STABILITY_CASES: if not _stability_model_matches(model_types, families): continue @@ -794,11 +826,17 @@ def run_stability_mode(jobs_dir: Path, model_types: list[str]) -> int: if not rel_path.is_file(): print(f"# skip (missing file): {rel_path}", file=sys.stderr) continue - pytest_line = f"pytest -s -v --run-level full_model {rel_posix}" + pytest_line = ( + f"pytest -s -v --run-level full_model " + f"-m {_shell_quote_pytest_mark(marker_expr)} " + f"{shlex.quote(rel_posix)}" + ) matched += 1 script_lines = [ "#!/usr/bin/env bash", f"# Stability: {key} — {rel_posix}", + f"# pytest -m {marker_expr!r}" + + (" (--pytest-mark)" if mark_from_cli else f" (default {STABILITY_DEFAULT_PYTEST_MARK})"), "set -euo pipefail", f'cd "{REPO_ROOT}"', pytest_line, @@ -816,6 +854,14 @@ def main() -> None: yaml_test_types = [t for t in test_types if t not in ("stability", "local")] needs_yaml = bool(yaml_test_types) + pytest_mark = (os.environ.get("PYTEST_MARK") or "").strip() + if pytest_mark and not want_stability: + print( + "PYTEST_MARK / --pytest-mark is only supported when --test-type includes stability", + file=sys.stderr, + ) + sys.exit(2) + jobs_dir = LOG_DIR / "jobs" if not DRY_RUN: jobs_dir.mkdir(parents=True, exist_ok=True)