diff --git a/.claude/skills/production-add-diffusion-model/references/production-validation.md b/.claude/skills/production-add-diffusion-model/references/production-validation.md index eed568415..3b2f505fa 100644 --- a/.claude/skills/production-add-diffusion-model/references/production-validation.md +++ b/.claude/skills/production-add-diffusion-model/references/production-validation.md @@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor: ```bash bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \ - pytest -s -v tests/dfx/stability/scripts/test_stability_.py -m slow + pytest -s -v tests/dfx/stability/scripts/run_stability_.py -m slow ``` Check the target revision's monitor backends. Do not claim CPU/NPU monitoring diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index 7597f72e4..d3b02d27f 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -177,10 +177,10 @@ Failed jobs: 1/2 cd tests # Stability: Qwen3-Omni - pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow + pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow # Stability: Wan2.2 (v1/videos diffusion benchmark loop) - pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow + pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow # Reliability: Qwen3-Omni (H100 × 2) pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 762863bb3..47a629112 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. - ***Run Command***: - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) + - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_.py` alongside a matching JSON config) - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`). - ***Script Example***: @@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn ##### Stability -When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_.py`. The following illustrates the Qwen3-Omni shape: +When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_.py`. The following illustrates the Qwen3-Omni shape: ```json { diff --git a/tests/dfx/stability/scripts/test_stability_hunyuan_image.py b/tests/dfx/stability/scripts/run_stability_hunyuan_image.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_hunyuan_image.py rename to tests/dfx/stability/scripts/run_stability_hunyuan_image.py index be2aec805..d9db77205 100644 --- a/tests/dfx/stability/scripts/test_stability_hunyuan_image.py +++ b/tests/dfx/stability/scripts/run_stability_hunyuan_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Hunyuan-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_hunyuan_image.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_hunyuan_image(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py b/tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py new file mode 100644 index 000000000..a0e7dfce8 --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +""" +HunyuanVideo-1.5 I2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. + +Configuration: ``tests/dfx/stability/tests/test_hunyuanvideo15.json``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_hunyuanvideo15.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 50 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600", "--init-timeout", "900"] + +try: + BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, diffusion) come from test_hunyuanvideo15.json. +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_hunyuanvideo15(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_diffusion_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py b/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py new file mode 100644 index 000000000..fcb0590ce --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +MiniCPM-o-4.5 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. + +Configuration: ``tests/dfx/stability/tests/test_minicpmo_4_5.json``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minicpmo_4_5.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"] + +try: + BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, omni) come from test_minicpmo_4_5.json. +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_minicpmo_4_5(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_vllm_bench_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/run_stability_minimax_h3.py b/tests/dfx/stability/scripts/run_stability_minimax_h3.py new file mode 100644 index 000000000..252d830d5 --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_minimax_h3.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +MiniMax-H3 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. + +Configuration: ``tests/dfx/stability/tests/test_minimax_h3.json``. + +``--lora-path`` is resolved like e2e FastH3 / Buildkite ``test-merge.yml``: +set ``VLLM_TEST_MINIMAX_H3_FASTH3_LORA`` to the adapter file, or populate the local +HF cache for ``FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop +from tests.e2e.online_serving.minimax_h3._common import FASTH3_LORA + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minimax_h3.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "1800", "--init-timeout", "1800"] + + +def _inject_fasth3_lora_path(configs: list[dict]) -> list[dict]: + """Fill serve_args.lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA / HF cache.""" + if not FASTH3_LORA: + return configs + for config in configs: + serve_args = config.setdefault("server_params", {}).setdefault("serve_args", {}) + serve_args["lora-path"] = FASTH3_LORA + return configs + + +try: + BENCHMARK_CONFIGS = _inject_fasth3_lora_path(load_configs(CONFIG_FILE_PATH)) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, diffusion) come from test_minimax_h3.json. +@pytest.mark.skipif( + FASTH3_LORA is None, + reason="set VLLM_TEST_MINIMAX_H3_FASTH3_LORA or populate the local HF cache", +) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_minimax_h3(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_diffusion_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py b/tests/dfx/stability/scripts/run_stability_qwen3_omni.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen3_omni.py rename to tests/dfx/stability/scripts/run_stability_qwen3_omni.py index 98ab5ee89..ddb3339d1 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py +++ b/tests/dfx/stability/scripts/run_stability_qwen3_omni.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen3-Omni stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.omni +# Pytest marks (hardware, local_model, slow, omni) come from test_qwen3_omni.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen3_omni(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py b/tests/dfx/stability/scripts/run_stability_qwen3_tts.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen3_tts.py rename to tests/dfx/stability/scripts/run_stability_qwen3_tts.py index 293b723ca..dde525b36 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py +++ b/tests/dfx/stability/scripts/run_stability_qwen3_tts.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen3-TTS stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.tts +# Pytest marks (hardware, local_model, slow, tts) come from test_qwen3_tts.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen3_tts(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_qwen_image.py b/tests/dfx/stability/scripts/run_stability_qwen_image.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen_image.py rename to tests/dfx/stability/scripts/run_stability_qwen_image.py index 62b0cd86c..84e15fccb 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen_image.py +++ b/tests/dfx/stability/scripts/run_stability_qwen_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_qwen_image.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen_image(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_wan22.py b/tests/dfx/stability/scripts/run_stability_wan22.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_wan22.py rename to tests/dfx/stability/scripts/run_stability_wan22.py index aae0ae429..08c711d85 100644 --- a/tests/dfx/stability/scripts/test_stability_wan22.py +++ b/tests/dfx/stability/scripts/run_stability_wan22.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Wan2.2 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_wan22.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_wan22(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/tests/test_hunyuan_image.json b/tests/dfx/stability/tests/test_hunyuan_image.json index b468ceec3..43c92d1e2 100644 --- a/tests/dfx/stability/tests/test_hunyuan_image.json +++ b/tests/dfx/stability/tests/test_hunyuan_image.json @@ -7,10 +7,11 @@ "res": { "cuda": "H800" }, - "num_cards": 2 + "num_cards": 4 } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -44,5 +45,53 @@ "enable_negative_prompt": true } ] + }, + { + "test_name": "test_hunyuan_image_stability_tp2_sp2_a3", + "description": "A3: same as perf test_hunyuan_image_tp2_sp2 — DiT-only, TP=2 USP=2, 4 dies, no fp8", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 4 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "ulysses-degree": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2i", + "backend": "vllm-omni", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "width": { + "min": 512, + "max": 1024 + }, + "height": { + "min": 512, + "max": 1024 + }, + "num_inference_steps": 50, + "enable_negative_prompt": true + } + ] } ] diff --git a/tests/dfx/stability/tests/test_hunyuanvideo15.json b/tests/dfx/stability/tests/test_hunyuanvideo15.json new file mode 100644 index 000000000..1da2feec3 --- /dev/null +++ b/tests/dfx/stability/tests/test_hunyuanvideo15.json @@ -0,0 +1,57 @@ +[ + { + "test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2_stability", + "description": "HunyuanVideo-1.5 I2V 24h: CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H800"] + }, + "num_cards": 2 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "i2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "num_input_images": 1, + "seed": 42, + "enable_negative_prompt": true, + "random_request_config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra_body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_minicpmo_4_5.json b/tests/dfx/stability/tests/test_minicpmo_4_5.json new file mode 100644 index 000000000..eaf38ec1d --- /dev/null +++ b/tests/dfx/stability/tests/test_minicpmo_4_5.json @@ -0,0 +1,53 @@ +[ + { + "test_name": "test_minicpmo_4_5_stability_ready", + "description": "MiniCPM-o-4.5 ready deploy + seed-tts chat/audio stability (from perf test_minicpmo_4_5_ready)", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 1 + } + }, + "local_model", + "slow", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_minimax_h3.json b/tests/dfx/stability/tests/test_minimax_h3.json new file mode 100644 index 000000000..d671c48fd --- /dev/null +++ b/tests/dfx/stability/tests/test_minimax_h3.json @@ -0,0 +1,62 @@ +[ + { + "test_name": "test_minimax_h3_t2v_stability_4gpu", + "description": "MiniMax-H3 FastH3 T2VA on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile); fused lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA; requests must use num_inference_steps=4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 4 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800 + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "seed": 42, + "random_request_config": [ + { + "width": 1344, + "height": 768, + "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", + "num_inference_steps": 4, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra_body": { + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\"}" + } + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_qwen3_omni.json b/tests/dfx/stability/tests/test_qwen3_omni.json index 816e981b6..bc2f7dacd 100644 --- a/tests/dfx/stability/tests/test_qwen3_omni.json +++ b/tests/dfx/stability/tests/test_qwen3_omni.json @@ -5,12 +5,14 @@ { "hardware_marks": { "res": { - "cuda": "H800" + "cuda": "H800", + "npu": "A3" }, "num_cards": 2 } }, "local_model", + "slow", "omni" ], "server_params": { diff --git a/tests/dfx/stability/tests/test_qwen3_tts.json b/tests/dfx/stability/tests/test_qwen3_tts.json index 1c9356574..37b7cca52 100644 --- a/tests/dfx/stability/tests/test_qwen3_tts.json +++ b/tests/dfx/stability/tests/test_qwen3_tts.json @@ -12,6 +12,7 @@ } }, "local_model", + "slow", "tts" ], "server_params": { diff --git a/tests/dfx/stability/tests/test_qwen_image.json b/tests/dfx/stability/tests/test_qwen_image.json index 0c21d0ac1..9756f1299 100644 --- a/tests/dfx/stability/tests/test_qwen_image.json +++ b/tests/dfx/stability/tests/test_qwen_image.json @@ -11,6 +11,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -36,5 +37,44 @@ "enable_negative_prompt": true } ] + }, + { + "test_name": "test_qwen_image_stability_a3", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 1 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "Qwen/Qwen-Image" + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2i", + "backend": "vllm-omni", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "width": { + "min": 512, + "max": 1000 + }, + "height": { + "min": 512, + "max": 1000 + }, + "num_inference_steps": 50, + "enable_negative_prompt": true + } + ] } ] diff --git a/tests/dfx/stability/tests/test_wan22.json b/tests/dfx/stability/tests/test_wan22.json index e6ab0b5cf..de6627de4 100644 --- a/tests/dfx/stability/tests/test_wan22.json +++ b/tests/dfx/stability/tests/test_wan22.json @@ -12,6 +12,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -54,6 +55,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index c1cca73e2..8a0809726 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -47,7 +47,10 @@ # From repo root: pytest -s -v --run-level full_model -m "" tests/dfx/stability/scripts/... # -m comes from --pytest-mark / PYTEST_MARK when set (e.g. H800 or A3). # Otherwise -m defaults to H800. PYTEST_MARK is rejected unless stability is enabled. -# model_type: omni → qwen3_omni; tts → qwen3_tts; diffusion → qwen_image + wan22 + hunyuan_image; all → all five +# model_type: omni → qwen3_omni + minicpmo_4_5; tts → qwen3_tts; +# diffusion → qwen_image + wan22 + hunyuan_image + minimax_h3 + hunyuanvideo15; all → all eight +# stability_minimax_h3: download FastH3 LoRA only when VLLM_TEST_MINIMAX_H3_FASTH3_LORA is unset, +# then export before pytest (collection skipif needs the path). # LABEL_SUBSTR: if set, script path / job key / filename must contain it # # Requirements: bash, python3, PyYAML (pip install pyyaml) @@ -596,11 +599,14 @@ def _write_job_timeouts_manifest(jobs_dir: Path, job_timeouts: dict[str, int]) - # Fixed stability scripts (when stability is selected); MODEL_TYPES narrows which run. STABILITY_CASES: list[tuple[str, str, tuple[str, ...]]] = [ - ("stability_qwen3_omni", "tests/dfx/stability/scripts/test_stability_qwen3_omni.py", ("omni",)), - ("stability_qwen3_tts", "tests/dfx/stability/scripts/test_stability_qwen3_tts.py", ("tts",)), - ("stability_qwen_image", "tests/dfx/stability/scripts/test_stability_qwen_image.py", ("diffusion",)), - ("stability_wan22", "tests/dfx/stability/scripts/test_stability_wan22.py", ("diffusion",)), - ("stability_hunyuan_image", "tests/dfx/stability/scripts/test_stability_hunyuan_image.py", ("diffusion",)), + ("stability_qwen3_omni", "tests/dfx/stability/scripts/run_stability_qwen3_omni.py", ("omni",)), + ("stability_qwen3_tts", "tests/dfx/stability/scripts/run_stability_qwen3_tts.py", ("tts",)), + ("stability_minicpmo_4_5", "tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py", ("omni",)), + ("stability_qwen_image", "tests/dfx/stability/scripts/run_stability_qwen_image.py", ("diffusion",)), + ("stability_wan22", "tests/dfx/stability/scripts/run_stability_wan22.py", ("diffusion",)), + ("stability_hunyuan_image", "tests/dfx/stability/scripts/run_stability_hunyuan_image.py", ("diffusion",)), + ("stability_minimax_h3", "tests/dfx/stability/scripts/run_stability_minimax_h3.py", ("diffusion",)), + ("stability_hunyuanvideo15", "tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py", ("diffusion",)), ] @@ -612,6 +618,25 @@ def _stability_model_matches(model_types: list[str], families: tuple[str, ...]) STABILITY_DEFAULT_PYTEST_MARK = "H800" +# Extra shell lines before pytest. MiniMax-H3 FastH3 skipif reads +# VLLM_TEST_MINIMAX_H3_FASTH3_LORA at collection; merge CI downloads the same way. +_FASTH3_LORA_DOWNLOAD = ( + 'VLLM_TEST_MINIMAX_H3_FASTH3_LORA="$(python3 -c ' + "'from huggingface_hub import hf_hub_download; " + "print(hf_hub_download(repo_id=\"FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA\", " + "filename=\"dense-datafree/adapter_model.safetensors\"))')\"" +) +STABILITY_JOB_PRELUDE: dict[str, list[str]] = { + "stability_minimax_h3": [ + "if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then", + ' if [ -z "${VLLM_TEST_MINIMAX_H3_FASTH3_LORA:-}" ]; then', + f" {_FASTH3_LORA_DOWNLOAD}", + " fi", + " export VLLM_TEST_MINIMAX_H3_FASTH3_LORA", + "fi", + ], +} + def _shell_quote_pytest_mark(marker_expr: str) -> str: """Shell-safe pytest ``-m`` value for generated bash job scripts.""" @@ -839,6 +864,7 @@ def run_stability_mode(jobs_dir: Path, model_types: list[str]) -> int: + (" (--pytest-mark)" if mark_from_cli else f" (default {STABILITY_DEFAULT_PYTEST_MARK})"), "set -euo pipefail", f'cd "{REPO_ROOT}"', + *STABILITY_JOB_PRELUDE.get(key, []), pytest_line, ] _write_job_script(key, script_lines, jobs_dir)