diff --git a/.buildkite/cuda/test-weekly.yml b/.buildkite/cuda/test-weekly.yml index 5ddbf22ce..ed20e78dc 100644 --- a/.buildkite/cuda/test-weekly.yml +++ b/.buildkite/cuda/test-weekly.yml @@ -30,14 +30,6 @@ steps: - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" mirror_hardwares: h100_4 - - label: "Reliability Test - voxcpm2" - timeout_in_minutes: 180 - depends_on: upload-weekly-pipeline - if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" - commands: - - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" - mirror_hardwares: l4_1 - - group: "Reliability Test - Invalid parameters Test" depends_on: upload-weekly-pipeline if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index d1c2c560f..ff53cbbf7 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -184,9 +184,6 @@ Failed jobs: 1/2 # Reliability: HunyuanImage DiT (H100 × 4) pytest -s -v dfx/reliability/test_reliability_hunyuan_image.py -m slow - # Reliability: VoxCPM2 (L4) - pytest -s -v dfx/reliability/test_reliability_voxcpm2.py -m slow - ``` The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml). diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 397f5e83c..ea80ea21e 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -562,7 +562,6 @@ L5 level testing focuses on the performance of model services under ***long-runn - `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default). - `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery). - `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4). - - `test_reliability_voxcpm2.py` — VoxCPM2 TTS reliability (`/v1/audio/speech` under OOM and process kill; deploy `voxcpm2.yaml`, L4 × 1). - `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`). - `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready. - `README.md` — Short local run commands for this directory. @@ -662,13 +661,12 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in | `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). | | `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). | | `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. | -| `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. | | `README.md` | Minimal run / collect examples. | ##### Parametrization and markers - Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`. -- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4). +- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100). - **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**. ##### CI trigger @@ -680,13 +678,11 @@ Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model sui | Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) | | Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) | | Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) | -| Reliability Test - voxcpm2 | `test_reliability_voxcpm2.py` | L4 × 1 (`gpu_1_queue`) | ```bash pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" -pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" ``` ##### Local commands @@ -696,7 +692,6 @@ pytest --collect-only tests/dfx/reliability pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow -pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` ##### Adding a new model suite @@ -709,4 +704,4 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`; add `test_reliability_.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models) diff --git a/docs/user_guide/fault_injection_reliability_matrix.md b/docs/user_guide/fault_injection_reliability_matrix.md index 28271affa..4a2fa0a16 100644 --- a/docs/user_guide/fault_injection_reliability_matrix.md +++ b/docs/user_guide/fault_injection_reliability_matrix.md @@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b ## Source of Conclusions -The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`. +The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, and **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`) and runs weekly via `.buildkite/cuda/test-weekly.yml`. ### Example: `SIGTERM` fault injection log (Qwen3-Omni) diff --git a/tests/dfx/reliability/README.md b/tests/dfx/reliability/README.md index 8dc69dc55..79d862025 100644 --- a/tests/dfx/reliability/README.md +++ b/tests/dfx/reliability/README.md @@ -7,12 +7,10 @@ The detailed RFC document is maintained outside this repository (local/internal - Qwen3 tests: `test_reliability_qwen3_omni.py` - Wan2.2 tests: `test_reliability_wan22.py` - HunyuanImage tests: `test_reliability_hunyuan_image.py` -- VoxCPM2 tests: `test_reliability_voxcpm2.py` ```bash pytest --collect-only tests/dfx/reliability pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow -pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` diff --git a/tests/dfx/reliability/test_reliability_qwen3_omni.py b/tests/dfx/reliability/test_reliability_qwen3_omni.py index 32c660954..d12a11ba1 100644 --- a/tests/dfx/reliability/test_reliability_qwen3_omni.py +++ b/tests/dfx/reliability/test_reliability_qwen3_omni.py @@ -71,7 +71,7 @@ def _default_oom_device_spec() -> str: OOM_INJECTION_CONFIG = { "device": _default_oom_device_spec(), - "target_mem_ratio": 0.92, + "target_mem_ratio": 0.95, "hold_seconds": 0, "startup_timeout_sec": 20, "strict": False, diff --git a/tests/dfx/reliability/test_reliability_voxcpm2.py b/tests/dfx/reliability/test_reliability_voxcpm2.py deleted file mode 100644 index 632c51081..000000000 --- a/tests/dfx/reliability/test_reliability_voxcpm2.py +++ /dev/null @@ -1,508 +0,0 @@ -""" -VoxCPM2 reliability integration tests. - -Scenarios follow ``docs/user_guide/fault_injection_reliability_matrix.md``. -""" - -from __future__ import annotations - -import concurrent.futures -import os -import time -from pathlib import Path -from typing import Any - -import pytest - -from tests.dfx.conftest import ( - assert_fault_exception, - create_reliability_omni_server_params, - resolve_oom_device_spec, -) -from tests.dfx.reliability.helpers import ( - FaultInjector, - assert_no_server_tree_process_residual_and_gpu_release, - extract_openai_error_contract_from_bytes, - get_health_raw, - inject_gpu_oom, - make_process_kill_fault_injector, - make_server_root_kill_fault_injector, - make_server_tree_kill_fault_injector, - post_json_raw, - run_fault_injection_with_rate_load, - stop_gpu_oom_hogs, - worker_residual_timeout_after_kill_signal, -) -from tests.helpers.mark import hardware_test -from tests.helpers.media import load_test_audio_data_url -from tests.helpers.runtime import OpenAIClientHandler - -RELIABILITY_SCENARIOS: list[dict[str, Any]] = [ - { - "test_name": "voxcpm2_reliability_default", - "server_params": { - "model": "openbmb/VoxCPM2", - "stage_config_name": "voxcpm2.yaml", - # VoxCPM2 uses a custom HF tokenizer; CLI default trust_remote_code=False - # would otherwise override deploy YAML settings during server startup. - "server_args": ["--trust-remote-code", "--disable-log-stats"], - }, - }, -] - -DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy" -OOM_INJECTION_CONFIG = { - "target_mem_ratio": 0.96, - "hold_seconds": 0, - "startup_timeout_sec": 20, - "strict": False, -} -FAULT_ERROR_KEYWORDS = ( - "the request failed", - "oom", - "out of memory", - "cuda", - "orchestrator", - "timeout", - "connection", - "500", - "503", -) -PROCESS_KILL_ERROR_KEYWORDS = ( - "timeout", - "did not complete within", - "connection", - "engine", - "orchestrator", - "dead", - "internal", - "500", - "503", -) -RUNTIME_WORKER_PATTERN = "VLLM::" -SERVE_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGINT", id="sigint"), - pytest.param("SIGKILL", id="sigkill"), -] -SERVE_WITH_LOAD_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGINT", id="sigint"), - pytest.param("SIGKILL", id="sigkill"), -] -TREE_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGKILL", id="sigkill"), -] -TREE_WITH_LOAD_SIGNAL_PARAMS = [ - pytest.param("SIGTERM", id="sigterm"), - pytest.param("SIGKILL", id="sigkill"), -] -WORKER_SIGNAL_FAULT_PARAMS = [ - pytest.param( - make_process_kill_fault_injector( - grep_patterns=RUNTIME_WORKER_PATTERN, - signal_name="SIGTERM", - limit=1, - post_kill_wait_seconds=2.0, - ), - id="runtime_process_chain_sigterm", - ), - pytest.param( - make_process_kill_fault_injector( - grep_patterns=RUNTIME_WORKER_PATTERN, - signal_name="SIGKILL", - limit=1, - post_kill_wait_seconds=2.0, - ), - id="runtime_process_chain_sigkill", - ), -] - -VOXCPM2_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR) -INFLIGHT_INJECTION_REQUEST_RATE = 0.3 -INFLIGHT_INJECTION_REQUEST_COUNT = 10 -_SPEECH_FF_HTTP_TIMEOUT_SEC = 30 -_SPEECH_FF_MAX_ELAPSED_SEC = 30 - - -def _speech_request_config(omni_server: Any) -> dict[str, Any]: - return { - "model": omni_server.model, - "input": "The weather is nice today, perfect for a walk in the park.", - "stream": False, - "timeout": 120.0, - "response_format": "wav", - "voice": "default", - } - - -def _submit_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None: - openai_client.send_audio_speech_request(_speech_request_config(omni_server), request_num=1) - - -def _oom_speech_request_config(omni_server: Any) -> dict[str, Any]: - base = "The weather is nice today, perfect for a walk in the park. " - return { - "model": omni_server.model, - "input": base * 80, - "max_new_tokens": 4096, - "ref_audio": load_test_audio_data_url("qwen3_tts/clone_2.wav"), - "stream": False, - "timeout": 300.0, - "response_format": "wav", - "voice": "default", - } - - -def _submit_oom_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None: - openai_client.send_audio_speech_request(_oom_speech_request_config(omni_server), request_num=1) - - -def _assert_post_fault_speech_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None: - payload = { - "model": model, - "input": "post-fault fast-fail check", - "voice": "default", - } - start = time.monotonic() - try: - status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=20) - elapsed = time.monotonic() - start - assert elapsed < 15, f"[{scenario} fast_fail] /v1/audio/speech did not fail fast: {elapsed:.2f}s" - assert status >= 500, ( - f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}" - ) - except Exception: # noqa: BLE001 - elapsed = time.monotonic() - start - assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s" - - -def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None: - deadline = time.monotonic() + 20.0 - last_observation = "" - while time.monotonic() < deadline: - try: - status, body = get_health_raw(host, port, timeout_sec=5) - last_observation = f"http={status}, body={body[:200]!r}" - if status == 503: - return - except Exception as exc: # noqa: BLE001 - last_observation = f"exception={exc!r}" - return - time.sleep(0.5) - pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}") - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skip(reason="issue#4285") -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_gpu_oom_speech_request_failure( - omni_server_function, - openai_client_function, -) -> None: - stage_config_path = getattr(omni_server_function, "stage_config_path", None) - device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) - handle = inject_gpu_oom( - device=device_spec, - target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], - hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], - startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], - strict=OOM_INJECTION_CONFIG["strict"], - ) - try: - try: - _submit_oom_speech_request(openai_client_function, omni_server_function) - except Exception as exc: - assert_fault_exception(exc, FAULT_ERROR_KEYWORDS) - else: - pytest.fail("expected /v1/audio/speech request failure during GPU OOM injection") - finally: - stop_gpu_oom_hogs(handle) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skip(reason="issue#4285") -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_gpu_oom_speech_error_contract( - omni_server_function, -) -> None: - stage_config_path = getattr(omni_server_function, "stage_config_path", None) - device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) - handle = inject_gpu_oom( - device=device_spec, - target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], - hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], - startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], - strict=OOM_INJECTION_CONFIG["strict"], - ) - host = omni_server_function.host - port = omni_server_function.port - oom_config = _oom_speech_request_config(omni_server_function) - payload = { - "model": oom_config["model"], - "input": oom_config["input"], - "voice": oom_config["voice"], - "max_new_tokens": oom_config["max_new_tokens"], - "ref_audio": oom_config["ref_audio"], - } - try: - status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=300) - finally: - stop_gpu_oom_hogs(handle) - - assert status >= 500, f"expected speech runtime error under OOM, got status={status}" - speech_error = extract_openai_error_contract_from_bytes(body) - assert speech_error is not None, f"speech error payload not OpenAI-compatible: {body[:300]!r}" - assert "code" in speech_error, f"speech error lacks code field: {speech_error!r}" - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize( - "fault_injector", - WORKER_SIGNAL_FAULT_PARAMS, - indirect=True, -) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_request_failure( - omni_server_after_fault_function, - openai_client_function, -) -> None: - try: - _submit_speech_request(openai_client_function, omni_server_after_fault_function) - except Exception as exc: - assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) - else: - pytest.fail("expected /v1/audio/speech request failure after process-kill injection") - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize( - "fault_injector", - WORKER_SIGNAL_FAULT_PARAMS, - indirect=True, -) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_fast_fail_and_concurrent( - omni_server_after_fault_function, - openai_client_function, -) -> None: - host = omni_server_after_fault_function.host - port = omni_server_after_fault_function.port - model = omni_server_after_fault_function.model - payload = { - "model": model, - "input": "Say hello in one short sentence.", - "voice": "default", - } - - start = time.monotonic() - try: - status, body = post_json_raw( - host, - port, - "/v1/audio/speech", - payload, - timeout_sec=_SPEECH_FF_HTTP_TIMEOUT_SEC, - ) - elapsed = time.monotonic() - start - assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, ( - f"[process_kill fast_fail] /v1/audio/speech did not fail fast after fault: {elapsed:.2f}s" - ) - assert status >= 500, ( - "[process_kill fast_fail] expected server-side failure after fault, " - f"got status={status}, body={body[:200]!r}" - ) - except Exception: - elapsed = time.monotonic() - start - assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, ( - f"[process_kill fast_fail] /v1/audio/speech exception was too slow after fault: {elapsed:.2f}s" - ) - - start = time.monotonic() - with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: - futures = [ - executor.submit(_submit_speech_request, openai_client_function, omni_server_after_fault_function) - for _ in range(3) - ] - done, pending = concurrent.futures.wait( - futures, - timeout=30, - return_when=concurrent.futures.ALL_COMPLETED, - ) - - elapsed = time.monotonic() - start - assert not pending, f"[process_kill concurrent] some fault-time speech requests hung: pending={len(pending)}" - assert elapsed < 30, f"[process_kill concurrent] fault-time speech request convergence is too slow: {elapsed:.2f}s" - - fault_observed = False - for future in done: - try: - future.result() - except Exception as exc: - fault_observed = True - assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) - assert fault_observed, ( - "[process_kill concurrent] expected at least one /v1/audio/speech request to fail after fault" - ) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize( - "fault_injector", - WORKER_SIGNAL_FAULT_PARAMS, - indirect=True, -) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_worker_with_load_request_failure( - omni_server_function, - openai_client_function, - fault_injector: FaultInjector, -) -> None: - scenario = "kill_worker_with_load" - load_result = run_fault_injection_with_rate_load( - submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), - inject_fault=lambda: fault_injector(omni_server_function), - num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, - request_rate=INFLIGHT_INJECTION_REQUEST_RATE, - completion_timeout_sec=120.0, - ) - assert load_result["failure_observed"], ( - f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" - ) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - - -@pytest.mark.slow -@pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup( - omni_server_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_root_no_load_{signal_name.lower()}" - injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) - injector(omni_server_function) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) - - -@pytest.mark.slow -@pytest.mark.tts -@hardware_test(res={"cuda": "L4"}, num_cards=1) -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", SERVE_WITH_LOAD_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup( - omni_server_function, - openai_client_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_root_with_load_{signal_name.lower()}" - injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) - load_result = run_fault_injection_with_rate_load( - submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), - inject_fault=lambda: injector(omni_server_function), - num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, - request_rate=INFLIGHT_INJECTION_REQUEST_RATE, - completion_timeout_sec=300.0, - ) - assert load_result["failure_observed"], ( - f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" - ) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup( - omni_server_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_tree_no_load_{signal_name.lower()}" - injector = make_server_tree_kill_fault_injector( - signal_name=signal_name, - post_kill_wait_seconds=2.0, - inter_kill_wait_seconds=0.1, - ) - injector(omni_server_function) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") -@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS) -@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) -def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup( - omni_server_function, - openai_client_function, - signal_name: str, -) -> None: - scenario = f"kill_serve_tree_with_load_{signal_name.lower()}" - injector = make_server_tree_kill_fault_injector( - signal_name=signal_name, - post_kill_wait_seconds=2.0, - inter_kill_wait_seconds=0.1, - ) - load_result = run_fault_injection_with_rate_load( - submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), - inject_fault=lambda: injector(omni_server_function), - num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, - request_rate=INFLIGHT_INJECTION_REQUEST_RATE, - completion_timeout_sec=120.0, - ) - assert load_result["failure_observed"], ( - f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" - ) - host = omni_server_function.host - port = omni_server_function.port - _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) - _assert_post_fault_health_terminal(host, port, scenario=scenario) - assert_no_server_tree_process_residual_and_gpu_release( - omni_server_function, - scenario=scenario, - timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), - ) diff --git a/tests/dfx/stability/scripts/test_stability_voxcpm2.py b/tests/dfx/stability/scripts/test_stability_voxcpm2.py deleted file mode 100644 index e26a2050c..000000000 --- a/tests/dfx/stability/scripts/test_stability_voxcpm2.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -VoxCPM2 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. - -Configuration: ``tests/dfx/stability/tests/test_voxcpm2.json``. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.dfx.conftest import ( - create_benchmark_indices, - create_test_parameter_mapping, - create_unique_server_params, - load_configs, -) -from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop - -STABILITY_DIR = Path(__file__).resolve().parent.parent -DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" -CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_voxcpm2.json") -DEFAULT_NUM_PROMPTS_PER_BATCH = 20 -STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"] - -try: - BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) -except FileNotFoundError: - BENCHMARK_CONFIGS = [] - -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] -server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} -benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) - - -@pytest.mark.slow -@pytest.mark.tts -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) -def test_stability_voxcpm2(omni_server, stability_benchmark_params): - test_name = stability_benchmark_params["test_name"] - params = stability_benchmark_params["params"] - duration_sec = params.get("duration_sec", 300) - num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) - request_rate = params.get("request_rate") - max_concurrency = params.get("max_concurrency") - - bench_params = { - k: v - for k, v in params.items() - if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") - } - - result = run_stability_benchmark_loop( - host=omni_server.host, - port=omni_server.port, - model=omni_server.model, - duration_sec=duration_sec, - params=bench_params, - request_rate=request_rate, - max_concurrency=max_concurrency, - result_dir=str(STABILITY_DIR), - num_prompts_per_batch=num_prompts_per_batch, - run_one_batch=_run_one_vllm_bench_batch, - ) - - assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" - assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index a95eb8876..501f6a605 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -43,7 +43,7 @@ # # stability (when included in TEST_TYPE): # From repo root: pytest -s -v --run-level full_model tests/dfx/stability/scripts/test_stability_*.py -# model_type: omni → qwen3_omni; tts → qwen3_tts + voxcpm2; diffusion → qwen_image + wan22 + hunyuan_image; all → all six +# model_type: omni → qwen3_omni; tts → qwen3_tts; diffusion → qwen_image + wan22 + hunyuan_image; all → all five # LABEL_SUBSTR: if set, script path / job key / filename must contain it # # Requirements: bash, python3, PyYAML (pip install pyyaml) @@ -588,7 +588,6 @@ def _write_job_timeouts_manifest(jobs_dir: Path, job_timeouts: dict[str, int]) - STABILITY_CASES: list[tuple[str, str, tuple[str, ...]]] = [ ("stability_qwen3_omni", "tests/dfx/stability/scripts/test_stability_qwen3_omni.py", ("omni",)), ("stability_qwen3_tts", "tests/dfx/stability/scripts/test_stability_qwen3_tts.py", ("tts",)), - ("stability_voxcpm2", "tests/dfx/stability/scripts/test_stability_voxcpm2.py", ("tts",)), ("stability_qwen_image", "tests/dfx/stability/scripts/test_stability_qwen_image.py", ("diffusion",)), ("stability_wan22", "tests/dfx/stability/scripts/test_stability_wan22.py", ("diffusion",)), ("stability_hunyuan_image", "tests/dfx/stability/scripts/test_stability_hunyuan_image.py", ("diffusion",)),