diff --git a/.buildkite/test-weekly.yml b/.buildkite/test-weekly.yml index 57eadeafd..aefb424e5 100644 --- a/.buildkite/test-weekly.yml +++ b/.buildkite/test-weekly.yml @@ -85,6 +85,66 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: "Reliability Test - hunyuan-image" + timeout_in_minutes: 180 + depends_on: upload-weekly-pipeline + if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" + commands: + - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + + - label: "Reliability Test - voxcpm2" + timeout_in_minutes: 180 + depends_on: upload-weekly-pipeline + if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test" + commands: + - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" + agents: + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + shm-size: "8gb" + environment: + - "HF_HOME=/fsx/hf_cache" + - "HF_TOKEN" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips). - group: "Reliability Test - Invalid parameters Test" depends_on: upload-weekly-pipeline diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index adf29bfa5..c27f1fc9a 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -155,7 +155,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari /tests/dfx/stability/tests/test_wan22.json
Reliability:
tests/dfx/reliability/test_reliability_{model_key}.py
- (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py) + (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py, test_reliability_hunyuan_image.py, test_reliability_voxcpm2.py) Chapter 4
@@ -741,14 +741,16 @@ L5 level testing focuses on the performance of model services under ***long-runn ### 4.2 Testing Content and Scope - ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable. -- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni or `multiprocessing.spawn` for Wan2.2 video workers), **large multimodal chat** or **`/v1/videos`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `
` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill). +- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `
` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill). ### 4.3 Test Directory and Execution Files - ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family) - ***Reliability Test Suite*** (`tests/dfx/reliability/`): - `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default). - - `test_reliability_wan22.py` — Wan2.2 T2V video API reliability (`/v1/videos` under OOM and process kill, recovery). + - `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery). + - `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4). + - `test_reliability_voxcpm2.py` — VoxCPM2 TTS reliability (`/v1/audio/speech` under OOM and process kill; deploy `voxcpm2.yaml`, L4 × 1). - `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`). - `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready. - `README.md` — Short local run commands for this directory. @@ -846,22 +848,33 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in | `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). | | `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). | | `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). | -| `test_reliability_wan22.py` | Wan2.2 T2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). | +| `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). | +| `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. | +| `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. | | `README.md` | Minimal run / collect examples. | #### Parametrization and markers - Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`. -- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni paths often require **2** cards; Wan2.2 video paths **1** card). +- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4). - **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**. #### CI trigger -Weekly Buildkite (`.buildkite/test-weekly.yml`) runs, for example: +Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example: + +| Buildkite step | Test file | CI hardware | +| -------------- | --------- | ----------- | +| Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) | +| Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) | +| Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) | +| Reliability Test - voxcpm2 | `test_reliability_voxcpm2.py` | L4 × 1 (`gpu_1_queue`) | ```bash pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow" pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" +pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow" +pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow" ``` #### Local commands @@ -870,6 +883,8 @@ pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow" pytest --collect-only tests/dfx/reliability pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow +pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow +pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` #### Adding a new model suite @@ -882,7 +897,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
- - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) - - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow` and/or `pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow` (add `test_reliability_.py` for new models) + - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/test-weekly.yml` for new models) ## Summary diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md index 36dae4451..0db814e08 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_guide.md @@ -96,15 +96,21 @@ Our test scripts use the pytest framework. First, please use `git clone https:// # Stability: Wan2.2 (v1/videos diffusion benchmark loop) pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow - # Reliability: Qwen3-Omni + # Reliability: Qwen3-Omni (H100 × 2) pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow - # Reliability: Wan2.2 + # Reliability: Wan2.2 (H100) pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow + # Reliability: HunyuanImage DiT (H100 × 4) + pytest -s -v dfx/reliability/test_reliability_hunyuan_image.py -m slow + + # Reliability: VoxCPM2 (L4) + pytest -s -v dfx/reliability/test_reliability_voxcpm2.py -m slow + ``` - The latest L5 commands for CI can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml). + The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml). You can find more information about markers in the documentation: [marker doc](./tests_markers.md) diff --git a/docs/user_guide/fault_injection_reliability_matrix.md b/docs/user_guide/fault_injection_reliability_matrix.md index 2b5fd2401..974982b3b 100644 --- a/docs/user_guide/fault_injection_reliability_matrix.md +++ b/docs/user_guide/fault_injection_reliability_matrix.md @@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b ## Source of Conclusions -The behaviors and conclusions above are summarized from current fault injection validation results on `Qwen3-Omni` and `Wan2.2`. +The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`. ### Example: `SIGTERM` fault injection log (Qwen3-Omni) diff --git a/tests/dfx/reliability/README.md b/tests/dfx/reliability/README.md index 7196101bc..8dc69dc55 100644 --- a/tests/dfx/reliability/README.md +++ b/tests/dfx/reliability/README.md @@ -6,9 +6,13 @@ The detailed RFC document is maintained outside this repository (local/internal - Fault helpers: `conftest.py`, `helpers.py` - Qwen3 tests: `test_reliability_qwen3_omni.py` - Wan2.2 tests: `test_reliability_wan22.py` +- HunyuanImage tests: `test_reliability_hunyuan_image.py` +- VoxCPM2 tests: `test_reliability_voxcpm2.py` ```bash pytest --collect-only tests/dfx/reliability pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow +pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow +pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow ``` diff --git a/tests/dfx/reliability/test_reliability_hunyuan_image.py b/tests/dfx/reliability/test_reliability_hunyuan_image.py new file mode 100644 index 000000000..43a684d1d --- /dev/null +++ b/tests/dfx/reliability/test_reliability_hunyuan_image.py @@ -0,0 +1,478 @@ +""" +HunyuanImage-3.0-Instruct reliability integration tests (DiT-only deploy). + +Uses ``hunyuan_image3_dit.yaml`` (single diffusion stage, 4-GPU TP=4, no +AR/Mooncake connector) for simpler local bring-up. Scenarios follow +``docs/user_guide/fault_injection_reliability_matrix.md``. +""" + +from __future__ import annotations + +import concurrent.futures +import os +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.dfx.conftest import ( + assert_fault_exception, + create_reliability_omni_server_params, + resolve_oom_device_spec, +) +from tests.dfx.reliability.helpers import ( + FaultInjector, + assert_no_server_tree_process_residual_and_gpu_release, + get_health_raw, + inject_gpu_oom, + make_process_kill_fault_injector, + make_server_root_kill_fault_injector, + make_server_tree_kill_fault_injector, + post_json_raw, + run_fault_injection_with_rate_load, + stop_gpu_oom_hogs, + worker_residual_timeout_after_kill_signal, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OpenAIClientHandler + +RELIABILITY_SCENARIOS: list[dict[str, Any]] = [ + { + "test_name": "hunyuan_image_dit_reliability_default", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "stage_config_name": "hunyuan_image3_dit.yaml", + }, + }, +] + +DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy" +OOM_INJECTION_CONFIG = { + "target_mem_ratio": 0.95, + "hold_seconds": 0, + "startup_timeout_sec": 20, + "strict": True, +} +FAULT_ERROR_KEYWORDS = ( + "oom", + "out of memory", + "cuda", + "job failed", + "unknown error", + "internal server error", + "500 server error", +) +PROCESS_KILL_ERROR_KEYWORDS = ( + "timeout", + "did not complete within", + "connection", + "engine", + "orchestrator", + "dead", + "internal", + "500", + "503", +) +# Matrix: worker SIGTERM/SIGKILL; DiT-only HunyuanImage workers are vLLM-Omni::. +WORKER_KILL_PATTERNS = ("vLLM-Omni::",) +SERVE_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGINT", id="sigint"), + pytest.param("SIGKILL", id="sigkill"), +] +TREE_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGKILL", id="sigkill"), +] +TREE_WITH_LOAD_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGKILL", id="sigkill"), +] +WORKER_SIGNAL_FAULT_PARAMS = [ + pytest.param( + make_process_kill_fault_injector( + grep_patterns=WORKER_KILL_PATTERNS, + signal_name="SIGTERM", + limit=1, + post_kill_wait_seconds=2.0, + ), + id="runtime_process_chain_sigterm", + ), + pytest.param( + make_process_kill_fault_injector( + grep_patterns=WORKER_KILL_PATTERNS, + signal_name="SIGKILL", + limit=1, + post_kill_wait_seconds=2.0, + ), + id="runtime_process_chain_sigkill", + ), +] + +HUNYUAN_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR) +INFLIGHT_INJECTION_REQUEST_RATE = 0.3 +INFLIGHT_INJECTION_REQUEST_COUNT = 10 + + +def _image_request_config(omni_server: Any) -> dict[str, Any]: + return { + "json": { + "model": omni_server.model, + "prompt": "A simple red apple on a white background.", + "size": "512x512", + "n": 1, + "response_format": "b64_json", + "num_inference_steps": 4, + "guidance_scale": 7.5, + "seed": 42, + "bot_task": "none", + "use_system_prompt": "en_unified", + }, + "timeout": 600, + } + + +def _submit_image_generation(openai_client: OpenAIClientHandler, omni_server: Any) -> None: + responses = openai_client.send_images_generations_http_request(_image_request_config(omni_server)) + resp = responses[0] + if resp.status_code >= 400: + raise RuntimeError( + f"/v1/images/generations failed: status={resp.status_code}, body={getattr(resp, 'error_message', '')!r}" + ) + + +def _assert_post_fault_image_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None: + payload = { + "model": model, + "prompt": "post-fault fast-fail check", + "size": "512x512", + "n": 1, + "response_format": "b64_json", + "num_inference_steps": 4, + "bot_task": "none", + "use_system_prompt": "en_unified", + } + start = time.monotonic() + try: + status, body = post_json_raw(host, port, "/v1/images/generations", payload, timeout_sec=20) + elapsed = time.monotonic() - start + assert elapsed < 15, f"[{scenario} fast_fail] /v1/images/generations did not fail fast: {elapsed:.2f}s" + assert status >= 500, ( + f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}" + ) + except Exception: # noqa: BLE001 + elapsed = time.monotonic() - start + assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s" + + +def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None: + deadline = time.monotonic() + 20.0 + last_observation = "" + while time.monotonic() < deadline: + try: + status, body = get_health_raw(host, port, timeout_sec=5) + last_observation = f"http={status}, body={body[:200]!r}" + if status == 503: + return + except Exception as exc: # noqa: BLE001 + last_observation = f"exception={exc!r}" + return + time.sleep(0.5) + pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}") + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.skip(reason="issue#4285") +@hardware_test(res={"cuda": "H100"}, num_cards=4) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_gpu_oom_image_large_request_failure( + omni_server_function, + openai_client_function, +) -> None: + stage_config_path = getattr(omni_server_function, "stage_config_path", None) + device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) + handle = inject_gpu_oom( + device=device_spec, + target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], + hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], + startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], + strict=OOM_INJECTION_CONFIG["strict"], + ) + try: + try: + _submit_image_generation(openai_client_function, omni_server_function) + except Exception as exc: + assert_fault_exception(exc, FAULT_ERROR_KEYWORDS) + else: + pytest.fail("expected large /v1/images/generations request failure during GPU OOM injection") + finally: + stop_gpu_oom_hogs(handle) + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize( + "fault_injector", + WORKER_SIGNAL_FAULT_PARAMS, + indirect=True, +) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_request_failure( + omni_server_after_fault_function, + openai_client_function, +) -> None: + try: + _submit_image_generation(openai_client_function, omni_server_after_fault_function) + except Exception as exc: + assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) + else: + pytest.fail("expected /v1/images/generations request failure after process-kill injection") + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.skip(reason="issue#4522") +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize( + "fault_injector", + WORKER_SIGNAL_FAULT_PARAMS, + indirect=True, +) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_health_fast_fail_and_concurrent( + omni_server_after_fault_function, + openai_client_function, +) -> None: + host = omni_server_after_fault_function.host + port = omni_server_after_fault_function.port + model = omni_server_after_fault_function.model + + payload = { + "model": model, + "prompt": "fast-fail check", + "size": "512x512", + "n": 1, + "response_format": "b64_json", + "num_inference_steps": 4, + "bot_task": "none", + "use_system_prompt": "en_unified", + } + start = time.monotonic() + try: + status, body = post_json_raw(host, port, "/v1/images/generations", payload, timeout_sec=20) + elapsed = time.monotonic() - start + assert elapsed < 15, ( + f"[process_kill fast_fail] /v1/images/generations did not fail fast after fault: {elapsed:.2f}s" + ) + assert status >= 500, ( + "[process_kill fast_fail] expected server-side error after fatal fault, " + f"got status={status}, body={body[:200]!r}" + ) + except Exception: + elapsed = time.monotonic() - start + assert elapsed < 15, ( + f"[process_kill fast_fail] /v1/images/generations exception was too slow after fault: {elapsed:.2f}s" + ) + + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + futures = [ + executor.submit(_submit_image_generation, openai_client_function, omni_server_after_fault_function) + for _ in range(3) + ] + done, pending = concurrent.futures.wait( + futures, + timeout=40, + return_when=concurrent.futures.ALL_COMPLETED, + ) + + elapsed = time.monotonic() - start + assert not pending, f"[process_kill concurrent] some fault-time image requests hung: pending={len(pending)}" + assert elapsed < 40, f"[process_kill concurrent] fault-time image request convergence is too slow: {elapsed:.2f}s" + + fault_observed = False + for future in done: + try: + future.result() + except Exception as exc: + fault_observed = True + assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) + assert fault_observed, ( + "[process_kill concurrent] expected at least one /v1/images/generations request to fail after fault" + ) + + deadline = time.monotonic() + 20.0 + last_observation = "" + final_health_status: int | None = None + while time.monotonic() < deadline: + try: + status, body = get_health_raw(host, port, timeout_sec=5) + last_observation = f"http={status}, body={body[:200]!r}" + final_health_status = status + if status == 503: + break + except Exception as exc: # noqa: BLE001 + last_observation = f"exception={exc!r}" + time.sleep(0.5) + assert final_health_status == 503, ( + "[process_kill health] expected /health 503 after fatal fault, " + f"got status={final_health_status}, last_observation={last_observation}" + ) + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.skip(reason="issue#4526") +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize( + "fault_injector", + WORKER_SIGNAL_FAULT_PARAMS, + indirect=True, +) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_worker_with_load_request_failure( + omni_server_function, + openai_client_function, + fault_injector: FaultInjector, +) -> None: + scenario = "kill_worker_with_load" + load_result = run_fault_injection_with_rate_load( + submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function), + inject_fault=lambda: fault_injector(omni_server_function), + num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, + request_rate=INFLIGHT_INJECTION_REQUEST_RATE, + completion_timeout_sec=120.0, + ) + assert load_result["failure_observed"], ( + f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" + ) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + + +@pytest.mark.slow +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}, num_cards=4) +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup( + omni_server_function, + openai_client_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_root_with_load_{signal_name.lower()}" + injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) + load_result = run_fault_injection_with_rate_load( + submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function), + inject_fault=lambda: injector(omni_server_function), + num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, + request_rate=INFLIGHT_INJECTION_REQUEST_RATE, + completion_timeout_sec=120.0, + ) + assert load_result["failure_observed"], ( + f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" + ) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) + + +@pytest.mark.slow +@pytest.mark.diffusion +@hardware_test(res={"cuda": "H100"}, num_cards=4) +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup( + omni_server_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_root_no_load_{signal_name.lower()}" + injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) + injector(omni_server_function) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup( + omni_server_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_tree_no_load_{signal_name.lower()}" + injector = make_server_tree_kill_fault_injector( + signal_name=signal_name, + post_kill_wait_seconds=2.0, + inter_kill_wait_seconds=0.1, + ) + injector(omni_server_function) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) + + +@pytest.mark.slow +@pytest.mark.diffusion +@pytest.mark.skip(reason="issue#4526") +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True) +def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup( + omni_server_function, + openai_client_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_tree_with_load_{signal_name.lower()}" + injector = make_server_tree_kill_fault_injector( + signal_name=signal_name, + post_kill_wait_seconds=2.0, + inter_kill_wait_seconds=0.1, + ) + load_result = run_fault_injection_with_rate_load( + submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function), + inject_fault=lambda: injector(omni_server_function), + num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, + request_rate=INFLIGHT_INJECTION_REQUEST_RATE, + completion_timeout_sec=120.0, + ) + assert load_result["failure_observed"], ( + f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" + ) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) diff --git a/tests/dfx/reliability/test_reliability_voxcpm2.py b/tests/dfx/reliability/test_reliability_voxcpm2.py new file mode 100644 index 000000000..014bb4e89 --- /dev/null +++ b/tests/dfx/reliability/test_reliability_voxcpm2.py @@ -0,0 +1,505 @@ +""" +VoxCPM2 reliability integration tests. + +Scenarios follow ``docs/user_guide/fault_injection_reliability_matrix.md``. +""" + +from __future__ import annotations + +import concurrent.futures +import os +import time +from pathlib import Path +from typing import Any + +import pytest + +from tests.dfx.conftest import ( + assert_fault_exception, + create_reliability_omni_server_params, + resolve_oom_device_spec, +) +from tests.dfx.reliability.helpers import ( + FaultInjector, + assert_no_server_tree_process_residual_and_gpu_release, + extract_openai_error_contract_from_bytes, + get_health_raw, + inject_gpu_oom, + make_process_kill_fault_injector, + make_server_root_kill_fault_injector, + make_server_tree_kill_fault_injector, + post_json_raw, + run_fault_injection_with_rate_load, + stop_gpu_oom_hogs, + worker_residual_timeout_after_kill_signal, +) +from tests.helpers.mark import hardware_test +from tests.helpers.media import load_test_audio_data_url +from tests.helpers.runtime import OpenAIClientHandler + +RELIABILITY_SCENARIOS: list[dict[str, Any]] = [ + { + "test_name": "voxcpm2_reliability_default", + "server_params": { + "model": "openbmb/VoxCPM2", + "stage_config_name": "voxcpm2.yaml", + }, + }, +] + +DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy" +OOM_INJECTION_CONFIG = { + "target_mem_ratio": 0.96, + "hold_seconds": 0, + "startup_timeout_sec": 20, + "strict": False, +} +FAULT_ERROR_KEYWORDS = ( + "the request failed", + "oom", + "out of memory", + "cuda", + "orchestrator", + "timeout", + "connection", + "500", + "503", +) +PROCESS_KILL_ERROR_KEYWORDS = ( + "timeout", + "did not complete within", + "connection", + "engine", + "orchestrator", + "dead", + "internal", + "500", + "503", +) +RUNTIME_WORKER_PATTERN = "VLLM::" +SERVE_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGINT", id="sigint"), + pytest.param("SIGKILL", id="sigkill"), +] +SERVE_WITH_LOAD_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGINT", id="sigint"), + pytest.param("SIGKILL", id="sigkill"), +] +TREE_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGKILL", id="sigkill"), +] +TREE_WITH_LOAD_SIGNAL_PARAMS = [ + pytest.param("SIGTERM", id="sigterm"), + pytest.param("SIGKILL", id="sigkill"), +] +WORKER_SIGNAL_FAULT_PARAMS = [ + pytest.param( + make_process_kill_fault_injector( + grep_patterns=RUNTIME_WORKER_PATTERN, + signal_name="SIGTERM", + limit=1, + post_kill_wait_seconds=2.0, + ), + id="runtime_process_chain_sigterm", + ), + pytest.param( + make_process_kill_fault_injector( + grep_patterns=RUNTIME_WORKER_PATTERN, + signal_name="SIGKILL", + limit=1, + post_kill_wait_seconds=2.0, + ), + id="runtime_process_chain_sigkill", + ), +] + +VOXCPM2_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR) +INFLIGHT_INJECTION_REQUEST_RATE = 0.3 +INFLIGHT_INJECTION_REQUEST_COUNT = 10 +_SPEECH_FF_HTTP_TIMEOUT_SEC = 30 +_SPEECH_FF_MAX_ELAPSED_SEC = 30 + + +def _speech_request_config(omni_server: Any) -> dict[str, Any]: + return { + "model": omni_server.model, + "input": "The weather is nice today, perfect for a walk in the park.", + "stream": False, + "timeout": 120.0, + "response_format": "wav", + "voice": "default", + } + + +def _submit_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None: + openai_client.send_audio_speech_request(_speech_request_config(omni_server), request_num=1) + + +def _oom_speech_request_config(omni_server: Any) -> dict[str, Any]: + base = "The weather is nice today, perfect for a walk in the park. " + return { + "model": omni_server.model, + "input": base * 80, + "max_new_tokens": 4096, + "ref_audio": load_test_audio_data_url("qwen3_tts/clone_2.wav"), + "stream": False, + "timeout": 300.0, + "response_format": "wav", + "voice": "default", + } + + +def _submit_oom_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None: + openai_client.send_audio_speech_request(_oom_speech_request_config(omni_server), request_num=1) + + +def _assert_post_fault_speech_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None: + payload = { + "model": model, + "input": "post-fault fast-fail check", + "voice": "default", + } + start = time.monotonic() + try: + status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=20) + elapsed = time.monotonic() - start + assert elapsed < 15, f"[{scenario} fast_fail] /v1/audio/speech did not fail fast: {elapsed:.2f}s" + assert status >= 500, ( + f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}" + ) + except Exception: # noqa: BLE001 + elapsed = time.monotonic() - start + assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s" + + +def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None: + deadline = time.monotonic() + 20.0 + last_observation = "" + while time.monotonic() < deadline: + try: + status, body = get_health_raw(host, port, timeout_sec=5) + last_observation = f"http={status}, body={body[:200]!r}" + if status == 503: + return + except Exception as exc: # noqa: BLE001 + last_observation = f"exception={exc!r}" + return + time.sleep(0.5) + pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}") + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skip(reason="issue#4285") +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_gpu_oom_speech_request_failure( + omni_server_function, + openai_client_function, +) -> None: + stage_config_path = getattr(omni_server_function, "stage_config_path", None) + device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) + handle = inject_gpu_oom( + device=device_spec, + target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], + hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], + startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], + strict=OOM_INJECTION_CONFIG["strict"], + ) + try: + try: + _submit_oom_speech_request(openai_client_function, omni_server_function) + except Exception as exc: + assert_fault_exception(exc, FAULT_ERROR_KEYWORDS) + else: + pytest.fail("expected /v1/audio/speech request failure during GPU OOM injection") + finally: + stop_gpu_oom_hogs(handle) + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skip(reason="issue#4285") +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_gpu_oom_speech_error_contract( + omni_server_function, +) -> None: + stage_config_path = getattr(omni_server_function, "stage_config_path", None) + device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path) + handle = inject_gpu_oom( + device=device_spec, + target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"], + hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"], + startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"], + strict=OOM_INJECTION_CONFIG["strict"], + ) + host = omni_server_function.host + port = omni_server_function.port + oom_config = _oom_speech_request_config(omni_server_function) + payload = { + "model": oom_config["model"], + "input": oom_config["input"], + "voice": oom_config["voice"], + "max_new_tokens": oom_config["max_new_tokens"], + "ref_audio": oom_config["ref_audio"], + } + try: + status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=300) + finally: + stop_gpu_oom_hogs(handle) + + assert status >= 500, f"expected speech runtime error under OOM, got status={status}" + speech_error = extract_openai_error_contract_from_bytes(body) + assert speech_error is not None, f"speech error payload not OpenAI-compatible: {body[:300]!r}" + assert "code" in speech_error, f"speech error lacks code field: {speech_error!r}" + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize( + "fault_injector", + WORKER_SIGNAL_FAULT_PARAMS, + indirect=True, +) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_request_failure( + omni_server_after_fault_function, + openai_client_function, +) -> None: + try: + _submit_speech_request(openai_client_function, omni_server_after_fault_function) + except Exception as exc: + assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) + else: + pytest.fail("expected /v1/audio/speech request failure after process-kill injection") + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize( + "fault_injector", + WORKER_SIGNAL_FAULT_PARAMS, + indirect=True, +) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_fast_fail_and_concurrent( + omni_server_after_fault_function, + openai_client_function, +) -> None: + host = omni_server_after_fault_function.host + port = omni_server_after_fault_function.port + model = omni_server_after_fault_function.model + payload = { + "model": model, + "input": "Say hello in one short sentence.", + "voice": "default", + } + + start = time.monotonic() + try: + status, body = post_json_raw( + host, + port, + "/v1/audio/speech", + payload, + timeout_sec=_SPEECH_FF_HTTP_TIMEOUT_SEC, + ) + elapsed = time.monotonic() - start + assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, ( + f"[process_kill fast_fail] /v1/audio/speech did not fail fast after fault: {elapsed:.2f}s" + ) + assert status >= 500, ( + "[process_kill fast_fail] expected server-side failure after fault, " + f"got status={status}, body={body[:200]!r}" + ) + except Exception: + elapsed = time.monotonic() - start + assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, ( + f"[process_kill fast_fail] /v1/audio/speech exception was too slow after fault: {elapsed:.2f}s" + ) + + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + futures = [ + executor.submit(_submit_speech_request, openai_client_function, omni_server_after_fault_function) + for _ in range(3) + ] + done, pending = concurrent.futures.wait( + futures, + timeout=30, + return_when=concurrent.futures.ALL_COMPLETED, + ) + + elapsed = time.monotonic() - start + assert not pending, f"[process_kill concurrent] some fault-time speech requests hung: pending={len(pending)}" + assert elapsed < 30, f"[process_kill concurrent] fault-time speech request convergence is too slow: {elapsed:.2f}s" + + fault_observed = False + for future in done: + try: + future.result() + except Exception as exc: + fault_observed = True + assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS) + assert fault_observed, ( + "[process_kill concurrent] expected at least one /v1/audio/speech request to fail after fault" + ) + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize( + "fault_injector", + WORKER_SIGNAL_FAULT_PARAMS, + indirect=True, +) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_worker_with_load_request_failure( + omni_server_function, + openai_client_function, + fault_injector: FaultInjector, +) -> None: + scenario = "kill_worker_with_load" + load_result = run_fault_injection_with_rate_load( + submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), + inject_fault=lambda: fault_injector(omni_server_function), + num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, + request_rate=INFLIGHT_INJECTION_REQUEST_RATE, + completion_timeout_sec=120.0, + ) + assert load_result["failure_observed"], ( + f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" + ) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + + +@pytest.mark.slow +@pytest.mark.tts +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup( + omni_server_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_root_no_load_{signal_name.lower()}" + injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) + injector(omni_server_function) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) + + +@pytest.mark.slow +@pytest.mark.tts +@hardware_test(res={"cuda": "L4"}, num_cards=1) +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", SERVE_WITH_LOAD_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup( + omni_server_function, + openai_client_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_root_with_load_{signal_name.lower()}" + injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0) + load_result = run_fault_injection_with_rate_load( + submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), + inject_fault=lambda: injector(omni_server_function), + num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, + request_rate=INFLIGHT_INJECTION_REQUEST_RATE, + completion_timeout_sec=300.0, + ) + assert load_result["failure_observed"], ( + f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" + ) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup( + omni_server_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_tree_no_load_{signal_name.lower()}" + injector = make_server_tree_kill_fault_injector( + signal_name=signal_name, + post_kill_wait_seconds=2.0, + inter_kill_wait_seconds=0.1, + ) + injector(omni_server_function) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only") +@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS) +@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True) +def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup( + omni_server_function, + openai_client_function, + signal_name: str, +) -> None: + scenario = f"kill_serve_tree_with_load_{signal_name.lower()}" + injector = make_server_tree_kill_fault_injector( + signal_name=signal_name, + post_kill_wait_seconds=2.0, + inter_kill_wait_seconds=0.1, + ) + load_result = run_fault_injection_with_rate_load( + submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function), + inject_fault=lambda: injector(omni_server_function), + num_requests=INFLIGHT_INJECTION_REQUEST_COUNT, + request_rate=INFLIGHT_INJECTION_REQUEST_RATE, + completion_timeout_sec=120.0, + ) + assert load_result["failure_observed"], ( + f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}" + ) + host = omni_server_function.host + port = omni_server_function.port + _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario) + _assert_post_fault_health_terminal(host, port, scenario=scenario) + assert_no_server_tree_process_residual_and_gpu_release( + omni_server_function, + scenario=scenario, + timeout_sec=worker_residual_timeout_after_kill_signal(signal_name), + ) diff --git a/tests/dfx/stability/scripts/test_stability_voxcpm2.py b/tests/dfx/stability/scripts/test_stability_voxcpm2.py new file mode 100644 index 000000000..e26a2050c --- /dev/null +++ b/tests/dfx/stability/scripts/test_stability_voxcpm2.py @@ -0,0 +1,69 @@ +""" +VoxCPM2 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. + +Configuration: ``tests/dfx/stability/tests/test_voxcpm2.json``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_voxcpm2.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"] + +try: + BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +@pytest.mark.slow +@pytest.mark.tts +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_voxcpm2(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_vllm_bench_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/tests/test_voxcpm2.json b/tests/dfx/stability/tests/test_voxcpm2.json new file mode 100644 index 000000000..dd912da36 --- /dev/null +++ b/tests/dfx/stability/tests/test_voxcpm2.json @@ -0,0 +1,35 @@ +[ + { + "test_name": "test_voxcpm2_stability", + "server_params": { + "model": "openbmb/VoxCPM2", + "trust_remote_code": true + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "trust_remote_code": true, + "duration_sec": 86400, + "max_concurrency": 2, + "num_prompts_per_batch": 50, + "random_input_len": { + "min": 0, + "max": 1000 + }, + "random_output_len": { + "min": 0, + "max": 1000 + }, + "random_range_ratio": 0.0, + "extra_body": { + "voice": "default", + "language": "English" + }, + "ignore_eos": true, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration" + } + ] + } +]