diff --git a/.buildkite/test-weekly.yml b/.buildkite/test-weekly.yml
index 57eadeafd..aefb424e5 100644
--- a/.buildkite/test-weekly.yml
+++ b/.buildkite/test-weekly.yml
@@ -85,6 +85,66 @@ steps:
path: /mnt/hf-cache
type: DirectoryOrCreate
+ - label: "Reliability Test - hunyuan-image"
+ timeout_in_minutes: 180
+ depends_on: upload-weekly-pipeline
+ if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+ commands:
+ - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
+ agents:
+ queue: "mithril-h100-pool"
+ plugins:
+ - kubernetes:
+ podSpec:
+ containers:
+ - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+ resources:
+ limits:
+ nvidia.com/gpu: 4
+ volumeMounts:
+ - name: devshm
+ mountPath: /dev/shm
+ - name: hf-cache
+ mountPath: /root/.cache/huggingface
+ env:
+ - name: HF_HOME
+ value: /root/.cache/huggingface
+ - name: HF_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: hf-token-secret
+ key: token
+ nodeSelector:
+ node.kubernetes.io/instance-type: gpu-h100-sxm
+ volumes:
+ - name: devshm
+ emptyDir:
+ medium: Memory
+ - name: hf-cache
+ hostPath:
+ path: /mnt/hf-cache
+ type: DirectoryOrCreate
+
+ - label: "Reliability Test - voxcpm2"
+ timeout_in_minutes: 180
+ depends_on: upload-weekly-pipeline
+ if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+ commands:
+ - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
+ agents:
+ queue: "gpu_1_queue"
+ plugins:
+ - docker#v5.2.0:
+ image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+ always-pull: true
+ propagate-environment: true
+ shm-size: "8gb"
+ environment:
+ - "HF_HOME=/fsx/hf_cache"
+ - "HF_TOKEN"
+ volumes:
+ - "/fsx/hf_cache:/fsx/hf_cache"
+
# Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
- group: "Reliability Test - Invalid parameters Test"
depends_on: upload-weekly-pipeline
diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md
index adf29bfa5..c27f1fc9a 100644
--- a/docs/contributing/ci/CI_5levels.md
+++ b/docs/contributing/ci/CI_5levels.md
@@ -155,7 +155,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
/tests/dfx/stability/tests/test_wan22.json
Reliability:
tests/dfx/reliability/test_reliability_{model_key}.py
- (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py)
+ (e.g. test_reliability_qwen3_omni.py, test_reliability_wan22.py, test_reliability_hunyuan_image.py, test_reliability_voxcpm2.py)
Chapter 4
@@ -741,14 +741,16 @@ L5 level testing focuses on the performance of model services under ***long-runn
### 4.2 Testing Content and Scope
- ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
-- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni or `multiprocessing.spawn` for Wan2.2 video workers), **large multimodal chat** or **`/v1/videos`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
+- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
### 4.3 Test Directory and Execution Files
- ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family)
- ***Reliability Test Suite*** (`tests/dfx/reliability/`):
- `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default).
- - `test_reliability_wan22.py` — Wan2.2 T2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
+ - `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
+ - `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4).
+ - `test_reliability_voxcpm2.py` — VoxCPM2 TTS reliability (`/v1/audio/speech` under OOM and process kill; deploy `voxcpm2.yaml`, L4 × 1).
- `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`).
- `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready.
- `README.md` — Short local run commands for this directory.
@@ -846,22 +848,33 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in
| `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). |
| `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). |
| `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). |
-| `test_reliability_wan22.py` | Wan2.2 T2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
+| `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
+| `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. |
+| `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. |
| `README.md` | Minimal run / collect examples. |
#### Parametrization and markers
- Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`.
-- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni paths often require **2** cards; Wan2.2 video paths **1** card).
+- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4).
- **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**.
#### CI trigger
-Weekly Buildkite (`.buildkite/test-weekly.yml`) runs, for example:
+Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example:
+
+| Buildkite step | Test file | CI hardware |
+| -------------- | --------- | ----------- |
+| Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) |
+| Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) |
+| Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) |
+| Reliability Test - voxcpm2 | `test_reliability_voxcpm2.py` | L4 × 1 (`gpu_1_queue`) |
```bash
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
+pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
+pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
```
#### Local commands
@@ -870,6 +883,8 @@ pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
pytest --collect-only tests/dfx/reliability
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
+pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow
+pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
```
#### Adding a new model suite
@@ -882,7 +897,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
- - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config)
- - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow` and/or `pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow` (add `test_reliability_.py` for new models)
+ - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_.py` and a matching step in `.buildkite/test-weekly.yml` for new models)
## Summary
diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md
index 36dae4451..0db814e08 100644
--- a/docs/contributing/ci/test_guide.md
+++ b/docs/contributing/ci/test_guide.md
@@ -96,15 +96,21 @@ Our test scripts use the pytest framework. First, please use `git clone https://
# Stability: Wan2.2 (v1/videos diffusion benchmark loop)
pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
- # Reliability: Qwen3-Omni
+ # Reliability: Qwen3-Omni (H100 × 2)
pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow
- # Reliability: Wan2.2
+ # Reliability: Wan2.2 (H100)
pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow
+ # Reliability: HunyuanImage DiT (H100 × 4)
+ pytest -s -v dfx/reliability/test_reliability_hunyuan_image.py -m slow
+
+ # Reliability: VoxCPM2 (L4)
+ pytest -s -v dfx/reliability/test_reliability_voxcpm2.py -m slow
+
```
- The latest L5 commands for CI can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml).
+ The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml).
You can find more information about markers in the documentation: [marker doc](./tests_markers.md)
diff --git a/docs/user_guide/fault_injection_reliability_matrix.md b/docs/user_guide/fault_injection_reliability_matrix.md
index 2b5fd2401..974982b3b 100644
--- a/docs/user_guide/fault_injection_reliability_matrix.md
+++ b/docs/user_guide/fault_injection_reliability_matrix.md
@@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b
## Source of Conclusions
-The behaviors and conclusions above are summarized from current fault injection validation results on `Qwen3-Omni` and `Wan2.2`.
+The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`.
### Example: `SIGTERM` fault injection log (Qwen3-Omni)
diff --git a/tests/dfx/reliability/README.md b/tests/dfx/reliability/README.md
index 7196101bc..8dc69dc55 100644
--- a/tests/dfx/reliability/README.md
+++ b/tests/dfx/reliability/README.md
@@ -6,9 +6,13 @@ The detailed RFC document is maintained outside this repository (local/internal
- Fault helpers: `conftest.py`, `helpers.py`
- Qwen3 tests: `test_reliability_qwen3_omni.py`
- Wan2.2 tests: `test_reliability_wan22.py`
+- HunyuanImage tests: `test_reliability_hunyuan_image.py`
+- VoxCPM2 tests: `test_reliability_voxcpm2.py`
```bash
pytest --collect-only tests/dfx/reliability
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
+pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow
+pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
```
diff --git a/tests/dfx/reliability/test_reliability_hunyuan_image.py b/tests/dfx/reliability/test_reliability_hunyuan_image.py
new file mode 100644
index 000000000..43a684d1d
--- /dev/null
+++ b/tests/dfx/reliability/test_reliability_hunyuan_image.py
@@ -0,0 +1,478 @@
+"""
+HunyuanImage-3.0-Instruct reliability integration tests (DiT-only deploy).
+
+Uses ``hunyuan_image3_dit.yaml`` (single diffusion stage, 4-GPU TP=4, no
+AR/Mooncake connector) for simpler local bring-up. Scenarios follow
+``docs/user_guide/fault_injection_reliability_matrix.md``.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from tests.dfx.conftest import (
+ assert_fault_exception,
+ create_reliability_omni_server_params,
+ resolve_oom_device_spec,
+)
+from tests.dfx.reliability.helpers import (
+ FaultInjector,
+ assert_no_server_tree_process_residual_and_gpu_release,
+ get_health_raw,
+ inject_gpu_oom,
+ make_process_kill_fault_injector,
+ make_server_root_kill_fault_injector,
+ make_server_tree_kill_fault_injector,
+ post_json_raw,
+ run_fault_injection_with_rate_load,
+ stop_gpu_oom_hogs,
+ worker_residual_timeout_after_kill_signal,
+)
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OpenAIClientHandler
+
+RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
+ {
+ "test_name": "hunyuan_image_dit_reliability_default",
+ "server_params": {
+ "model": "tencent/HunyuanImage-3.0-Instruct",
+ "stage_config_name": "hunyuan_image3_dit.yaml",
+ },
+ },
+]
+
+DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy"
+OOM_INJECTION_CONFIG = {
+ "target_mem_ratio": 0.95,
+ "hold_seconds": 0,
+ "startup_timeout_sec": 20,
+ "strict": True,
+}
+FAULT_ERROR_KEYWORDS = (
+ "oom",
+ "out of memory",
+ "cuda",
+ "job failed",
+ "unknown error",
+ "internal server error",
+ "500 server error",
+)
+PROCESS_KILL_ERROR_KEYWORDS = (
+ "timeout",
+ "did not complete within",
+ "connection",
+ "engine",
+ "orchestrator",
+ "dead",
+ "internal",
+ "500",
+ "503",
+)
+# Matrix: worker SIGTERM/SIGKILL; DiT-only HunyuanImage workers are vLLM-Omni::.
+WORKER_KILL_PATTERNS = ("vLLM-Omni::",)
+SERVE_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGINT", id="sigint"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+TREE_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+TREE_WITH_LOAD_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+WORKER_SIGNAL_FAULT_PARAMS = [
+ pytest.param(
+ make_process_kill_fault_injector(
+ grep_patterns=WORKER_KILL_PATTERNS,
+ signal_name="SIGTERM",
+ limit=1,
+ post_kill_wait_seconds=2.0,
+ ),
+ id="runtime_process_chain_sigterm",
+ ),
+ pytest.param(
+ make_process_kill_fault_injector(
+ grep_patterns=WORKER_KILL_PATTERNS,
+ signal_name="SIGKILL",
+ limit=1,
+ post_kill_wait_seconds=2.0,
+ ),
+ id="runtime_process_chain_sigkill",
+ ),
+]
+
+HUNYUAN_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR)
+INFLIGHT_INJECTION_REQUEST_RATE = 0.3
+INFLIGHT_INJECTION_REQUEST_COUNT = 10
+
+
+def _image_request_config(omni_server: Any) -> dict[str, Any]:
+ return {
+ "json": {
+ "model": omni_server.model,
+ "prompt": "A simple red apple on a white background.",
+ "size": "512x512",
+ "n": 1,
+ "response_format": "b64_json",
+ "num_inference_steps": 4,
+ "guidance_scale": 7.5,
+ "seed": 42,
+ "bot_task": "none",
+ "use_system_prompt": "en_unified",
+ },
+ "timeout": 600,
+ }
+
+
+def _submit_image_generation(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
+ responses = openai_client.send_images_generations_http_request(_image_request_config(omni_server))
+ resp = responses[0]
+ if resp.status_code >= 400:
+ raise RuntimeError(
+ f"/v1/images/generations failed: status={resp.status_code}, body={getattr(resp, 'error_message', '')!r}"
+ )
+
+
+def _assert_post_fault_image_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None:
+ payload = {
+ "model": model,
+ "prompt": "post-fault fast-fail check",
+ "size": "512x512",
+ "n": 1,
+ "response_format": "b64_json",
+ "num_inference_steps": 4,
+ "bot_task": "none",
+ "use_system_prompt": "en_unified",
+ }
+ start = time.monotonic()
+ try:
+ status, body = post_json_raw(host, port, "/v1/images/generations", payload, timeout_sec=20)
+ elapsed = time.monotonic() - start
+ assert elapsed < 15, f"[{scenario} fast_fail] /v1/images/generations did not fail fast: {elapsed:.2f}s"
+ assert status >= 500, (
+ f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}"
+ )
+ except Exception: # noqa: BLE001
+ elapsed = time.monotonic() - start
+ assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s"
+
+
+def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None:
+ deadline = time.monotonic() + 20.0
+ last_observation = ""
+ while time.monotonic() < deadline:
+ try:
+ status, body = get_health_raw(host, port, timeout_sec=5)
+ last_observation = f"http={status}, body={body[:200]!r}"
+ if status == 503:
+ return
+ except Exception as exc: # noqa: BLE001
+ last_observation = f"exception={exc!r}"
+ return
+ time.sleep(0.5)
+ pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}")
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@pytest.mark.skip(reason="issue#4285")
+@hardware_test(res={"cuda": "H100"}, num_cards=4)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_image_large_request_failure(
+ omni_server_function,
+ openai_client_function,
+) -> None:
+ stage_config_path = getattr(omni_server_function, "stage_config_path", None)
+ device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
+ handle = inject_gpu_oom(
+ device=device_spec,
+ target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+ hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+ startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+ strict=OOM_INJECTION_CONFIG["strict"],
+ )
+ try:
+ try:
+ _submit_image_generation(openai_client_function, omni_server_function)
+ except Exception as exc:
+ assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+ else:
+ pytest.fail("expected large /v1/images/generations request failure during GPU OOM injection")
+ finally:
+ stop_gpu_oom_hogs(handle)
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+ "fault_injector",
+ WORKER_SIGNAL_FAULT_PARAMS,
+ indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_request_failure(
+ omni_server_after_fault_function,
+ openai_client_function,
+) -> None:
+ try:
+ _submit_image_generation(openai_client_function, omni_server_after_fault_function)
+ except Exception as exc:
+ assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
+ else:
+ pytest.fail("expected /v1/images/generations request failure after process-kill injection")
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@pytest.mark.skip(reason="issue#4522")
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+ "fault_injector",
+ WORKER_SIGNAL_FAULT_PARAMS,
+ indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
+ omni_server_after_fault_function,
+ openai_client_function,
+) -> None:
+ host = omni_server_after_fault_function.host
+ port = omni_server_after_fault_function.port
+ model = omni_server_after_fault_function.model
+
+ payload = {
+ "model": model,
+ "prompt": "fast-fail check",
+ "size": "512x512",
+ "n": 1,
+ "response_format": "b64_json",
+ "num_inference_steps": 4,
+ "bot_task": "none",
+ "use_system_prompt": "en_unified",
+ }
+ start = time.monotonic()
+ try:
+ status, body = post_json_raw(host, port, "/v1/images/generations", payload, timeout_sec=20)
+ elapsed = time.monotonic() - start
+ assert elapsed < 15, (
+ f"[process_kill fast_fail] /v1/images/generations did not fail fast after fault: {elapsed:.2f}s"
+ )
+ assert status >= 500, (
+ "[process_kill fast_fail] expected server-side error after fatal fault, "
+ f"got status={status}, body={body[:200]!r}"
+ )
+ except Exception:
+ elapsed = time.monotonic() - start
+ assert elapsed < 15, (
+ f"[process_kill fast_fail] /v1/images/generations exception was too slow after fault: {elapsed:.2f}s"
+ )
+
+ start = time.monotonic()
+ with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+ futures = [
+ executor.submit(_submit_image_generation, openai_client_function, omni_server_after_fault_function)
+ for _ in range(3)
+ ]
+ done, pending = concurrent.futures.wait(
+ futures,
+ timeout=40,
+ return_when=concurrent.futures.ALL_COMPLETED,
+ )
+
+ elapsed = time.monotonic() - start
+ assert not pending, f"[process_kill concurrent] some fault-time image requests hung: pending={len(pending)}"
+ assert elapsed < 40, f"[process_kill concurrent] fault-time image request convergence is too slow: {elapsed:.2f}s"
+
+ fault_observed = False
+ for future in done:
+ try:
+ future.result()
+ except Exception as exc:
+ fault_observed = True
+ assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
+ assert fault_observed, (
+ "[process_kill concurrent] expected at least one /v1/images/generations request to fail after fault"
+ )
+
+ deadline = time.monotonic() + 20.0
+ last_observation = ""
+ final_health_status: int | None = None
+ while time.monotonic() < deadline:
+ try:
+ status, body = get_health_raw(host, port, timeout_sec=5)
+ last_observation = f"http={status}, body={body[:200]!r}"
+ final_health_status = status
+ if status == 503:
+ break
+ except Exception as exc: # noqa: BLE001
+ last_observation = f"exception={exc!r}"
+ time.sleep(0.5)
+ assert final_health_status == 503, (
+ "[process_kill health] expected /health 503 after fatal fault, "
+ f"got status={final_health_status}, last_observation={last_observation}"
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@pytest.mark.skip(reason="issue#4526")
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+ "fault_injector",
+ WORKER_SIGNAL_FAULT_PARAMS,
+ indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_worker_with_load_request_failure(
+ omni_server_function,
+ openai_client_function,
+ fault_injector: FaultInjector,
+) -> None:
+ scenario = "kill_worker_with_load"
+ load_result = run_fault_injection_with_rate_load(
+ submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
+ inject_fault=lambda: fault_injector(omni_server_function),
+ num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
+ request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+ completion_timeout_sec=120.0,
+ )
+ assert load_result["failure_observed"], (
+ f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards=4)
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
+ omni_server_function,
+ openai_client_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
+ injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
+ load_result = run_fault_injection_with_rate_load(
+ submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
+ inject_fault=lambda: injector(omni_server_function),
+ num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
+ request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+ completion_timeout_sec=120.0,
+ )
+ assert load_result["failure_observed"], (
+ f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@hardware_test(res={"cuda": "H100"}, num_cards=4)
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup(
+ omni_server_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_root_no_load_{signal_name.lower()}"
+ injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
+ injector(omni_server_function)
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
+ omni_server_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_tree_no_load_{signal_name.lower()}"
+ injector = make_server_tree_kill_fault_injector(
+ signal_name=signal_name,
+ post_kill_wait_seconds=2.0,
+ inter_kill_wait_seconds=0.1,
+ )
+ injector(omni_server_function)
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.diffusion
+@pytest.mark.skip(reason="issue#4526")
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", HUNYUAN_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
+ omni_server_function,
+ openai_client_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_tree_with_load_{signal_name.lower()}"
+ injector = make_server_tree_kill_fault_injector(
+ signal_name=signal_name,
+ post_kill_wait_seconds=2.0,
+ inter_kill_wait_seconds=0.1,
+ )
+ load_result = run_fault_injection_with_rate_load(
+ submit_request=lambda: _submit_image_generation(openai_client_function, omni_server_function),
+ inject_fault=lambda: injector(omni_server_function),
+ num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
+ request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+ completion_timeout_sec=120.0,
+ )
+ assert load_result["failure_observed"], (
+ f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_image_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
diff --git a/tests/dfx/reliability/test_reliability_voxcpm2.py b/tests/dfx/reliability/test_reliability_voxcpm2.py
new file mode 100644
index 000000000..014bb4e89
--- /dev/null
+++ b/tests/dfx/reliability/test_reliability_voxcpm2.py
@@ -0,0 +1,505 @@
+"""
+VoxCPM2 reliability integration tests.
+
+Scenarios follow ``docs/user_guide/fault_injection_reliability_matrix.md``.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from tests.dfx.conftest import (
+ assert_fault_exception,
+ create_reliability_omni_server_params,
+ resolve_oom_device_spec,
+)
+from tests.dfx.reliability.helpers import (
+ FaultInjector,
+ assert_no_server_tree_process_residual_and_gpu_release,
+ extract_openai_error_contract_from_bytes,
+ get_health_raw,
+ inject_gpu_oom,
+ make_process_kill_fault_injector,
+ make_server_root_kill_fault_injector,
+ make_server_tree_kill_fault_injector,
+ post_json_raw,
+ run_fault_injection_with_rate_load,
+ stop_gpu_oom_hogs,
+ worker_residual_timeout_after_kill_signal,
+)
+from tests.helpers.mark import hardware_test
+from tests.helpers.media import load_test_audio_data_url
+from tests.helpers.runtime import OpenAIClientHandler
+
+RELIABILITY_SCENARIOS: list[dict[str, Any]] = [
+ {
+ "test_name": "voxcpm2_reliability_default",
+ "server_params": {
+ "model": "openbmb/VoxCPM2",
+ "stage_config_name": "voxcpm2.yaml",
+ },
+ },
+]
+
+DEPLOY_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "vllm_omni" / "deploy"
+OOM_INJECTION_CONFIG = {
+ "target_mem_ratio": 0.96,
+ "hold_seconds": 0,
+ "startup_timeout_sec": 20,
+ "strict": False,
+}
+FAULT_ERROR_KEYWORDS = (
+ "the request failed",
+ "oom",
+ "out of memory",
+ "cuda",
+ "orchestrator",
+ "timeout",
+ "connection",
+ "500",
+ "503",
+)
+PROCESS_KILL_ERROR_KEYWORDS = (
+ "timeout",
+ "did not complete within",
+ "connection",
+ "engine",
+ "orchestrator",
+ "dead",
+ "internal",
+ "500",
+ "503",
+)
+RUNTIME_WORKER_PATTERN = "VLLM::"
+SERVE_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGINT", id="sigint"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+SERVE_WITH_LOAD_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGINT", id="sigint"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+TREE_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+TREE_WITH_LOAD_SIGNAL_PARAMS = [
+ pytest.param("SIGTERM", id="sigterm"),
+ pytest.param("SIGKILL", id="sigkill"),
+]
+WORKER_SIGNAL_FAULT_PARAMS = [
+ pytest.param(
+ make_process_kill_fault_injector(
+ grep_patterns=RUNTIME_WORKER_PATTERN,
+ signal_name="SIGTERM",
+ limit=1,
+ post_kill_wait_seconds=2.0,
+ ),
+ id="runtime_process_chain_sigterm",
+ ),
+ pytest.param(
+ make_process_kill_fault_injector(
+ grep_patterns=RUNTIME_WORKER_PATTERN,
+ signal_name="SIGKILL",
+ limit=1,
+ post_kill_wait_seconds=2.0,
+ ),
+ id="runtime_process_chain_sigkill",
+ ),
+]
+
+VOXCPM2_PARAMS = create_reliability_omni_server_params(RELIABILITY_SCENARIOS, DEPLOY_CONFIGS_DIR)
+INFLIGHT_INJECTION_REQUEST_RATE = 0.3
+INFLIGHT_INJECTION_REQUEST_COUNT = 10
+_SPEECH_FF_HTTP_TIMEOUT_SEC = 30
+_SPEECH_FF_MAX_ELAPSED_SEC = 30
+
+
+def _speech_request_config(omni_server: Any) -> dict[str, Any]:
+ return {
+ "model": omni_server.model,
+ "input": "The weather is nice today, perfect for a walk in the park.",
+ "stream": False,
+ "timeout": 120.0,
+ "response_format": "wav",
+ "voice": "default",
+ }
+
+
+def _submit_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
+ openai_client.send_audio_speech_request(_speech_request_config(omni_server), request_num=1)
+
+
+def _oom_speech_request_config(omni_server: Any) -> dict[str, Any]:
+ base = "The weather is nice today, perfect for a walk in the park. "
+ return {
+ "model": omni_server.model,
+ "input": base * 80,
+ "max_new_tokens": 4096,
+ "ref_audio": load_test_audio_data_url("qwen3_tts/clone_2.wav"),
+ "stream": False,
+ "timeout": 300.0,
+ "response_format": "wav",
+ "voice": "default",
+ }
+
+
+def _submit_oom_speech_request(openai_client: OpenAIClientHandler, omni_server: Any) -> None:
+ openai_client.send_audio_speech_request(_oom_speech_request_config(omni_server), request_num=1)
+
+
+def _assert_post_fault_speech_fast_fail(host: str, port: int, *, model: str, scenario: str) -> None:
+ payload = {
+ "model": model,
+ "input": "post-fault fast-fail check",
+ "voice": "default",
+ }
+ start = time.monotonic()
+ try:
+ status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=20)
+ elapsed = time.monotonic() - start
+ assert elapsed < 15, f"[{scenario} fast_fail] /v1/audio/speech did not fail fast: {elapsed:.2f}s"
+ assert status >= 500, (
+ f"[{scenario} fast_fail] expected server-side error after fault, got status={status}, body={body[:200]!r}"
+ )
+ except Exception: # noqa: BLE001
+ elapsed = time.monotonic() - start
+ assert elapsed < 15, f"[{scenario} fast_fail] exception was too slow after fault: {elapsed:.2f}s"
+
+
+def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -> None:
+ deadline = time.monotonic() + 20.0
+ last_observation = ""
+ while time.monotonic() < deadline:
+ try:
+ status, body = get_health_raw(host, port, timeout_sec=5)
+ last_observation = f"http={status}, body={body[:200]!r}"
+ if status == 503:
+ return
+ except Exception as exc: # noqa: BLE001
+ last_observation = f"exception={exc!r}"
+ return
+ time.sleep(0.5)
+ pytest.fail(f"[{scenario} health] no terminal post-fault health observed: {last_observation}")
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skip(reason="issue#4285")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_speech_request_failure(
+ omni_server_function,
+ openai_client_function,
+) -> None:
+ stage_config_path = getattr(omni_server_function, "stage_config_path", None)
+ device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
+ handle = inject_gpu_oom(
+ device=device_spec,
+ target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+ hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+ startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+ strict=OOM_INJECTION_CONFIG["strict"],
+ )
+ try:
+ try:
+ _submit_oom_speech_request(openai_client_function, omni_server_function)
+ except Exception as exc:
+ assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
+ else:
+ pytest.fail("expected /v1/audio/speech request failure during GPU OOM injection")
+ finally:
+ stop_gpu_oom_hogs(handle)
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skip(reason="issue#4285")
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_gpu_oom_speech_error_contract(
+ omni_server_function,
+) -> None:
+ stage_config_path = getattr(omni_server_function, "stage_config_path", None)
+ device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
+ handle = inject_gpu_oom(
+ device=device_spec,
+ target_mem_ratio=OOM_INJECTION_CONFIG["target_mem_ratio"],
+ hold_seconds=OOM_INJECTION_CONFIG["hold_seconds"],
+ startup_timeout_sec=OOM_INJECTION_CONFIG["startup_timeout_sec"],
+ strict=OOM_INJECTION_CONFIG["strict"],
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ oom_config = _oom_speech_request_config(omni_server_function)
+ payload = {
+ "model": oom_config["model"],
+ "input": oom_config["input"],
+ "voice": oom_config["voice"],
+ "max_new_tokens": oom_config["max_new_tokens"],
+ "ref_audio": oom_config["ref_audio"],
+ }
+ try:
+ status, body = post_json_raw(host, port, "/v1/audio/speech", payload, timeout_sec=300)
+ finally:
+ stop_gpu_oom_hogs(handle)
+
+ assert status >= 500, f"expected speech runtime error under OOM, got status={status}"
+ speech_error = extract_openai_error_contract_from_bytes(body)
+ assert speech_error is not None, f"speech error payload not OpenAI-compatible: {body[:300]!r}"
+ assert "code" in speech_error, f"speech error lacks code field: {speech_error!r}"
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+ "fault_injector",
+ WORKER_SIGNAL_FAULT_PARAMS,
+ indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_request_failure(
+ omni_server_after_fault_function,
+ openai_client_function,
+) -> None:
+ try:
+ _submit_speech_request(openai_client_function, omni_server_after_fault_function)
+ except Exception as exc:
+ assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
+ else:
+ pytest.fail("expected /v1/audio/speech request failure after process-kill injection")
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+ "fault_injector",
+ WORKER_SIGNAL_FAULT_PARAMS,
+ indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_fast_fail_and_concurrent(
+ omni_server_after_fault_function,
+ openai_client_function,
+) -> None:
+ host = omni_server_after_fault_function.host
+ port = omni_server_after_fault_function.port
+ model = omni_server_after_fault_function.model
+ payload = {
+ "model": model,
+ "input": "Say hello in one short sentence.",
+ "voice": "default",
+ }
+
+ start = time.monotonic()
+ try:
+ status, body = post_json_raw(
+ host,
+ port,
+ "/v1/audio/speech",
+ payload,
+ timeout_sec=_SPEECH_FF_HTTP_TIMEOUT_SEC,
+ )
+ elapsed = time.monotonic() - start
+ assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, (
+ f"[process_kill fast_fail] /v1/audio/speech did not fail fast after fault: {elapsed:.2f}s"
+ )
+ assert status >= 500, (
+ "[process_kill fast_fail] expected server-side failure after fault, "
+ f"got status={status}, body={body[:200]!r}"
+ )
+ except Exception:
+ elapsed = time.monotonic() - start
+ assert elapsed <= _SPEECH_FF_MAX_ELAPSED_SEC + 1.0, (
+ f"[process_kill fast_fail] /v1/audio/speech exception was too slow after fault: {elapsed:.2f}s"
+ )
+
+ start = time.monotonic()
+ with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+ futures = [
+ executor.submit(_submit_speech_request, openai_client_function, omni_server_after_fault_function)
+ for _ in range(3)
+ ]
+ done, pending = concurrent.futures.wait(
+ futures,
+ timeout=30,
+ return_when=concurrent.futures.ALL_COMPLETED,
+ )
+
+ elapsed = time.monotonic() - start
+ assert not pending, f"[process_kill concurrent] some fault-time speech requests hung: pending={len(pending)}"
+ assert elapsed < 30, f"[process_kill concurrent] fault-time speech request convergence is too slow: {elapsed:.2f}s"
+
+ fault_observed = False
+ for future in done:
+ try:
+ future.result()
+ except Exception as exc:
+ fault_observed = True
+ assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
+ assert fault_observed, (
+ "[process_kill concurrent] expected at least one /v1/audio/speech request to fail after fault"
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize(
+ "fault_injector",
+ WORKER_SIGNAL_FAULT_PARAMS,
+ indirect=True,
+)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_worker_with_load_request_failure(
+ omni_server_function,
+ openai_client_function,
+ fault_injector: FaultInjector,
+) -> None:
+ scenario = "kill_worker_with_load"
+ load_result = run_fault_injection_with_rate_load(
+ submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function),
+ inject_fault=lambda: fault_injector(omni_server_function),
+ num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
+ request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+ completion_timeout_sec=120.0,
+ )
+ assert load_result["failure_observed"], (
+ f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", SERVE_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_serve_root_no_load_fast_fail_and_cleanup(
+ omni_server_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_root_no_load_{signal_name.lower()}"
+ injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
+ injector(omni_server_function)
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", SERVE_WITH_LOAD_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
+ omni_server_function,
+ openai_client_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
+ injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
+ load_result = run_fault_injection_with_rate_load(
+ submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function),
+ inject_fault=lambda: injector(omni_server_function),
+ num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
+ request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+ completion_timeout_sec=300.0,
+ )
+ assert load_result["failure_observed"], (
+ f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", TREE_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
+ omni_server_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_tree_no_load_{signal_name.lower()}"
+ injector = make_server_tree_kill_fault_injector(
+ signal_name=signal_name,
+ post_kill_wait_seconds=2.0,
+ inter_kill_wait_seconds=0.1,
+ )
+ injector(omni_server_function)
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.skipif(os.name == "nt", reason="process-kill injection helper is POSIX-only")
+@pytest.mark.parametrize("signal_name", TREE_WITH_LOAD_SIGNAL_PARAMS)
+@pytest.mark.parametrize("omni_server_function", VOXCPM2_PARAMS, indirect=True)
+def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
+ omni_server_function,
+ openai_client_function,
+ signal_name: str,
+) -> None:
+ scenario = f"kill_serve_tree_with_load_{signal_name.lower()}"
+ injector = make_server_tree_kill_fault_injector(
+ signal_name=signal_name,
+ post_kill_wait_seconds=2.0,
+ inter_kill_wait_seconds=0.1,
+ )
+ load_result = run_fault_injection_with_rate_load(
+ submit_request=lambda: _submit_speech_request(openai_client_function, omni_server_function),
+ inject_fault=lambda: injector(omni_server_function),
+ num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
+ request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
+ completion_timeout_sec=120.0,
+ )
+ assert load_result["failure_observed"], (
+ f"[{scenario}] expected at least one load request failure after fault; load_result={load_result}"
+ )
+ host = omni_server_function.host
+ port = omni_server_function.port
+ _assert_post_fault_speech_fast_fail(host, port, model=omni_server_function.model, scenario=scenario)
+ _assert_post_fault_health_terminal(host, port, scenario=scenario)
+ assert_no_server_tree_process_residual_and_gpu_release(
+ omni_server_function,
+ scenario=scenario,
+ timeout_sec=worker_residual_timeout_after_kill_signal(signal_name),
+ )
diff --git a/tests/dfx/stability/scripts/test_stability_voxcpm2.py b/tests/dfx/stability/scripts/test_stability_voxcpm2.py
new file mode 100644
index 000000000..e26a2050c
--- /dev/null
+++ b/tests/dfx/stability/scripts/test_stability_voxcpm2.py
@@ -0,0 +1,69 @@
+"""
+VoxCPM2 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration.
+
+Configuration: ``tests/dfx/stability/tests/test_voxcpm2.json``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tests.dfx.conftest import (
+ create_benchmark_indices,
+ create_test_parameter_mapping,
+ create_unique_server_params,
+ load_configs,
+)
+from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop
+
+STABILITY_DIR = Path(__file__).resolve().parent.parent
+DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy"
+CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_voxcpm2.json")
+DEFAULT_NUM_PROMPTS_PER_BATCH = 20
+STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"]
+
+try:
+ BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
+except FileNotFoundError:
+ BENCHMARK_CONFIGS = []
+
+test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else []
+server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {}
+benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
+
+
+@pytest.mark.slow
+@pytest.mark.tts
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True)
+def test_stability_voxcpm2(omni_server, stability_benchmark_params):
+ test_name = stability_benchmark_params["test_name"]
+ params = stability_benchmark_params["params"]
+ duration_sec = params.get("duration_sec", 300)
+ num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH)
+ request_rate = params.get("request_rate")
+ max_concurrency = params.get("max_concurrency")
+
+ bench_params = {
+ k: v
+ for k, v in params.items()
+ if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch")
+ }
+
+ result = run_stability_benchmark_loop(
+ host=omni_server.host,
+ port=omni_server.port,
+ model=omni_server.model,
+ duration_sec=duration_sec,
+ params=bench_params,
+ request_rate=request_rate,
+ max_concurrency=max_concurrency,
+ result_dir=str(STABILITY_DIR),
+ num_prompts_per_batch=num_prompts_per_batch,
+ run_one_batch=_run_one_vllm_bench_batch,
+ )
+
+ assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}"
+ assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed"
diff --git a/tests/dfx/stability/tests/test_voxcpm2.json b/tests/dfx/stability/tests/test_voxcpm2.json
new file mode 100644
index 000000000..dd912da36
--- /dev/null
+++ b/tests/dfx/stability/tests/test_voxcpm2.json
@@ -0,0 +1,35 @@
+[
+ {
+ "test_name": "test_voxcpm2_stability",
+ "server_params": {
+ "model": "openbmb/VoxCPM2",
+ "trust_remote_code": true
+ },
+ "benchmark_params": [
+ {
+ "dataset_name": "random",
+ "backend": "openai-audio-speech",
+ "endpoint": "/v1/audio/speech",
+ "trust_remote_code": true,
+ "duration_sec": 86400,
+ "max_concurrency": 2,
+ "num_prompts_per_batch": 50,
+ "random_input_len": {
+ "min": 0,
+ "max": 1000
+ },
+ "random_output_len": {
+ "min": 0,
+ "max": 1000
+ },
+ "random_range_ratio": 0.0,
+ "extra_body": {
+ "voice": "default",
+ "language": "English"
+ },
+ "ignore_eos": true,
+ "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration"
+ }
+ ]
+ }
+]
|