From 2a49b2b21722df40efdfaa4a6ce1f4dd5f802d05 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:45:00 +0800 Subject: [PATCH] =?UTF-8?q?[CI][Perf]=20Add=20JSON=20mark=20support=20and?= =?UTF-8?q?=20paired=20parametrize=20for=20L4=20perf,=20so=20as=20to=20fur?= =?UTF-8?q?ther=20support=20performance=E2=80=91case=20selection=20for=20N?= =?UTF-8?q?PU=E2=80=91CI.=20(#5084)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .claude/skills/add-diffusion-model/SKILL.md | 2 +- .claude/skills/vllm-omni-test/SKILL.md | 18 +- .../vllm-omni-test/references/test-routing.md | 31 +- docs/contributing/ci/CI_5levels.md | 17 +- .../test_examples/l4_performance_tests.inc.md | 122 +++- docs/contributing/ci/test_guide.md | 14 +- docs/contributing/ci/tests_markers.md | 31 +- recipes/LTX/LTX-2.3.md | 4 +- tests/dfx/conftest.py | 352 ++++++++- tests/dfx/perf/scripts/run_benchmark.py | 49 +- .../perf/scripts/run_diffusion_benchmark.py | 189 +++-- .../dfx/perf/tests/test_bagel_vllm_omni.json | 48 ++ .../perf/tests/test_cosmos3_vllm_omni.json | 458 ++++++------ tests/dfx/perf/tests/test_higgs_audio_v3.json | 12 + .../perf/tests/test_hunyuan_image3_it2i.json | 284 ++++---- .../tests/test_hunyuan_image_tp2_cfgp2.json | 78 +- .../tests/test_hunyuan_image_tp2_sp2.json | 78 +- .../perf/tests/test_hunyuan_image_tp4.json | 78 +- tests/dfx/perf/tests/test_ltx2_vllm_omni.json | 478 +++++++------ .../tests/test_qwen3_omni_async_chunk.json | 668 +++++++++--------- .../tests/test_qwen3_omni_multi_replicas.json | 236 ++++--- .../tests/test_qwen3_omni_no_async_chunk.json | 346 ++++----- .../perf/tests/test_qwen3_omni_vllm_text.json | 366 +++++----- .../test_qwen_image_edit_2511_vllm_omni.json | 324 +++++---- .../tests/test_qwen_image_edit_vllm_omni.json | 314 ++++---- .../test_qwen_image_layered_vllm_omni.json | 106 +-- .../perf/tests/test_qwen_image_vllm_omni.json | 484 +++++++------ tests/dfx/perf/tests/test_runner_metadata.py | 199 ++++++ tests/dfx/perf/tests/test_tts.json | 36 + tests/dfx/perf/tests/test_voxcpm2.json | 12 + .../perf/tests/test_wan22_i2v_vllm_omni.json | 232 +++--- 31 files changed, 3475 insertions(+), 2191 deletions(-) diff --git a/.claude/skills/add-diffusion-model/SKILL.md b/.claude/skills/add-diffusion-model/SKILL.md index 906f3c485..76c894473 100644 --- a/.claude/skills/add-diffusion-model/SKILL.md +++ b/.claude/skills/add-diffusion-model/SKILL.md @@ -334,7 +334,7 @@ Classify the model's **CI priority** first: | **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts | | **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt | | **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke | -| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` | +| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` with per-case **`mark`** (`hardware_marks` + `full_model` + `diffusion`) | **L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`). diff --git a/.claude/skills/vllm-omni-test/SKILL.md b/.claude/skills/vllm-omni-test/SKILL.md index 40a81f4cf..dcc2506da 100644 --- a/.claude/skills/vllm-omni-test/SKILL.md +++ b/.claude/skills/vllm-omni-test/SKILL.md @@ -97,7 +97,7 @@ Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-styl | **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required | | **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` | | **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). | -| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: `full_model` + `benchmark`; wire `test-nightly.yml` Perf steps | +| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps | | **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) | **If the user’s test plan includes invalid parameter validation / invalid params / negative HTTP / 400 validation:** do **not** add those `test_*` functions to `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move or author them** under `tests/dfx/reliability/invalid_param_test/` in the **endpoint-matching script** (see **Invalid parameter validation** below). Success-path e2e and invalid-param dfx tests must stay in separate modules. @@ -270,7 +270,7 @@ def test_text_to_video_001(omni_server, openai_client) -> None: *Documentation example tests:* follow the **Preferred Test Strategy** in [l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md): dynamic extraction for offline READMEs; explicit copied client code for online pages until extraction is justified; use the documented **naming**, **output directory** (page folder + case id), and **skipping** rules (e.g. Gradio-only scripts). -*Performance tests:* add or extend entries under `tests/perf/` (and JSON configs where the project uses them), with **explicit baselines** and the same marker/run-level pairing as the CI step that will execute them. +*Performance tests:* add or extend entries under `tests/dfx/perf/tests/` (and JSON configs where the project uses them), with **explicit baselines**, **`mark`** on each case (`hardware_marks` + `full_model` + type marker), and the same nightly Perf step pattern as in-tree configs. **3. Cross-cutting rules** @@ -592,27 +592,31 @@ Implementation strategy: **When the user explicitly asks for L4 perf (or full L4 including perf):** 1. **Do not** put throughput/latency assertions inside `test_*_expansion.py` — perf uses the **dfx benchmark harness**, not `omni_server` e2e fixtures. -2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`). -3. Each JSON entry: `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]` with explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …). +2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, `test_qwen3_omni_async_chunk.json`). +3. Each JSON **case**: `test_name`, optional **`mark`** (`hardware_marks` required when present; `marks`: `full_model` + `omni`/`tts`/`diffusion`), `server_params` (`model`, `serve_args` or `stage_config_name`), `benchmark_params[]` with **`name`** and explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …). Diffusion cases also set `server_type` (e.g. `"vllm-omni"`) and usually `benchmark_endpoint`. 4. Run locally via the matching script (model type → runner): | Model type | Runner script | Example config | |------------|---------------|----------------| | **Diffusion X2I/X2V** | `tests/dfx/perf/scripts/run_diffusion_benchmark.py` | `tests/dfx/perf/tests/test_qwen_image_vllm_omni.json` | | **TTS** | `tests/dfx/perf/scripts/run_benchmark.py` | `tests/dfx/perf/tests/test_tts.json` (shared) or `test_{slug}.json` (dedicated; use when the model must not join the shared nightly matrix before integration — e.g. `test_voxcpm2.json`, `test_coqui_tts.json`) | -| **Omni** | (see existing Omni Perf steps — `run_benchmark.py` / dedicated omni configs in-tree) | `tests/dfx/perf/tests/test_omni*.json` | +| **Omni** | `tests/dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` | -5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with `--test-config-file`, **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf. +5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with **`--test-config-file`** (nightly perf steps do not use `-m`), **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf. 6. **One benchmark scenario → one `test_name` block** in JSON (same spirit as one `test_*` per function case). Combine server `serve_args` + workload in one entry; use multiple `benchmark_params` rows for size/step sweeps under the same server config. -**Perf local command (diffusion example):** +**Perf local commands:** ```bash cd tests export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN +# Single file (CI-like) pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file dfx/perf/tests/test__vllm_omni.json +# Bulk load + filter by JSON mark +pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" +pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" ``` **Clarify in the test plan** which L4 pillars you deliver: `Function only` | `Function + Perf` | `Function + Accuracy + Perf`. diff --git a/.claude/skills/vllm-omni-test/references/test-routing.md b/.claude/skills/vllm-omni-test/references/test-routing.md index 445a78399..f78e1c77d 100644 --- a/.claude/skills/vllm-omni-test/references/test-routing.md +++ b/.claude/skills/vllm-omni-test/references/test-routing.md @@ -127,8 +127,12 @@ pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and cd tests export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN +# CI-like: single JSON file (nightly Perf Test steps) pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \ --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json +# Local bulk load: all *.json under tests/dfx/perf/tests/, filter by JSON mark +pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" +pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" ``` Broad marker sweep (when no explicit file shard): @@ -150,7 +154,26 @@ Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not **Rule:** “L4 functional cases” → **Function only** by default. State that Perf/Accuracy are separate pillars; add them only when requested. -**Diffusion perf JSON** (`tests/dfx/perf/tests/test__vllm_omni.json`): array of objects with `test_name`, `server_params` (`model`, `serve_args`), `benchmark_params[]`, and per-workload **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`). Copy structure from `test_qwen_image_vllm_omni.json` in-tree. +**Diffusion perf JSON** (`tests/dfx/perf/tests/test__vllm_omni.json`): array of **case objects**, each with: + +| Field | Required | Notes | +|-------|----------|-------| +| `test_name` | Yes | Unique server/workload id | +| `mark` | Recommended | When present: **`hardware_marks` required** + optional `marks` (`full_model`, `diffusion`, `local_model`, …). Parsed by `resolve_pytest_marks`; enables local `-m` filtering. | +| `server_type` | Diffusion | e.g. `"vllm-omni"` — routes case to `run_diffusion_benchmark.py` | +| `benchmark_endpoint` | Diffusion | e.g. `/v1/videos`, `/v1/images/generations` | +| `server_params` | Yes | `model`, `serve_args`, … | +| `benchmark_params[]` | Yes | Each row: **`name`** (pytest id suffix), workload fields, **`baseline`** | + +Copy structure from `test_qwen_image_vllm_omni.json` or `test_cosmos3_vllm_omni.json` (2-GPU mark example). + +**Omni perf JSON**: `test_qwen3_omni_*.json` — same `mark` shape with `"marks": ["full_model", "omni"]`; run via `run_benchmark.py`. + +**Runner split**: `is_diffusion_perf_config()` → diffusion when `server_type` is set or `"diffusion"` ∈ `mark.marks`; otherwise omni/tts → `run_benchmark.py`. + +**Param ids**: `{test_name}-{benchmark_params.name}` (e.g. `test_tts-p0`, not `test_tts-0`). + +**Result filenames**: runtime hardware from `get_runtime_resource_label()`; `H100` omitted on default CI pool. **Diffusion perf nightly step** (under **Diffusion X2I(&A&T)**; copy full `kubernetes` plugins from `· Perf Test · Qwen-Image`): @@ -177,9 +200,9 @@ Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not | Model type | Perf runner | Config example | |------------|-------------|----------------| -| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json` | -| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json` (`BENCHMARK_DIR`) | -| Omni | `run_benchmark.py` / omni configs in-tree | `test_omni*.json` | +| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json` | +| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | +| Omni | `dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_no_async_chunk.json`, … | Do **not** put throughput/latency baselines inside `test_*_expansion.py` — that belongs in the dfx perf JSON + nightly Perf job. diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index 17b4a3792..c9d86e1c4 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -126,11 +126,12 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari Interface tests:
/tests/entrypoints/test_xxx
Performance:
- /tests/dfx/perf/tests/test_qwen_omni.json (Omni), test_tts.json (TTS),
- and /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
+ /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),
+ test_voxcpm2.json, test_higgs_audio_v3.json, and
+ /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)
Doc Test:
- tests/example/online_serving/test_{model_name}.py
- tests/example/offline_inference/test_{model_name}.py
+ tests/examples/online_serving/test_{model_name}.py
+ tests/examples/offline_inference/test_{model_name}.py
Accuracy Test:
/tests/e2e/accuracy/test_{model_name}.py @@ -708,16 +709,16 @@ L4 level testing is a comprehensive quality audit before a version release. It e ### 3.2 Testing Content and Scope - ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling. -- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen_omni.json`, `tests/dfx/perf/tests/test_tts.json`, and diffusion configs in the form `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` via `--test-config-file`) to drive performance testing tools for stress, load, and endurance tests, collecting metrics like throughput, response time, and resource utilization. +- ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below. - ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description. ### 3.3 Test Directory and Execution Files - ***Functional Testing***: Same directories as L3. -- ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen_omni.json`, `tests/dfx/perf/tests/test_tts.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`) +- ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen3_omni_*.json`, `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`). Optional per-case `mark` for local `-m` filtering is documented in Section 3.4 Performance Tests. - ***Documentation Example Tests***: -- - `tests/example/online_serving/test_{model_name}.py` - - `tests/example/offline_inference/test_{model_name}.py` + - `tests/examples/online_serving/test_{model_name}.py` + - `tests/examples/offline_inference/test_{model_name}.py` ### 3.4 Execution Method and Example diff --git a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md index f1f3073dc..5c59ba499 100644 --- a/docs/contributing/ci/test_examples/l4_performance_tests.inc.md +++ b/docs/contributing/ci/test_examples/l4_performance_tests.inc.md @@ -1,14 +1,41 @@ -When you want to add L4-level ***performance test*** cases, you can refer to the following format for case addition in `tests/dfx/perf/tests/test_qwen_omni.json`, `tests/dfx/perf/tests/test_tts.json`, or diffusion configs such as `tests/dfx/perf/tests/test_*_vllm_omni.json` (selected via `pytest ... run_benchmark.py --test-config-file `): +When you want to add L4-level ***performance test*** cases, add entries to JSON files under `tests/dfx/perf/tests/` and run them via `tests/dfx/perf/scripts/run_benchmark.py` (omni / TTS) or `run_diffusion_benchmark.py` (diffusion). + +**Config file layout (in-tree examples)** + +| Model type | Runner | Example JSON files | +| ---------- | ------ | ------------------ | +| Omni | `run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` | +| TTS | `run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` | +| Diffusion | `run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_bagel_vllm_omni.json`, `test_wan22_i2v_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, … | + +**How runners pick cases** + +Without **`--test-config-file`**, each runner scans all `*.json` under `tests/dfx/perf/tests/` but only keeps its own model type: + +- **`run_benchmark.py`**: omni and TTS cases only (skips diffusion JSON). +- **`run_diffusion_benchmark.py`**: diffusion cases only (skips omni / TTS JSON). + +Diffusion cases are detected when the JSON has `server_type` (typically `"vllm-omni"`) or `"diffusion"` in the `mark` array. Omni / TTS JSON has neither. + +**Running perf cases** + +Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bulk scan above and filter with pytest **`-m`** on each case's JSON `mark` (for example `pytest … run_benchmark.py -m "full_model and tts and H100"`). ```JSON { - "test_name": "test_qwen3_omni", + "test_name": "test_qwen3_omni_async_chunk", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "omni" + ], "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "stage_config_name": "qwen3_omni.yaml" }, "benchmark_params": [ { + "name": "random_10p", "dataset_name": "random", "num_prompts": [10, 20], "max_concurrency": [1, 4], @@ -33,8 +60,95 @@ When you want to add L4-level ***performance test*** cases, you can refer to the | Field | Required | Description | | ---------------- | -------- | --------------------------------------------------------------- | | test_name | Yes | Unique identifier for the test case | +| mark | No | Pytest marks for this case (see **`mark` field** below). Omit only for configs not meant to be filtered by `-m`. | | server_params | Yes | Server-side configuration parameters | | benchmark_params | Yes | Benchmark running parameters (supports multiple configurations) | +| server_type | Diffusion only | When set (e.g. `"vllm-omni"`), routes the case to `run_diffusion_benchmark.py`. | +| benchmark_endpoint | Diffusion only | API path for the benchmark client (e.g. `/v1/videos`, `/v1/images/generations`). | + +**`mark` field** + +Optional top-level field on each perf JSON **case object** (one per `test_name`). `run_benchmark.py` and `run_diffusion_benchmark.py` read it via `tests.dfx.conftest.resolve_pytest_marks` and attach the marks to the corresponding `pytest.param`, so you can filter locally with `-m`. + +When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same shape as `@hardware_test` / `hardware_marks()` in `tests/helpers/mark.py`), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, `diffusion`, or `local_model`. + +Supported form: + +| Form | Example | Effect | +| ---- | ------- | ------ | +| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` > 1 adds `distributed_*` (+ CUDA `skipif_cuda` when applicable). String entries become extra pytest marks. | +| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325", "npu": "A2"}, "num_cards": {"cuda": 2, "rocm": 2, "npu": 1}}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** in one case. Filter examples: `-m "full_model and H100 and cuda"`, `-m "full_model and MI325 and rocm"`. | + +Recommended for L4 perf cases: + +```JSON +{ + "test_name": "test_bagel_single_device_single_stage_t2i", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "server_params": { "...": "..." }, + "benchmark_params": [ { "name": "1024x1024_steps20", "...": "..." } ] +} +``` + +Multi-GPU diffusion (example: Cosmos3 with `cfg-parallel-size=2`): + +```JSON +{ + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/generations", + "server_params": { "...": "..." }, + "benchmark_params": [ { "name": "1024x1024_steps4", "...": "..." } ] +} +``` + +- Omni perf: include `"omni"` in the `mark` array +- TTS perf: include `"tts"` in the `mark` array +- Diffusion perf: include `"diffusion"` in the `mark` array +- HunyuanImage local-weight cases: add `"local_model"` to the `mark` array + +**Parametrization IDs** + +Each `(server, benchmark index)` pair becomes one `pytest.param` with id `{test_name}-{suffix}`. The suffix comes from `benchmark_params[].name` when set (for example `test_tts-p0`, `test_omni-p1`); otherwise it is derived from fields like `task` / `eval_phase`. + +**Benchmark result filenames** + +Result files use the **runtime** hardware label from `get_runtime_resource_label()` (detected GPU/NPU name on the machine that ran the job), **not** `mark.hardware_marks.res`. On the default H100 CI pool, `H100` is **omitted** from filenames (`resource_label_for_filename`). + +Examples: + +- Omni/TTS: `result_{test_name}_{optional_hw}_{dataset}_....json` under `BENCHMARK_DIR` +- Diffusion: one aggregate `diffusion_result_{config_stem}_{optional_hw}_{timestamp}.json` per source JSON file (array of all runs from that file) + +**Local commands** + +```bash +# Bulk load + filter by JSON mark +pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and H100 and diffusion" +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100" + +# Single file (same as nightly CI Perf steps) +pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json +pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \ + --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + +# Optional baseline assertion (default off) +pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \ + --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +``` + +See also [Markers for Tests](./tests_markers.md) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …). **`server_params` Configuration** @@ -79,10 +193,12 @@ You can add any benchmark running parameters you need here. For all optional par 1. Change the --xxx-xx-xx running parameters to xxx_xx_xx format and fill them as keys in the JSON file. 2. For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file. 3. Optionally add a `baseline` object (see **Baseline thresholds** below). If you omit `baseline` or leave it empty, the performance test still runs but does not assert metric thresholds from this field. -4. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: +4. Set `"name"` on each `benchmark_params` entry for stable pytest ids and readable result keys. +5. The qps and concurrency modes are recommended to be mutually exclusive. For detailed explanations, see the table below: | Parameter | Type | Required | Example/Values | Description | | --------------- | ----------- | -------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| name | string | No | `"1024x1024_steps4"` | Stable suffix for pytest param id and logs. | | num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. | | request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. | | max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). | diff --git a/docs/contributing/ci/test_guide.md b/docs/contributing/ci/test_guide.md index 26996129e..ad6a63963 100644 --- a/docs/contributing/ci/test_guide.md +++ b/docs/contributing/ci/test_guide.md @@ -149,12 +149,18 @@ Failed jobs: 1/2 pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model ``` - - To run a standalone perf benchmark (defaults to ``test_qwen_omni.json``; use ``--test-config-file tests/dfx/perf/tests/test_tts.json`` for TTS): - + If you only want to run specific test cases on a particular platform, you can use: ```bash - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py + pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model ``` + Note: ``run_benchmark.py`` and ``run_diffusion_benchmark.py`` accept an optional ``--test-config-file``. If omitted, each loads every ``*.json`` under ``tests/dfx/perf/tests/`` (omni/tts vs diffusion split by ``is_diffusion_perf_config``) and pytest ``-m`` filters by each case's JSON ``mark``: + ```bash + pytest -sv tests/dfx/perf/scripts/run_benchmark.py -m "full_model and tts and H100" + pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100" + pytest -sv tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json + pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json + ``` + Nightly **Perf Test** jobs in ``test-nightly.yml`` use ``--test-config-file`` only (no ``-m``). E2e L4 function tests still use ``full_model`` + ``--run-level full_model`` (see [test-nightly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-nightly.yml)). Example: === "L5 level" diff --git a/docs/contributing/ci/tests_markers.md b/docs/contributing/ci/tests_markers.md index cc89cee47..0a0a974e6 100644 --- a/docs/contributing/ci/tests_markers.md +++ b/docs/contributing/ci/tests_markers.md @@ -33,7 +33,8 @@ Defined in `pyproject.toml`: | `skipif_rocm` | Skip if the num of ROCm cards is less than the required * | | `skipif_npu` | Skip if the num of NPU cards is less than the required * | | `slow` | Slow tests (may skip in quick CI) | -| `benchmark` | Benchmark tests | +| `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) | +| `local_model` | Tests requiring local / non-HF-hub model weights | \* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). @@ -94,7 +95,7 @@ This decorator is intended to make hardware-aware, cross-platform test authoring num_cards=2, ) ``` -- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), NPU (A2/A3) +- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3) - `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing - Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases - Filtering examples: @@ -120,6 +121,32 @@ def test_feature(omni_server): ... ``` +### JSON `mark` field (L4 perf configs) + +Perf JSON under `tests/dfx/perf/tests/` can attach pytest marks per **case** (`test_name` block). Parsed by `tests.dfx.conftest.resolve_pytest_marks` and applied to each parametrized run in `run_benchmark.py` / `run_diffusion_benchmark.py`. + +When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same semantics as `hardware_marks()` above), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, or `diffusion`. + +```json +{ + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ], + "server_type": "vllm-omni", + "server_params": { "...": "..." }, + "benchmark_params": [{ "name": "1024x1024_steps4", "...": "..." }] +} +``` + +- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"` +- Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test__vllm_omni.json` (file selects cases; no `-m`) +- Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool + +See [L4 performance test examples](./test_examples/l4_performance_tests.inc.md) for the full schema. + ## Add Support for a New Platform If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps: diff --git a/recipes/LTX/LTX-2.3.md b/recipes/LTX/LTX-2.3.md index c83a14883..7a4f1507c 100644 --- a/recipes/LTX/LTX-2.3.md +++ b/recipes/LTX/LTX-2.3.md @@ -148,7 +148,7 @@ Do not copy latency or VRAM numbers between commits or machines. Record them only after running a latest-head sweep on the target hardware. For formal PR benchmarking, reuse -`tests/dfx/perf/tests/test_ltx2_3_vllm_omni.json` with -`tests/dfx/perf/scripts/run_diffusion_benchmark.py`. That config captures the +`tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with +`tests/dfx/perf/scripts/run_diffusion_benchmark.py`. Each case in that file includes a JSON **`mark`** (`hardware_marks` + `full_model` + `diffusion`) for local `-m` filtering. That config captures the single-device eager baseline and CFG-parallel=2 case for a small 384x512, 25-frame, 20-step workload. diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index a22f760b2..2eb88bbd6 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -11,11 +11,180 @@ from typing import Any import pytest from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw +from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServerParams from tests.helpers.stage_config import modify_stage_config from vllm_omni.platforms import current_omni_platform +def _named_pytest_marks(names: list[str]) -> list[pytest.MarkDecorator]: + marks: list[pytest.MarkDecorator] = [] + for name in names: + name = name.strip() + if not name: + raise ValueError("mark name must be a non-empty string") + marks.append(getattr(pytest.mark, name)) + return marks + + +def _hardware_marks_from_dict(hw: Any) -> list[pytest.MarkDecorator]: + if not isinstance(hw, dict): + raise ValueError(f"mark.hardware_marks must be a dict, got {type(hw).__name__}") + res = hw.get("res") + if not isinstance(res, dict): + raise ValueError(f"mark.hardware_marks.res must be a dict, got {type(res).__name__}") + num_cards = hw.get("num_cards", 1) + return list(hardware_marks(res=res, num_cards=num_cards)) + + +def resolve_pytest_marks(mark_field: Any) -> list[pytest.MarkDecorator]: + """Convert a JSON ``mark`` field into pytest mark decorators. + + Supported form (per test-case object in perf/stability JSON):: + + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion" + ] + + Exactly one ``hardware_marks`` object is required when ``mark`` is present. + ``hardware_marks`` delegates to :func:`tests.helpers.mark.hardware_marks` + (same shape as ``@hardware_test``). Additional array entries are registered + pytest marker names (strings). + """ + if mark_field is None: + return [] + + if isinstance(mark_field, list): + marks: list[pytest.MarkDecorator] = [] + hw_seen = False + for item in mark_field: + if isinstance(item, dict) and "hardware_marks" in item: + if hw_seen: + raise ValueError("mark array must contain at most one hardware_marks object") + hw_seen = True + marks.extend(_hardware_marks_from_dict(item["hardware_marks"])) + unknown_keys = set(item) - {"hardware_marks"} + if unknown_keys: + raise ValueError( + f"mark hardware_marks object only allows hardware_marks; unknown keys: {sorted(unknown_keys)}" + ) + elif isinstance(item, str): + item = item.strip() + if not item: + raise ValueError("mark name must be a non-empty string") + marks.extend(_named_pytest_marks([item])) + else: + raise ValueError( + f"mark array entries must be hardware_marks objects or marker name strings; got {type(item).__name__}" + ) + if not hw_seen: + raise ValueError("mark array must contain a hardware_marks object") + return marks + + raise ValueError(f"mark must be a list; got {type(mark_field).__name__}") + + +def _mark_names(mark_field: Any) -> set[str]: + if isinstance(mark_field, list): + return {str(item) for item in mark_field if isinstance(item, str)} + return set() + + +def is_diffusion_perf_config(cfg: dict[str, Any]) -> bool: + """True for perf JSON cases intended for ``run_diffusion_benchmark.py``.""" + if cfg.get("server_type") is not None: + return True + return "diffusion" in _mark_names(cfg.get("mark")) + + +def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: + return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs} + + +def create_unique_server_pytest_params( + configs: list[dict[str, Any]], + stage_configs_dir: Path, +) -> list[Any]: + """Like :func:`create_unique_server_params`, but wrap each row in ``pytest.param`` with JSON marks.""" + marks_by_name = _marks_by_test_name(configs) + return [ + pytest.param( + row, + marks=marks_by_name.get(row[0], []), + id=row[0], + ) + for row in create_unique_server_params(configs, stage_configs_dir) + ] + + +def create_benchmark_pytest_params( + benchmark_configs: list[dict[str, Any]], + server_to_benchmark_mapping: dict[str, dict], +) -> list[Any]: + """Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks.""" + marks_by_name = _marks_by_test_name(benchmark_configs) + params: list[Any] = [] + seen: set[str] = set() + for config in benchmark_configs: + test_name = config["test_name"] + if test_name in seen: + continue + seen.add(test_name) + params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) + id_suffixes = _unique_benchmark_param_id_suffixes(params_list) + for idx, id_suffix in enumerate(id_suffixes): + params.append( + pytest.param( + (test_name, idx), + marks=marks_by_name.get(test_name, []), + id=f"{test_name}-{id_suffix}", + ) + ) + return params + + +def create_paired_benchmark_pytest_params( + server_entries: list[tuple[Any, str]], + params_by_test_name: dict[str, list[Any]], + marks_by_name: dict[str, list[pytest.MarkDecorator]], +) -> list[Any]: + """One ``pytest.param`` per ``(server entry, benchmark index)`` pair. + + Pass ``server_entry`` and ``(test_name, idx)`` as separate ``pytest.param`` + arguments for ``@pytest.mark.parametrize("server_fixture,benchmark_params", ...)``. + """ + pairs: list[Any] = [] + for server_entry, test_name in server_entries: + params_list = params_by_test_name.get(test_name, []) + id_suffixes = _unique_benchmark_param_id_suffixes(params_list) + for idx, id_suffix in enumerate(id_suffixes): + pairs.append( + pytest.param( + server_entry, + (test_name, idx), + marks=marks_by_name.get(test_name, []), + id=f"{test_name}-{id_suffix}", + ) + ) + return pairs + + +def create_paired_omni_benchmark_pytest_params( + configs: list[dict[str, Any]], + stage_configs_dir: Path, +) -> list[Any]: + """Paired params for ``run_benchmark.py`` (omni/tts).""" + mapping = create_test_parameter_mapping(configs) + marks_by_name = _marks_by_test_name(configs) + server_entries = [(row, row[0]) for row in create_unique_server_params(configs, stage_configs_dir)] + params_by_test_name = { + test_name: get_benchmark_params_for_server(test_name, mapping) for _, test_name in server_entries + } + return create_paired_benchmark_pytest_params(server_entries, params_by_test_name, marks_by_name) + + def load_configs(config_path: str) -> list[dict[str, Any]]: try: abs_path = Path(config_path).resolve() @@ -32,6 +201,29 @@ def load_configs(config_path: str) -> list[dict[str, Any]]: raise RuntimeError(f"Failed to load configuration file: {str(e)}") +def load_benchmark_configs( + config_path: str | None = None, + *, + config_dir: Path | None = None, +) -> list[dict[str, Any]]: + """Load one benchmark JSON file, or merge all ``*.json`` under *config_dir*. + + When *config_path* is omitted, every ``*.json`` in *config_dir* is loaded and + concatenated. Pytest ``-m`` expressions then select cases via each entry's + ``mark`` field (e.g. ``-m tts`` after bulk load). + """ + if config_path is not None: + return load_configs(config_path) + if config_dir is None: + raise ValueError("load_benchmark_configs requires config_path or config_dir") + configs: list[dict[str, Any]] = [] + for path in sorted(config_dir.glob("*.json")): + configs.extend(load_configs(str(path))) + if not configs: + raise ValueError(f"No benchmark JSON files found under {config_dir}") + return configs + + def modify_stage(default_path: str, updates: dict[str, Any] | None, deletes: dict[str, Any] | None) -> str: kwargs: dict[str, Any] = {} if updates is not None: @@ -293,6 +485,158 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: return s if s else default +def _benchmark_param_id_suffix(param: dict[str, Any], *, idx: int) -> str: + """Derive a readable pytest id suffix from one ``benchmark_params`` entry.""" + name = param.get("name") + if isinstance(name, str) and name.strip(): + return _safe_filename_token(name.strip()) + + parts: list[str] = [] + for key in ("task", "eval_phase", "dataset_name", "dataset"): + value = param.get(key) + if value is None or value == "": + continue + token = _safe_filename_token(str(value)) + if token != "na": + parts.append(token) + if parts: + return "_".join(parts) + + return f"case{idx}" + + +def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> list[str]: + """Return unique pytest id suffixes for a server's benchmark param list.""" + raw = [_benchmark_param_id_suffix(param, idx=idx) for idx, param in enumerate(params_list)] + seen: dict[str, int] = {} + unique: list[str] = [] + for suffix in raw: + count = seen.get(suffix, 0) + seen[suffix] = count + 1 + if count == 0: + unique.append(suffix) + else: + unique.append(f"{suffix}_{count}") + return unique + + +def extract_mark_resource_label(mark_field: Any) -> str: + """Return a filename-safe hardware label from ``mark.hardware_marks.res`` values. + + Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``. + + Prefer :func:`get_runtime_resource_label` for perf result filenames so labels + reflect the machine that actually ran the benchmark. + """ + if isinstance(mark_field, list): + for item in mark_field: + if isinstance(item, dict) and "hardware_marks" in item: + return extract_mark_resource_label(item) + return "na" + if not isinstance(mark_field, dict): + return "na" + hw = mark_field.get("hardware_marks") + if not isinstance(hw, dict): + return "na" + res = hw.get("res") + if not isinstance(res, dict) or not res: + return "na" + labels = [_safe_filename_token(value) for value in res.values()] + return "-".join(labels) if labels else "na" + + +_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = ( + "H100", + "H800", + "H200", + "H20", + "L40S", + "L40", + "L4", + "A100", + "A800", + "A10G", + "A10", + "A30", + "MI325", + "MI300", + "MI250", + "B60", + "S5000", + "910B4", + "910B", + "910", + "310P", + "A2", + "A3", +) +_RUNTIME_RESOURCE_LABEL: str | None = None + + +def _normalize_runtime_device_label(raw: str) -> str: + """Map a platform device name to a short filename-safe resource token.""" + if not raw or not str(raw).strip(): + return "na" + upper = str(raw).upper() + for token in _KNOWN_RUNTIME_RESOURCE_TOKENS: + if token.upper() in upper: + return _safe_filename_token(token) + compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw)) + for prefix in ("NVIDIA", "AMD", "ASCEND", "HUAWEI"): + if compact.upper().startswith(prefix): + compact = compact[len(prefix) :] + break + return _safe_filename_token(compact[:48]) if compact else "na" + + +def _read_runtime_device_name(*, device_id: int = 0) -> str | None: + """Device name from the active Omni platform.""" + if current_omni_platform.device_count() <= device_id: + return None + get_name = getattr(current_omni_platform, "get_device_name", None) + if not callable(get_name): + return None + raw = get_name(device_id) + if isinstance(raw, str) and raw.strip(): + return raw.strip() + return None + + +def get_runtime_resource_label(*, device_id: int = 0, refresh: bool = False) -> str: + """Return a filename-safe hardware label detected on the running machine.""" + global _RUNTIME_RESOURCE_LABEL + if not refresh and _RUNTIME_RESOURCE_LABEL is not None: + return _RUNTIME_RESOURCE_LABEL + raw = _read_runtime_device_name(device_id=device_id) + label = _normalize_runtime_device_label(raw) if raw else "na" + if not refresh: + _RUNTIME_RESOURCE_LABEL = label + return label + + +_FILENAME_OMIT_RESOURCE_LABELS = frozenset({"H100"}) + + +def hardware_json_value(resource_label: str | None) -> str: + """Hardware token stored in perf result JSON (empty when unknown).""" + token = _safe_filename_token(resource_label) + return "" if token == "na" else token + + +def resource_label_for_filename(resource_label: str | None) -> str: + """Hardware token embedded in result filenames (H100 omitted on default CI pool).""" + token = _safe_filename_token(resource_label) + if token in _FILENAME_OMIT_RESOURCE_LABELS: + return "" + return token + + +def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str: + """Return runtime hardware label for perf result filenames.""" + del configs + return get_runtime_resource_label() + + def resolve_baseline_value( baseline_raw: Any, *, @@ -357,6 +701,7 @@ def run_benchmark( max_concurrency: Any | None = None, random_input_len: Any | None = None, random_output_len: Any | None = None, + resource_label: str | None = None, ) -> dict[str, Any]: """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. @@ -367,7 +712,11 @@ def run_benchmark( current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) ro = _safe_filename_token(random_output_len) - result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" + hw = resource_label_for_filename(resource_label) + if hw: + result_filename = f"result_{test_name}_{hw}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" + else: + result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json" if "--result-filename" in args: print(f"The result file will be overwritten by {result_filename}") command = ( @@ -437,6 +786,7 @@ def run_benchmark( result["random_input_len"] = random_input_len if random_output_len is not None: result["random_output_len"] = random_output_len + result["Hardware"] = hardware_json_value(resource_label) with open(result_path, "w", encoding="utf-8") as f: json.dump(result, f, ensure_ascii=False, indent=2) return result diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 13329a209..21167cb07 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -7,21 +7,24 @@ from typing import Any import pytest from tests.dfx.conftest import ( - create_benchmark_indices, + create_paired_omni_benchmark_pytest_params, create_test_parameter_mapping, - create_unique_server_params, get_benchmark_params_for_server, - load_configs, + get_runtime_resource_label, + is_diffusion_perf_config, + load_benchmark_configs, resolve_baseline_value, run_benchmark, ) from tests.helpers.runtime import OmniServer -pytestmark = [pytest.mark.full_model] - # Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline`` # (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are # defined in the same module. +# +# Optional JSON field ``mark`` is applied as pytest marks via +# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks": +# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``). os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -40,22 +43,22 @@ def _get_config_file_from_argv() -> str | None: _PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests" -_DEFAULT_CONFIG_FILE = str(_PERF_TESTS_DIR / "test_qwen_omni.json") CONFIG_FILE_PATH = _get_config_file_from_argv() if CONFIG_FILE_PATH is None: + _all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR) + BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)] print( - "No --test-config-file in argv, using default: tests/dfx/perf/tests/test_qwen_omni.json " - "(override with e.g. --test-config-file tests/dfx/perf/tests/test_tts.json)" + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts case(s) from " + f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; " + f"use -m to filter, e.g. -m tts)" ) - CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE - -BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) - +else: + BENCHMARK_CONFIGS = load_benchmark_configs(CONFIG_FILE_PATH) DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy" -test_params = create_unique_server_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) +paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) _omni_server_lock = threading.Lock() @@ -91,17 +94,11 @@ def omni_server(request): print("OmniServer stopped") -benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) - - @pytest.fixture -def benchmark_params(request, omni_server): - """Benchmark parameters fixture with proper parametrization""" +def benchmark_params(request): + """Benchmark parameters fixture; paired with ``omni_server`` via parametrization.""" test_name, param_index = request.param - if test_name != omni_server.test_name: - pytest.skip(f"Skipping parameter for {test_name} - current server is {omni_server.test_name}") - all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) if not all_params: @@ -153,8 +150,11 @@ def assert_result( @pytest.mark.benchmark -@pytest.mark.parametrize("omni_server", test_params, indirect=True) -@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) +@pytest.mark.parametrize( + "omni_server,benchmark_params", + paired_benchmark_params, + indirect=["omni_server", "benchmark_params"], +) def test_performance_benchmark(omni_server, benchmark_params, request): test_name = benchmark_params["test_name"] params = benchmark_params["params"] @@ -168,6 +168,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): print(f"Benchmark parameters: {benchmark_params}") assert_baseline = request.config.getoption("--assert-baseline", default=False) + resource_label = get_runtime_resource_label() def to_list(value, default=None): if value is None: @@ -239,6 +240,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): max_concurrency=None, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), + resource_label=resource_label, ) assert_result( result, @@ -264,6 +266,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request): max_concurrency=concurrency, random_input_len=params.get("random_input_len"), random_output_len=params.get("random_output_len"), + resource_label=resource_label, ) assert_result( result, diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index b0860978b..d19c0900e 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -8,15 +8,21 @@ This runner separates two concepts: 2. ``benchmark_endpoint``: which serving API the benchmark client calls. Examples: ``/v1/chat/completions`` and ``/v1/videos``. -A config JSON file is REQUIRED via --test-config-file: +A config JSON file may be passed via --test-config-file. If omitted, every ``*.json`` under +``tests/dfx/perf/tests/`` is loaded and pytest ``-m`` filters by each case's ``mark``: + pytest run_diffusion_benchmark.py -m "diffusion" pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off). -All benchmark results for a session are consolidated into a single JSON file under -BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable). -Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params, -timestamp) together with the raw metrics returned by the benchmark script. +Optional JSON field ``mark`` is applied as pytest marks on that case via +``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``). + +All benchmark results are written under BENCHMARK_RESULT_DIR (override via the +DIFFUSION_BENCHMARK_DIR environment variable). Each source JSON file gets one +aggregated ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` (JSON array +of all runs from cases in that file). Bulk load without ``--test-config-file`` uses +the same per-file aggregation; ``-m`` only selects which cases run. """ import json @@ -35,8 +41,15 @@ import psutil import pytest from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint - -pytestmark = [pytest.mark.diffusion, pytest.mark.full_model, pytest.mark.local_model] +from tests.dfx.conftest import ( + create_paired_benchmark_pytest_params, + get_runtime_resource_label, + hardware_json_value, + is_diffusion_perf_config, + resolve_pytest_marks, + resource_label_for_filename, +) +from tests.helpers.runtime import get_open_port os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" os.environ.setdefault("DIFFUSION_ATTENTION_BACKEND", "FLASH_ATTN") @@ -126,6 +139,10 @@ _BRANCHPOINT_COMMIT_SHA: str | None = None DIFFUSION_RESULT_TEMPLATE_PATH = Path(__file__).parent / "diffusion_result_template.json" +_DIFFUSION_SOURCE_CONFIG_KEY = "_source_config_file" +_PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests" + + def _get_config_file_from_argv() -> str | None: """Read --test-config-file from sys.argv at import time so pytest parametrize can use it. @@ -142,9 +159,6 @@ def _get_config_file_from_argv() -> str | None: CONFIG_FILE_PATH = _get_config_file_from_argv() -if CONFIG_FILE_PATH is None: - print("No config file provided, using default config file: tests/dfx/perf/tests/test_qwen_image_vllm_omni.json") - CONFIG_FILE_PATH = "tests/dfx/perf/tests/test_qwen_image_vllm_omni.json" # --------------------------------------------------------------------------- # Config loading @@ -184,28 +198,85 @@ def load_configs(config_path: str) -> list[dict[str, Any]]: raise RuntimeError(f"Failed to load configuration file: {str(e)}") -BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) - -_config_stem = Path(CONFIG_FILE_PATH).stem # e.g. "test_qwen_image_vllm_omni" -AGGREGATED_RESULT_FILE = BENCHMARK_RESULT_DIR / f"diffusion_result_{_config_stem}_{_SESSION_TIMESTAMP}.json" +def load_diffusion_benchmark_configs( + config_path: str | None = None, + *, + config_dir: Path | None = None, +) -> list[dict[str, Any]]: + """Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*.""" + if config_path is not None: + configs = load_configs(config_path) + source = str(Path(config_path).resolve()) + for cfg in configs: + cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source) + return configs + if config_dir is None: + raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir") + configs: list[dict[str, Any]] = [] + for path in sorted(config_dir.glob("*.json")): + source = str(path.resolve()) + for cfg in load_configs(str(path)): + cfg[_DIFFUSION_SOURCE_CONFIG_KEY] = source + configs.append(cfg) + if not configs: + raise ValueError(f"No benchmark JSON files found under {config_dir}") + return configs -def _append_to_aggregated_file(record: dict[str, Any]) -> None: - """Thread-safe append of *record* to the session-level aggregated JSON file. +if CONFIG_FILE_PATH is None: + _all_configs = load_diffusion_benchmark_configs(config_dir=_PERF_TESTS_DIR) + BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if is_diffusion_perf_config(cfg)] + print( + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s) from " + f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} omni/tts; " + f"use -m to filter, e.g. -m diffusion)" + ) +else: + BENCHMARK_CONFIGS = load_diffusion_benchmark_configs(CONFIG_FILE_PATH) - The file contains a JSON array; each call loads the existing array (or - starts a new one), appends the record, and writes the file back atomically. - """ +_AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {} + + +def _normalized_source_path(source_file: str) -> str: + return str(Path(source_file).resolve()) + + +def _aggregated_result_file_for_source(source_file: str) -> Path: + """One session aggregate per source JSON (same naming as single ``--test-config-file``).""" + key = _normalized_source_path(source_file) + if key not in _AGGREGATED_RESULT_FILES_BY_SOURCE: + stem = Path(key).stem + resource = resource_label_for_filename(get_runtime_resource_label()) + if resource: + result_name = f"diffusion_result_{stem}_{resource}_{_SESSION_TIMESTAMP}.json" + else: + result_name = f"diffusion_result_{stem}_{_SESSION_TIMESTAMP}.json" + _AGGREGATED_RESULT_FILES_BY_SOURCE[key] = BENCHMARK_RESULT_DIR / result_name + return _AGGREGATED_RESULT_FILES_BY_SOURCE[key] + + +def _write_result_record(record: dict[str, Any]) -> Path: + """Append one benchmark record to the aggregate file for its source JSON.""" + source_file = record.get("source_file") or CONFIG_FILE_PATH + if not source_file: + raise ValueError("benchmark record missing source_file") + target = _aggregated_result_file_for_source(str(source_file)) with _RESULT_LOCK: BENCHMARK_RESULT_DIR.mkdir(parents=True, exist_ok=True) - if AGGREGATED_RESULT_FILE.exists(): - with open(AGGREGATED_RESULT_FILE, encoding="utf-8") as f: + if target.exists(): + with open(target, encoding="utf-8") as f: records: list[dict] = json.load(f) else: records = [] records.append(record) - with open(AGGREGATED_RESULT_FILE, "w", encoding="utf-8") as f: + with open(target, "w", encoding="utf-8") as f: json.dump(records, f, indent=2, ensure_ascii=False) + return target + + +def _append_to_aggregated_file(record: dict[str, Any]) -> None: + """Backward-compatible wrapper; prefer :func:`_write_result_record`.""" + _write_result_record(record) _server_lock = threading.Lock() @@ -215,14 +286,6 @@ _server_lock = threading.Lock() # --------------------------------------------------------------------------- -def _get_open_port() -> int: - """Return an available TCP port on localhost.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return s.getsockname()[1] - - def _wait_for_port(host: str, port: int, timeout: int = 1200, proc: subprocess.Popen | None = None) -> None: """Block until the given host:port accepts connections or timeout expires. @@ -312,7 +375,7 @@ class DiffusionServer: self.model = server_cfg["model"] self.serve_args = server_cfg["serve_args"] self.host = "127.0.0.1" - self.port = port if port is not None else _get_open_port() + self.port = port if port is not None else get_open_port(self.host) self.proc: subprocess.Popen | None = None self.test_name: str = "" @@ -482,6 +545,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]] "serve_args": _build_serve_args(serve_args_dict), "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")), "server_params": cfg["server_params"], + "mark": cfg.get("mark"), } ) return result @@ -496,19 +560,29 @@ def _test_param_mapping(configs: list[dict[str, Any]]) -> dict[str, list[dict]]: return mapping -def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer: - """Factory: return a vLLM-Omni diffusion server instance for the config.""" - return DiffusionServer(server_cfg=server_cfg) +def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: + return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs} + + +def _paired_diffusion_benchmark_pytest_params(configs: list[dict[str, Any]]) -> list[Any]: + """Paired params for ``run_diffusion_benchmark.py``; same shape as omni runner.""" + test_param_map = _test_param_mapping(configs) + server_entries = [(cfg, cfg["test_name"]) for cfg in _unique_server_params(configs)] + return create_paired_benchmark_pytest_params(server_entries, test_param_map, _marks_by_test_name(configs)) # --------------------------------------------------------------------------- # Parametrize data # --------------------------------------------------------------------------- -server_params = _unique_server_params(BENCHMARK_CONFIGS) test_param_map = _test_param_mapping(BENCHMARK_CONFIGS) +paired_benchmark_params = _paired_diffusion_benchmark_pytest_params(BENCHMARK_CONFIGS) + + +def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer: + """Factory: return a vLLM-Omni diffusion server instance for the config.""" + return DiffusionServer(server_cfg=server_cfg) -benchmark_indices: list[int] = list(range(max(len(v) for v in test_param_map.values()))) # --------------------------------------------------------------------------- # Pytest fixtures @@ -534,16 +608,13 @@ def diffusion_server(request): @pytest.fixture -def benchmark_params(request, diffusion_server): - """Yield the benchmark params dict for the current (server, index) pair.""" - param_index: int = request.param - test_name = diffusion_server.test_name +def benchmark_params(request): + """Benchmark params for the paired server/index parametrization.""" + test_name, param_index = request.param params_list = test_param_map.get(test_name, []) if not params_list: raise ValueError(f"No benchmark params for test: {test_name}") - if param_index >= len(params_list): - pytest.skip(f"Param index {param_index} out of range for {test_name} (has {len(params_list)} params)") current = param_index + 1 total = len(params_list) @@ -575,8 +646,8 @@ def run_benchmark( The raw metrics are written to a temporary file by the subprocess. After the run completes the metrics are merged with full metadata (test_name, endpoint, benchmark_params, timestamp, flat reporting fields) and appended - to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE). The - temporary file is removed afterwards. Subprocess stdout/stderr are tee'd + to ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` for the + source JSON file. The temporary file is removed afterwards. Subprocess stdout/stderr are tee'd to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in the record. """ @@ -586,7 +657,12 @@ def run_benchmark( log_dir = BENCHMARK_RESULT_DIR / "logs" log_dir.mkdir(parents=True, exist_ok=True) endpoint_label = endpoint_filename_token(endpoint) - log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log" + resource_label = get_runtime_resource_label() + hw_for_filename = resource_label_for_filename(resource_label) + if hw_for_filename: + log_file = log_dir / f"{test_name}_{hw_for_filename}_{endpoint_label}_{timestamp}.log" + else: + log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log" with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) @@ -696,7 +772,7 @@ def run_benchmark( "Model": model, "Framework": server_type, "API Endpoint": endpoint, - "Hardware": "", + "Hardware": hardware_json_value(resource_label), "Deployment": "", "Task": params.get("task", "t2i"), "Dataset": params.get("dataset", "random"), @@ -728,8 +804,8 @@ def run_benchmark( "build_url": os.environ.get("BUILDKITE_BUILD_URL", ""), "source_file": source_file, } - _append_to_aggregated_file(record) - print(f"\n Result appended to: {AGGREGATED_RESULT_FILE}") + result_path = _write_result_record(record) + print(f"\n Result saved to: {result_path}") print(f" Log saved to: {log_file}") return metrics @@ -932,12 +1008,10 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: # --------------------------------------------------------------------------- @pytest.mark.benchmark @pytest.mark.parametrize( - "diffusion_server", - server_params, - ids=[p["test_name"] for p in server_params], - indirect=True, + "diffusion_server,benchmark_params", + paired_benchmark_params, + indirect=["diffusion_server", "benchmark_params"], ) -@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request): """Run the diffusion performance benchmark and verify request completion. @@ -962,7 +1036,10 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req test_name=test_name, endpoint=endpoint, server_cfg=server_cfg, - source_file=cast(str, CONFIG_FILE_PATH), + source_file=server_cfg.get( + _DIFFUSION_SOURCE_CONFIG_KEY, + CONFIG_FILE_PATH or f"{_PERF_TESTS_DIR}/*.json", + ), ) print(f"\n{'=' * 60}") @@ -980,7 +1057,9 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req if key in result: print(f" {key}: {result[key]:.4f}") - print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}") + source = server_cfg.get(_DIFFUSION_SOURCE_CONFIG_KEY) or CONFIG_FILE_PATH + if source: + print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}") print("=" * 60) assert_baseline = request.config.getoption("--assert-baseline", default=False) diff --git a/tests/dfx/perf/tests/test_bagel_vllm_omni.json b/tests/dfx/perf/tests/test_bagel_vllm_omni.json index 09fc96030..303839387 100644 --- a/tests/dfx/perf/tests/test_bagel_vllm_omni.json +++ b/tests/dfx/perf/tests/test_bagel_vllm_omni.json @@ -1,6 +1,18 @@ [ { "test_name": "test_bagel_single_device_single_stage_t2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Single-stage BAGEL (TP=1, CFG=1), t2i Task", "server_type": "vllm-omni", "server_params": { @@ -38,6 +50,18 @@ }, { "test_name": "test_bagel_single_device_single_stage_i2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Single-stage BAGEL (TP=1, CFG=1), i2i Task", "server_type": "vllm-omni", "server_params": { @@ -76,6 +100,18 @@ }, { "test_name": "test_bagel_single_device_multi_stage_t2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, t2i Task", "server_type": "vllm-omni", "server_params": { @@ -113,6 +149,18 @@ }, { "test_name": "test_bagel_single_device_multi_stage_i2i", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], "description": "Two-stage BAGEL (bagel.yaml), TP=1, CFG=1, i2i Task", "server_type": "vllm-omni", "server_params": { diff --git a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json index b0fc9a5a4..9bf31658d 100644 --- a/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json +++ b/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -1,213 +1,261 @@ [ - { - "test_name": "test_cosmos3_t2i_official_demo_2gpu", - "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/images/generations", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps4", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 4, - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "extra-body": { - "guidance_scale": 7.0, - "flow_shift": 3.0, - "guardrails": false - }, - "baseline": { - "throughput_qps": 1.107, - "latency_mean": 0.8929, - "peak_memory_mb_mean": 80000.0 - } - } - ] + { + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/generations", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_cosmos3_t2v_official_demo_2gpu", - "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1024x1024_steps4", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 4, + "num-prompts": 3, + "max-concurrency": 1, + "seed": 42, + "extra-body": { + "guidance_scale": 7.0, + "flow_shift": 3.0, + "guardrails": false }, - "benchmark_params": [ - { - "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "t2v", - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "A robot arm is cleaning a plate in the kitchen.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { - "guidance_scale": 6.0, - "flow_shift": 10.0, - "max_sequence_length": 4096, - "guardrails": false, - "use_resolution_template": false, - "use_duration_template": false - }, - "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.6824, - "peak_memory_mb_mean": 30155.4 - } - } - ] + "baseline": { + "throughput_qps": 1.107, + "latency_mean": 0.8929, + "peak_memory_mb_mean": 80000.0 + } + } + ] + }, + { + "test_name": "test_cosmos3_t2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_cosmos3_i2v_official_demo_2gpu", - "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset": "random", + "task": "t2v", + "num-prompts": 3, + "max-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "prompt": "A robot arm is cleaning a plate in the kitchen.", + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false, + "use_resolution_template": false, + "use_duration_template": false }, - "benchmark_params": [ - { - "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 3, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "The scene comes to life with smooth, natural motion.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { - "guidance_scale": 6.0, - "flow_shift": 10.0, - "max_sequence_length": 4096, - "guardrails": false - }, - "baseline": { - "throughput_qps": 0.027, - "latency_mean": 36.968, - "peak_memory_mb_mean": 30443.6 - } - } - ] + "baseline": { + "throughput_qps": 0.027, + "latency_mean": 36.6824, + "peak_memory_mb_mean": 30155.4 + } + } + ] + }, + { + "test_name": "test_cosmos3_i2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_cosmos3_v2v_official_demo_2gpu", - "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/videos", - "server_params": { - "model": "nvidia/Cosmos3-Nano", - "serve_args": { - "cfg-parallel-size": 2, - "use-hsdp": true, - "hsdp-shard-size": 2, - "vae-patch-parallel-size": 2, - "vae-use-tiling": true, - "model-class-name": "Cosmos3OmniDiffusersPipeline", - "no-guardrails": true, - "stage-init-timeout": 900, - "init-timeout": 1200, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 3, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "prompt": "The scene comes to life with smooth, natural motion.", + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false }, - "benchmark_params": [ - { - "name": "1280x720_frames189_steps4", - "dataset": "random", - "task": "v2v", - "num-prompts": 3, - "max-concurrency": 1, - "seed": 42, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "prompt": "Continue the same scene with smooth natural motion.", - "num_inference_steps": 4, - "num_frames": 189, - "fps": 24, - "weight": 1 - } - ], - "extra-body": { - "guidance_scale": 6.0, - "flow_shift": 10.0, - "max_sequence_length": 4096, - "guardrails": false, - "condition_frame_indexes_vision": [ - 0, - 1 - ], - "condition_video_keep": "first" - }, - "baseline": { - "throughput_qps": 0.027, - "latency_mean": 37.2117, - "peak_memory_mb_mean": 30135.6 - } - } - ] - } + "baseline": { + "throughput_qps": 0.027, + "latency_mean": 36.968, + "peak_memory_mb_mean": 30443.6 + } + } + ] + }, + { + "test_name": "test_cosmos3_v2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset": "random", + "task": "v2v", + "num-prompts": 3, + "max-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "prompt": "Continue the same scene with smooth natural motion.", + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false, + "condition_frame_indexes_vision": [ + 0, + 1 + ], + "condition_video_keep": "first" + }, + "baseline": { + "throughput_qps": 0.027, + "latency_mean": 37.2117, + "peak_memory_mb_mean": 30135.6 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_higgs_audio_v3.json b/tests/dfx/perf/tests/test_higgs_audio_v3.json index 6fdaec580..9b7b2eb5c 100644 --- a/tests/dfx/perf/tests/test_higgs_audio_v3.json +++ b/tests/dfx/perf/tests/test_higgs_audio_v3.json @@ -1,6 +1,18 @@ [ { "test_name": "test_higgs_audio_v3", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "bosonai/higgs-audio-v3-tts-4b", "stage_overrides": { diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index fa955c763..1b6902cba 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -1,140 +1,152 @@ [ - { - "test_name": "test_hunyuan_image3_ti2i_inline", - "description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config", - "server_type": "vllm-omni", - "benchmark_endpoint": "/v1/images/edits", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "trust-remote-code": true, - "deploy-config-inline": { - "type": "yaml", - "content": { - "pipeline": "hunyuan_image_3_moe", - "async_chunk": false, - "trust_remote_code": true, - "connectors": { - "rdma_connector": { - "name": "MooncakeTransferEngineConnector", - "extra": { - "host": "auto", - "zmq_port": 50051, - "protocol": "rdma", - "device_name": "", - "memory_pool_size": 17179869184, - "memory_pool_device": "cpu" - } - } - }, - "stages": [ - { - "stage_id": 0, - "is_comprehension": true, - "final_output": true, - "final_output_type": "text", - "max_num_seqs": 16, - "gpu_memory_utilization": 0.95, - "enforce_eager": false, - "enable_prefix_caching": false, - "max_num_batched_tokens": 32768, - "devices": "0,1", - "tensor_parallel_size": 2, - "hf_overrides": { - "rope_parameters": { - "mrope_section": [ - 0, - 32, - 32 - ], - "rope_type": "default" - } - }, - "omni_kv_config": { - "need_send_cache": true - }, - "output_connectors": { - "to_stage_1": "rdma_connector" - }, - "default_sampling_params": { - "temperature": 0.0, - "top_p": 1, - "top_k": -1, - "max_tokens": 8192, - "detokenize": true, - "skip_special_tokens": false, - "stop_token_ids": [ - 128025 - ] - } - }, - { - "stage_id": 1, - "max_num_seqs": 1, - "enforce_eager": true, - "devices": "2,3", - "distributed_executor_backend": "mp", - "omni_kv_config": { - "need_recv_cache": true - }, - "parallel_config": { - "tensor_parallel_size": 2, - "vae_patch_parallel_size": 2, - "enable_expert_parallel": true - }, - "input_connectors": { - "from_stage_0": "rdma_connector" - }, - "default_sampling_params": { - "num_inference_steps": 50, - "guidance_scale": 0 - } - } - ], - "edges": [ - { - "from": 0, - "to": 1, - "window_size": -1, - "max_inflight": 1 - } - ] - } - }, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "inline_dataset_5_requests", - "dataset": "custom", - "dataset-path-inline": { - "type": "jsonl", - "content": [ - { - "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。", - "image_urls": [ - "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png" - ], - "num_inference_steps": 8, - "seed": 42, - "guidance_scale": 1.0 - } - ] - }, - "task": "ti2i", - "bot-task": "think_recaption", - "max-concurrency": 16, - "num-prompts": 96, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.1874, - "latency_p99": 101.852, - "latency_mean": 79.0385, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + { + "test_name": "test_hunyuan_image3_ti2i_inline", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "AR (TP=2) + 2 * DiT (TP=2) - using inline dataset and deploy-config", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/images/edits", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "trust-remote-code": true, + "deploy-config-inline": { + "type": "yaml", + "content": { + "pipeline": "hunyuan_image_3_moe", + "async_chunk": false, + "trust_remote_code": true, + "connectors": { + "rdma_connector": { + "name": "MooncakeTransferEngineConnector", + "extra": { + "host": "auto", + "zmq_port": 50051, + "protocol": "rdma", + "device_name": "", + "memory_pool_size": 17179869184, + "memory_pool_device": "cpu" } + } + }, + "stages": [ + { + "stage_id": 0, + "is_comprehension": true, + "final_output": true, + "final_output_type": "text", + "max_num_seqs": 16, + "gpu_memory_utilization": 0.95, + "enforce_eager": false, + "enable_prefix_caching": false, + "max_num_batched_tokens": 32768, + "devices": "0,1", + "tensor_parallel_size": 2, + "hf_overrides": { + "rope_parameters": { + "mrope_section": [ + 0, + 32, + 32 + ], + "rope_type": "default" + } + }, + "omni_kv_config": { + "need_send_cache": true + }, + "output_connectors": { + "to_stage_1": "rdma_connector" + }, + "default_sampling_params": { + "temperature": 0.0, + "top_p": 1, + "top_k": -1, + "max_tokens": 8192, + "detokenize": true, + "skip_special_tokens": false, + "stop_token_ids": [ + 128025 + ] + } + }, + { + "stage_id": 1, + "max_num_seqs": 1, + "enforce_eager": true, + "devices": "2,3", + "distributed_executor_backend": "mp", + "omni_kv_config": { + "need_recv_cache": true + }, + "parallel_config": { + "tensor_parallel_size": 2, + "vae_patch_parallel_size": 2, + "enable_expert_parallel": true + }, + "input_connectors": { + "from_stage_0": "rdma_connector" + }, + "default_sampling_params": { + "num_inference_steps": 50, + "guidance_scale": 0 + } + } + ], + "edges": [ + { + "from": 0, + "to": 1, + "window_size": -1, + "max_inflight": 1 + } + ] + } + }, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "inline_dataset_5_requests", + "dataset": "custom", + "dataset-path-inline": { + "type": "jsonl", + "content": [ + { + "prompt": "新年宠物海报，Q版圆润的可爱标题“新年快乐汪”，副标题“HAPPY NEW YEAR”。 鱼眼镜头，背景是房间门口，近景，上传的主体歪头笑，围着红色围巾，戴着红色毛线帽，高清，绒毛细节，面部特写。 宝丽莱相纸，超现实主义，写实主义，胶片摄影，打印颗粒感肌理。肌理，超写实，复古感。", + "image_urls": [ + "https://raw.githubusercontent.com/Tencent-Hunyuan/HunyuanImage-3.0/main/assets/demo_instruct_imgs/input_0_0.png" + ], + "num_inference_steps": 8, + "seed": 42, + "guidance_scale": 1.0 } - ] - } + ] + }, + "task": "ti2i", + "bot-task": "think_recaption", + "max-concurrency": 16, + "num-prompts": 96, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.1874, + "latency_p99": 101.852, + "latency_mean": 79.0385, + "peak_memory_mb_max": 70000, + "peak_memory_mb_mean": 70000 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index e6d7ce659..7a699f53a 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -1,35 +1,47 @@ [ - { - "test_name": "test_hunyuan_image_tp2_cfgp2", - "description": "TP=2 CfgP=2 baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", - "tensor-parallel-size": 2, - "cfg-parallel-size": 2, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps8", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 8, - "num-prompts": 10, - "max-concurrency": 1, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.21, - "latency_p99": 4.7469, - "peak_memory_mb_max": 101100, - "peak_memory_mb_mean": 101100 - } - } - ] - } + { + "test_name": "test_hunyuan_image_tp2_cfgp2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "TP=2 CfgP=2 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy_config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "cfg-parallel-size": 2, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps8", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 8, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.21, + "latency_p99": 4.7469, + "peak_memory_mb_max": 101100, + "peak_memory_mb_mean": 101100 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index fbbe1643e..521febd2d 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -1,35 +1,47 @@ [ - { - "test_name": "test_hunyuan_image_tp2_sp2", - "description": "TP=2 SP=2 baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", - "tensor-parallel-size": 2, - "ulysses-degree": 2, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps8", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 8, - "num-prompts": 10, - "max-concurrency": 1, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.20, - "latency_p99": 5.1025, - "peak_memory_mb_max": 97402, - "peak_memory_mb_mean": 97402 - } - } - ] - } + { + "test_name": "test_hunyuan_image_tp2_sp2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "TP=2 SP=2 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "ulysses-degree": 2, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps8", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 8, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.20, + "latency_p99": 5.1025, + "peak_memory_mb_max": 97402, + "peak_memory_mb_mean": 97402 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index eb37f7a22..f2d05564b 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -1,35 +1,47 @@ [ - { - "test_name": "test_hunyuan_image_tp4", - "description": "TP=4 baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "tencent/HunyuanImage-3.0-Instruct", - "serve_args": { - "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", - "tensor-parallel-size": 4, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1024x1024_steps8", - "dataset": "random", - "task": "t2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 8, - "num-prompts": 10, - "max-concurrency": 1, - "skip-performance-assertion": true, - "baseline": { - "throughput_qps": 0.213, - "latency_mean": 4.6953, - "latency_p99": 4.8601, - "peak_memory_mb_max": 56912.0, - "peak_memory_mb_mean": 56912.0 - } - } - ] - } + { + "test_name": "test_hunyuan_image_tp4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "diffusion", + "local_model" + ], + "description": "TP=4 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "../vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 4, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps8", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 8, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.213, + "latency_mean": 4.6953, + "latency_p99": 4.8601, + "peak_memory_mb_max": 56912.0, + "peak_memory_mb_mean": 56912.0 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json index 4a6f9e350..1a0f31e98 100644 --- a/tests/dfx/perf/tests/test_ltx2_vllm_omni.json +++ b/tests/dfx/perf/tests/test_ltx2_vllm_omni.json @@ -1,217 +1,273 @@ [ - { - "test_name": "test_ltx2_baseline_eager", - "description": "Single-device baseline with enforce-eager (no torch.compile)", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "enforce-eager": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + { + "test_name": "test_ltx2_baseline_eager", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline with enforce-eager (no torch.compile)", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_torch_compile", - "description": "Single-device with torch.compile (default, no enforce-eager)", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_torch_compile", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device with torch.compile (default, no enforce-eager)", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_cfg2_eager", - "description": "CFG-parallel=2 with enforce-eager", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "cfg-parallel-size": 2, - "enforce-eager": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_cfg2_eager", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CFG-parallel=2 with enforce-eager", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "cfg-parallel-size": 2, + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_cfg2_compile", - "description": "CFG-parallel=2 with torch.compile", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "cfg-parallel-size": 2, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_cfg2_compile", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CFG-parallel=2 with torch.compile", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "cfg-parallel-size": 2, + "enable-diffusion-pipeline-profiler": true + } }, - - { - "test_name": "test_ltx2_cache_dit_eager", - "description": "CacheDiT with enforce-eager", - "server_type": "vllm-omni", - "server_params": { - "model": "Lightricks/LTX-2", - "serve_args": { - "cache-backend": "cache_dit", - "enforce-eager": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "256x256_145f_steps6", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 256, - "height": 256, - "num-frames": 145, - "fps": 24, - "num-inference-steps": 6, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - }, - { - "name": "480x768_41f_steps20", - "dataset": "random", - "task": "t2v", - "backend": "v1/videos", - "width": 768, - "height": 480, - "num-frames": 41, - "fps": 24, - "num-inference-steps": 20, - "num-prompts": 3, - "max-concurrency": 1, - "enable-negative-prompt": true - } - ] - } + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_ltx2_cache_dit_eager", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "CacheDiT with enforce-eager", + "server_type": "vllm-omni", + "server_params": { + "model": "Lightricks/LTX-2", + "serve_args": { + "cache-backend": "cache_dit", + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "256x256_145f_steps6", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 256, + "height": 256, + "num-frames": 145, + "fps": 24, + "num-inference-steps": 6, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "480x768_41f_steps20", + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "width": 768, + "height": 480, + "num-frames": 41, + "fps": 24, + "num-inference-steps": 20, + "num-prompts": 3, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 459554f63..66ffa231f 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -1,331 +1,343 @@ [ - { - "test_name": "test_qwen3_omni_chunk", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": [ - "--async-chunk" - ] + { + "test_name": "test_qwen3_omni_chunk", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 135.4379, + 187.0896, + 400, + 435.675, + 724.6836 + ], + "mean_e2el_ms": [ + 17619.9851, + 34811.8335, + 43656.9501, + 56657.1548, + 85097.6016 + ], + "mean_audio_rtf": [ + 0.1279, + 0.2238, + 0.2735, + 0.3712, + 0.5634 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 135.4379, - 187.0896, - 400, - 435.675, - 724.6836 - ], - "mean_e2el_ms": [ - 17619.9851, - 34811.8335, - 43656.9501, - 56657.1548, - 85097.6016 - ], - "mean_audio_rtf": [ - 0.1279, - 0.2238, - 0.2735, - 0.3712, - 0.5634 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 340.329 - ], - "mean_e2el_ms": [ - 4305.477 - ], - "mean_audio_rtf": [ - 0.1305 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 220.0909 - ], - "mean_e2el_ms": [ - 5475.9565 - ], - "mean_audio_rtf": [ - 0.1818 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 450.0155 - ], - "mean_e2el_ms": [ - 8516.8316 - ], - "mean_audio_rtf": [ - 0.2717 - ] - } - }, - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 78.4278, - 195.8037, - 333.7318, - 561.805, - 1038.043 - ], - "mean_e2el_ms": [ - 4288.3029, - 7429.9429, - 9451.6184, - 12607.5292, - 18153.2547 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 275.3875 - ], - "mean_e2el_ms": [ - 730.6995 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 166.2201 - ], - "mean_e2el_ms": [ - 673.5789 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 357.7781 - ], - "mean_e2el_ms": [ - 953.3493 - ] - } - } - ] - } + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 340.329 + ], + "mean_e2el_ms": [ + 4305.477 + ], + "mean_audio_rtf": [ + 0.1305 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 220.0909 + ], + "mean_e2el_ms": [ + 5475.9565 + ], + "mean_audio_rtf": [ + 0.1818 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 450.0155 + ], + "mean_e2el_ms": [ + 8516.8316 + ], + "mean_audio_rtf": [ + 0.2717 + ] + } + }, + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 78.4278, + 195.8037, + 333.7318, + 561.805, + 1038.043 + ], + "mean_e2el_ms": [ + 4288.3029, + 7429.9429, + 9451.6184, + 12607.5292, + 18153.2547 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 275.3875 + ], + "mean_e2el_ms": [ + 730.6995 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 166.2201 + ], + "mean_e2el_ms": [ + 673.5789 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 357.7781 + ], + "mean_e2el_ms": [ + 953.3493 + ] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json index bee799a77..ac6f679c1 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json @@ -1,115 +1,127 @@ [ - { - "test_name": "test_qwen3_omni_3gpu_replica2", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "stage_overrides": { - "1": {"num_replicas": 2, "devices": "1,2"}, - "2": {"num_replicas": 2, "devices": "1,2"} - }, - "extra_cli_args": [ - "--async-chunk" - ] + { + "test_name": "test_qwen3_omni_3gpu_replica2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 3 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "stage_overrides": { + "1": {"num_replicas": 2, "devices": "1,2"}, + "2": {"num_replicas": 2, "devices": "1,2"} + }, + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [128, 128, 128, 128], + "max_concurrency": [8, 16, 24, 32], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [203, 283, 470, 869], + "mean_tpot_ms": [28, 36, 41, 45], + "mean_audio_ttfp_ms": [861, 1200, 1528, 2100], + "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [10], + "request_rate": [0.1], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [128, 128, 128, 128], - "max_concurrency": [8, 16, 24, 32], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [203, 283, 470, 869], - "mean_tpot_ms": [28, 36, 41, 45], - "mean_audio_ttfp_ms": [861, 1200, 1528, 2100], - "mean_audio_rtf": [0.24, 0.30, 0.35, 0.39] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [377], - "mean_audio_ttfp_ms": [684], - "mean_audio_rtf": [0.14] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [228], - "mean_audio_ttfp_ms": [686], - "mean_audio_rtf": [0.16] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [522], - "mean_audio_ttfp_ms": [1119], - "mean_audio_rtf": [0.21] - } - } - ] - } + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [377], + "mean_audio_ttfp_ms": [684], + "mean_audio_rtf": [0.14] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [40], + "request_rate": [0.5], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [228], + "mean_audio_ttfp_ms": [686], + "mean_audio_rtf": [0.16] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [100], + "request_rate": [1.0], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [522], + "mean_audio_ttfp_ms": [1119], + "mean_audio_rtf": [0.21] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index d01ca06b4..78daf691b 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -1,170 +1,182 @@ [ - { - "test_name": "test_qwen3_omni", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": [ - "--no-async-chunk" - ] + { + "test_name": "test_qwen3_omni", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--no-async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 109.1218, + 247.4411, + 354.9384, + 447.7885, + 694.7487 + ], + "mean_e2el_ms": [ + 22413.667, + 30808.9715, + 35677.844, + 46345.8978, + 68350.4853 + ], + "mean_audio_rtf": [ + 0.1695, + 0.2033, + 0.2467, + 0.3234, + 0.4438 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 109.1218, - 247.4411, - 354.9384, - 447.7885, - 694.7487 - ], - "mean_e2el_ms": [ - 22413.667, - 30808.9715, - 35677.844, - 46345.8978, - 68350.4853 - ], - "mean_audio_rtf": [ - 0.1695, - 0.2033, - 0.2467, - 0.3234, - 0.4438 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 323.3573 - ], - "mean_e2el_ms": [ - 4623.4173 - ], - "mean_audio_rtf": [ - 0.1406 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 215.4502 - ], - "mean_e2el_ms": [ - 4924.149 - ], - "mean_audio_rtf": [ - 0.1637 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [ - 442.5815 - ], - "mean_e2el_ms": [ - 6441.0986 - ], - "mean_audio_rtf": [ - 0.2091 - ] - } - } - ] - } + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 323.3573 + ], + "mean_e2el_ms": [ + 4623.4173 + ], + "mean_audio_rtf": [ + 0.1406 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 215.4502 + ], + "mean_e2el_ms": [ + 4924.149 + ], + "mean_audio_rtf": [ + 0.1637 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [ + 442.5815 + ], + "mean_e2el_ms": [ + 6441.0986 + ], + "mean_audio_rtf": [ + 0.2091 + ] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index f608eca0e..b9d553098 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -1,180 +1,192 @@ [ - { - "test_name": "test_qwen3_omni_vllm_text", - "server_params": { - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "use_omni": false, - "extra_cli_args": [ - "--no-enable-prefix-caching", - "--skip-mm-profiling" - ] + { + "test_name": "test_qwen3_omni_vllm_text", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "use_omni": false, + "extra_cli_args": [ + "--no-enable-prefix-caching", + "--skip-mm-profiling" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] }, - "benchmark_params": [ - { - "dataset_name": "random", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 4, - 16, - 32, - 64, - 128 - ], - "max_concurrency": [ - 1, - 4, - 8, - 16, - 32 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 69.1276, - 255.1882, - 310.639, - 422.0337, - 532.0582 - ], - "mean_e2el_ms": [ - 4382.7211, - 7510.3815, - 9440.4854, - 12553.0186, - 16932.6065 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 10 - ], - "request_rate": [ - 0.1 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 1, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "audio": 1 - }, - "random_mm_bucket_config": { - "(0, 60, 3)": 1.0 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 26.3381 - ], - "mean_e2el_ms": [ - 485.2436 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 40 - ], - "request_rate": [ - 0.5 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 2, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.5, - "(720, 1280, 2)": 0.5 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 26.4517 - ], - "mean_e2el_ms": [ - 517.2017 - ] - } - }, - { - "dataset_name": "random-mm", - "backend": "vllm", - "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "endpoint": "/v1/completions", - "num_prompts": [ - 100 - ], - "request_rate": [ - 1.0 - ], - "random_input_len": 100, - "random_output_len": 100, - "random_range_ratio": 0.0, - "ignore_eos": true, - "extra_body": { - "modalities": [ - "text" - ] - }, - "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0.5, - "random_mm_limit_mm_per_prompt": { - "image": 1, - "video": 1, - "audio": 1 - }, - "random_mm_bucket_config": { - "(256, 256, 1)": 0.34, - "(720, 1280, 2)": 0.33, - "(0, 60, 3)": 0.33 - }, - "percentile-metrics": "ttft,tpot,itl,e2el", - "baseline": { - "mean_ttft_ms": [ - 27.6972 - ], - "mean_e2el_ms": [ - 561.3684 - ] - } - } - ] - } + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 69.1276, + 255.1882, + 310.639, + 422.0337, + 532.0582 + ], + "mean_e2el_ms": [ + 4382.7211, + 7510.3815, + 9440.4854, + 12553.0186, + 16932.6065 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 26.3381 + ], + "mean_e2el_ms": [ + 485.2436 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 26.4517 + ], + "mean_e2el_ms": [ + 517.2017 + ] + } + }, + { + "dataset_name": "random-mm", + "backend": "vllm", + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "endpoint": "/v1/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [ + 27.6972 + ], + "mean_e2el_ms": [ + 561.3684 + ] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 118fa6733..53e13daed 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -1,149 +1,185 @@ [ - { - "test_name": "test_qwen_image_edit_2511_single_device", - "description": "Single-device baseline (two input images)", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit-2511", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0721, - "latency_mean": 13.8398, - "peak_memory_mb_max": 54731.4, - "peak_memory_mb_mean": 54731.4 - } - }, - { - "name": "1536x1536_steps35_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0186, - "latency_mean": 53.6942, - "peak_memory_mb_max": 64402.4, - "peak_memory_mb_mean": 64402.4 - } - } - ] + { + "test_name": "test_qwen_image_edit_2511_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (two input images)", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit-2511", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4", - "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit-2511", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0538, - "latency_mean": 18.5591, - "peak_memory_mb_max": 53485.0, - "peak_memory_mb_mean": 53485.0 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0721, + "latency_mean": 13.8398, + "peak_memory_mb_max": 54731.4, + "peak_memory_mb_mean": 54731.4 + } + }, + { + "name": "1536x1536_steps35_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0186, + "latency_mean": 53.6942, + "peak_memory_mb_max": 64402.4, + "peak_memory_mb_mean": 64402.4 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit-2511", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit", - "description": "Ulysses SP=2 + CFG=2 + CacheDiT", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit-2511", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1536x1536_steps35_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0538, + "latency_mean": 18.5591, + "peak_memory_mb_max": 53485.0, + "peak_memory_mb_mean": 53485.0 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_2511_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + CacheDiT", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit-2511", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.2776, - "latency_mean": 3.5938, - "peak_memory_mb_max": 55214.0, - "peak_memory_mb_mean": 55214.0 - } - }, - { - "name": "1536x1536_steps35_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1051, - "latency_mean": 9.4918, - "peak_memory_mb_max": 64265.6, - "peak_memory_mb_mean": 64265.6 - } - } - ] - } + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.2776, + "latency_mean": 3.5938, + "peak_memory_mb_max": 55214.0, + "peak_memory_mb_mean": 55214.0 + } + }, + { + "name": "1536x1536_steps35_i2i_2img", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 2, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1051, + "latency_mean": 9.4918, + "peak_memory_mb_max": 64265.6, + "peak_memory_mb_mean": 64265.6 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index da3288182..bfda30731 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -1,144 +1,180 @@ [ - { - "test_name": "test_qwen_image_edit_single_device", - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1129, - "latency_mean": 8.8352, - "peak_memory_mb_max": 55356.5, - "peak_memory_mb_mean": 55356.5 - } - }, - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0240, - "latency_mean": 41.5686, - "peak_memory_mb_max": 62762.7, - "peak_memory_mb_mean": 62762.7 - } - } - ] + { + "test_name": "test_qwen_image_edit_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4", - "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0672, - "latency_mean": 14.8517, - "peak_memory_mb_max": 53931.5, - "peak_memory_mb_mean": 53931.5 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1129, + "latency_mean": 8.8352, + "peak_memory_mb_max": 55356.5, + "peak_memory_mb_mean": 55356.5 + } + }, + { + "name": "1536x1536_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0240, + "latency_mean": 41.5686, + "peak_memory_mb_max": 62762.7, + "peak_memory_mb_mean": 62762.7 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit", - "description": "Ulysses SP=2 + CFG=2 + CacheDiT", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Edit", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1536x1536_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0672, + "latency_mean": 14.8517, + "peak_memory_mb_max": 53931.5, + "peak_memory_mb_mean": 53931.5 + } + } + ] + }, + { + "test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG=2 + CacheDiT", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Edit", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" }, - "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.2879, - "latency_mean": 3.4701, - "peak_memory_mb_max": 55525.6, - "peak_memory_mb_mean": 55525.6 - } - }, - { - "name": "1536x1536_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1113, - "latency_mean": 8.9656, - "peak_memory_mb_max": 63758.3, - "peak_memory_mb_mean": 63758.3 - } - } - ] - } + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.2879, + "latency_mean": 3.4701, + "peak_memory_mb_max": 55525.6, + "peak_memory_mb_mean": 55525.6 + } + }, + { + "name": "1536x1536_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1113, + "latency_mean": 8.9656, + "peak_memory_mb_max": 63758.3, + "peak_memory_mb_mean": 63758.3 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index a94db56f5..3445dd94c 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -1,49 +1,61 @@ [ - { - "test_name": "test_qwen_image_layered_single_device", - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image-Layered", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "640x640_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 640, - "height": 640, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0703, - "latency_mean": 14.208, - "peak_memory_mb_max": 60543.5, - "peak_memory_mb_mean": 60543.5 - } - }, - { - "name": "1024x1024_steps35_i2i", - "dataset": "random", - "task": "i2i", - "width": 1024, - "height": 1024, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0417, - "latency_mean": 23.914, - "peak_memory_mb_max": 60543.5, - "peak_memory_mb_mean": 60543.5 - } - } - ] - } + { + "test_name": "test_qwen_image_layered_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image-Layered", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "640x640_steps20_i2i", + "dataset": "random", + "task": "i2i", + "width": 640, + "height": 640, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0703, + "latency_mean": 14.208, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 + } + }, + { + "name": "1024x1024_steps35_i2i", + "dataset": "random", + "task": "i2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0417, + "latency_mean": 23.914, + "peak_memory_mb_max": 60543.5, + "peak_memory_mb_mean": 60543.5 + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index 5ca6ed61a..b3daf7587 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -1,219 +1,279 @@ [ - { - "test_name": "test_qwen_image_single_device", - "description": "Single-device baseline (no parallelism)", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.4153, - "latency_mean": 2.4065, - "peak_memory_mb_mean": 52289.9 - } - }, - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5991, - "peak_memory_mb_mean": 61360.5 - } - } - ] + { + "test_name": "test_qwen_image_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (no parallelism)", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_single_device_step_execution", - "description": "Single-device baseline (no parallelism) with step execution", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "enable-diffusion-pipeline-profiler": true, - "step-execution": true - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.4175, - "latency_mean": 2.3953, - "peak_memory_mb_mean": 56000 - } - }, - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.0423, - "latency_mean": 23.5768, - "peak_memory_mb_mean": 65000 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 52289.9 + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 61360.5 + } + } + ] + }, + { + "test_name": "test_qwen_image_single_device_step_execution", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (no parallelism) with step execution", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "step-execution": true + } }, - { - "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4", - "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "vae-patch-parallel-size": 4, - "vae-use-tiling": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1214, - "latency_mean": 8.2211, - "peak_memory_mb_mean": 51704.7 - } - } - ] + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.4175, + "latency_mean": 2.3953, + "peak_memory_mb_mean": 56000 + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.0423, + "latency_mean": 23.5768, + "peak_memory_mb_mean": 65000 + } + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit", - "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "ulysses-degree": 2, - "cfg-parallel-size": 2, - "cache-backend": "cache_dit", - "cache-config": { - "Fn_compute_blocks": 1, - "Bn_compute_blocks": 0, - "max_warmup_steps": 4, - "residual_diff_threshold": 0.24, - "max_continuous_cached_steps": 3, - "enable_taylorseer": false, - "taylorseer_order": 1, - "scm_steps_mask_policy": null, - "scm_steps_policy": "dynamic" - }, - "enable-diffusion-pipeline-profiler": true - } + "benchmark_params": [ + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 51704.7 + } + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" }, - "benchmark_params": [ - { - "name": "512x512_steps20", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.559, - "latency_mean": 1.794, - "peak_memory_mb_mean": 52397.6775 - } - }, - { - "name": "1536x1536_steps35", - "dataset": "random", - "task": "t2i", - "width": 1536, - "height": 1536, - "num-inference-steps": 35, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.1918, - "latency_mean": 5.2022, - "peak_memory_mb_mean": 61468.8 - } - } - ] + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_qwen_image_single_device_step_execution_high_concurrency", - "description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep", - "server_type": "vllm-omni", - "server_params": { - "model": "Qwen/Qwen-Image", - "serve_args": { - "enable-diffusion-pipeline-profiler": true, - "step-execution": true, - "max-num-seqs": 8 - } - }, - "benchmark_params": [ - { - "name": "512x512_steps20_high_concurrency", - "dataset": "random", - "task": "t2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": [20, 40, 80, 160], - "max-concurrency": [1, 2, 4, 8], - "warmup-requests": 8, - "warmup-concurrency": 8, - "warmup-num-inference-steps": 20, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], - "latency_mean": [2.4215, 2.992, 5.9293, 9.656], - "peak_memory_mb_mean": [67000, 67000, 67000, 67000] - } - } - ] - } + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 52397.6775 + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 61468.8 + } + } + ] + }, + { + "test_name": "test_qwen_image_single_device_step_execution_high_concurrency", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "step-execution": true, + "max-num-seqs": 8 + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20_high_concurrency", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": [20, 40, 80, 160], + "max-concurrency": [1, 2, 4, 8], + "warmup-requests": 8, + "warmup-concurrency": 8, + "warmup-num-inference-steps": 20, + "enable-negative-prompt": true, + "baseline": { + "throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159], + "latency_mean": [2.4215, 2.992, 5.9293, 9.656], + "peak_memory_mb_mean": [67000, 67000, 67000, 67000] + } + } + ] + } ] diff --git a/tests/dfx/perf/tests/test_runner_metadata.py b/tests/dfx/perf/tests/test_runner_metadata.py index a195c4546..71d4aa95b 100644 --- a/tests/dfx/perf/tests/test_runner_metadata.py +++ b/tests/dfx/perf/tests/test_runner_metadata.py @@ -86,3 +86,202 @@ def test_enabled_false_entry_is_skipped(): # Only the enabled=True entry should appear assert len(params) == 1 assert params[0].get("task") == "voice_clone" + + +def test_resolve_pytest_marks_hardware_dict_with_extra(): + from tests.dfx.conftest import resolve_pytest_marks + + marks = resolve_pytest_marks( + [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, + "full_model", + "diffusion", + "local_model", + ] + ) + names = {m.name for m in marks} + assert "H100" in names + assert "cuda" in names + assert "gpu" in names + assert "distributed_cuda" in names + assert "full_model" in names + assert "diffusion" in names + assert "local_model" in names + + +def test_resolve_pytest_marks_rejects_legacy_object_format(): + from tests.dfx.conftest import resolve_pytest_marks + + with pytest.raises(ValueError, match="mark must be a list"): + resolve_pytest_marks( + { + "hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}, + "marks": ["full_model"], + } + ) + + +def test_extract_mark_resource_label(): + from tests.dfx.conftest import extract_mark_resource_label + + assert extract_mark_resource_label(None) == "na" + assert ( + extract_mark_resource_label( + [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "full_model", + ] + ) + == "H100" + ) + assert ( + extract_mark_resource_label( + [ + { + "hardware_marks": { + "res": {"cuda": "H100", "rocm": "MI325"}, + "num_cards": 2, + } + }, + "diffusion", + ] + ) + == "H100-MI325" + ) + + +def test_resource_label_for_filename(): + from tests.dfx.conftest import resource_label_for_filename + + assert resource_label_for_filename("H100") == "" + assert resource_label_for_filename("L4") == "L4" + assert resource_label_for_filename("910B") == "910B" + assert resource_label_for_filename("na") == "na" + + +def test_hardware_json_value(): + from tests.dfx.conftest import hardware_json_value + + assert hardware_json_value("H100") == "H100" + assert hardware_json_value("na") == "" + assert hardware_json_value(None) == "" + + +def test_extract_configs_resource_label(monkeypatch): + from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label + + monkeypatch.setattr( + "tests.dfx.conftest._read_runtime_device_name", + lambda *, device_id=0: "NVIDIA H100 80GB HBM3", + ) + get_runtime_resource_label(refresh=True) + assert extract_configs_resource_label([]) == "H100" + get_runtime_resource_label(refresh=True) + monkeypatch.setattr( + "tests.dfx.conftest._read_runtime_device_name", + lambda *, device_id=0: "Ascend910B2", + ) + assert get_runtime_resource_label(refresh=True) == "910B" + + +def test_load_benchmark_configs_from_dir(tmp_path): + from tests.dfx.conftest import load_benchmark_configs + + (tmp_path / "a.json").write_text( + json.dumps([{"test_name": "test_a", "server_params": {"model": "m/a"}, "benchmark_params": []}]), + encoding="utf-8", + ) + (tmp_path / "b.json").write_text( + json.dumps([{"test_name": "test_b", "server_params": {"model": "m/b"}, "benchmark_params": []}]), + encoding="utf-8", + ) + configs = load_benchmark_configs(config_dir=tmp_path) + assert [c["test_name"] for c in configs] == ["test_a", "test_b"] + + +def test_create_unique_server_pytest_params_applies_marks(tmp_path): + from tests.dfx.conftest import create_unique_server_pytest_params + + configs = [ + { + "test_name": "test_with_mark", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "full_model", + ], + "server_params": {"model": "some/model"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}], + }, + { + "test_name": "test_without_mark", + "server_params": {"model": "other/model"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}], + }, + ] + params = create_unique_server_pytest_params(configs, tmp_path) + by_id = {p.id: p for p in params} + assert len(by_id["test_with_mark"].values) == 1 + assert isinstance(by_id["test_with_mark"].values[0], tuple) + assert any(m.name == "H100" for m in by_id["test_with_mark"].marks) + assert not any(m.name == "H100" for m in by_id["test_without_mark"].marks) + + +def test_is_diffusion_perf_config(): + from tests.dfx.conftest import is_diffusion_perf_config + + assert not is_diffusion_perf_config( + {"test_name": "omni_a", "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "omni"]} + ) + assert is_diffusion_perf_config( + { + "test_name": "diff_a", + "server_type": "vllm-omni", + "mark": [{"hardware_marks": {"res": {"cuda": "H100"}}}, "diffusion"], + } + ) + + +def test_benchmark_param_id_suffix_from_task_eval_phase(): + from tests.dfx.conftest import _unique_benchmark_param_id_suffixes + + params = [ + {"task": "default_voice", "eval_phase": "latency"}, + {"task": "default_voice", "eval_phase": "throughput"}, + ] + assert _unique_benchmark_param_id_suffixes(params) == [ + "default_voice_latency", + "default_voice_throughput", + ] + + +def test_create_paired_omni_benchmark_pytest_params(tmp_path): + from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params + + configs = [ + { + "test_name": "test_omni", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "omni", + ], + "server_params": {"model": "m/omni"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}, {"name": "p1", "num_prompts": 2}], + }, + { + "test_name": "test_tts", + "mark": [ + {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, + "tts", + ], + "server_params": {"model": "m/tts"}, + "benchmark_params": [{"name": "p0", "num_prompts": 1}], + }, + ] + params = create_paired_omni_benchmark_pytest_params(configs, tmp_path) + by_id = {p.id: p for p in params} + assert set(by_id) == {"test_omni-p0", "test_omni-p1", "test_tts-p0"} + omni_row, bench_row = by_id["test_tts-p0"].values + assert omni_row[0] == "test_tts" + assert bench_row == ("test_tts", 0) + assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks) + assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks) diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index 1ba273a2c..3bbf60935 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -1,6 +1,18 @@ [ { "test_name": "test_qwen3_tts_base", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "stage_overrides": { @@ -98,6 +110,18 @@ }, { "test_name": "test_qwen3_tts_customvoice", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "stage_overrides": { @@ -342,6 +366,18 @@ }, { "test_name": "test_voxtral_tts", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "mistralai/Voxtral-4B-TTS-2603" }, diff --git a/tests/dfx/perf/tests/test_voxcpm2.json b/tests/dfx/perf/tests/test_voxcpm2.json index 5c99f2596..ef72b5d88 100644 --- a/tests/dfx/perf/tests/test_voxcpm2.json +++ b/tests/dfx/perf/tests/test_voxcpm2.json @@ -1,6 +1,18 @@ [ { "test_name": "test_voxcpm2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], "server_params": { "model": "openbmb/VoxCPM2", "trust_remote_code": true, diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 07873a64b..78f720a6a 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -1,107 +1,131 @@ [ - { - "test_name": "test_wan22_i2v_single_device", - "description": "Single-device baseline", - "server_type": "vllm-omni", - "server_params": { - "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", - "serve_args": { - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "832x480_frames81_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 81, - "fps": 16, - "weight": 1 - } - ], - "baseline": { - "throughput_qps": 0.0465, - "latency_mean": 21.4354, - "peak_memory_mb_mean": 72557.58 - } - } - ] + { + "test_name": "test_wan22_i2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } }, - { - "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing", - "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", - "server_type": "vllm-omni", - "server_params": { - "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", - "serve_args": { - "usp": 2, - "vae-patch-parallel-size": 2, - "use-hsdp": true, - "vae-use-slicing": true, - "enable-diffusion-pipeline-profiler": true - } - }, - "benchmark_params": [ - { - "name": "832x480_frames81_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 832, - "height": 480, - "num_inference_steps": 4, - "num_frames": 81, - "fps": 16, - "weight": 1 - } - ], - "baseline": { - "throughput_qps": 0.0642, - "latency_mean": 15.5589, - "peak_memory_mb_mean": 45742.88 - } - }, - { - "name": "1280x720_frames121_steps4", - "dataset": "random", - "task": "i2v", - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 1, - "seed": 42, - "enable-negative-prompt": true, - "random-request-config": [ - { - "width": 1280, - "height": 720, - "num_inference_steps": 4, - "num_frames": 121, - "fps": 16, - "weight": 1 - } - ], - "baseline": { - "throughput_qps": 0.0128, - "latency_mean": 77.8565, - "peak_memory_mb_mean": 53947.84 - } - } - ] - } + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0465, + "latency_mean": 21.4354, + "peak_memory_mb_mean": 72557.58 + } + } + ] + }, + { + "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", + "server_type": "vllm-omni", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "usp": 2, + "vae-patch-parallel-size": 2, + "use-hsdp": true, + "vae-use-slicing": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0642, + "latency_mean": 15.5589, + "peak_memory_mb_mean": 45742.88 + } + }, + { + "name": "1280x720_frames121_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 121, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0128, + "latency_mean": 77.8565, + "peak_memory_mb_mean": 53947.84 + } + } + ] + } ]