[Refactor]Rename diffusion benchmark backend to endpoint (#3137)

Signed-off-by: bjf-frz <frz123db@gmail.com>
Signed-off-by: bjfwhite <baijingfan1@huawei.com>
Co-authored-by: bjfwhite <baijingfan1@huawei.com>

benchmarks/diffusion/README.md

@@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 **Notes**
 
-- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
+- By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
 
 ## 2. Supported Datasets
@@ -95,8 +95,9 @@ You can point to your own trace using `--dataset-path`.
 
 ### Basic flags
 
-- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
+- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.
 - `--model`: The OpenAI-compatible `model` field.
+- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`.
 - `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
 - `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
 - `--num-prompts`: Number of requests to send.

benchmarks/diffusion/backends.py

@@ -151,7 +151,7 @@ async def async_request_chat_completions(
     return output
 
 
-async def async_request_openai_images(
+async def async_request_openai_image_generations(
     input: RequestFuncInput,
     session: aiohttp.ClientSession,
     pbar: tqdm | None = None,
@@ -348,12 +348,38 @@ async def async_request_v1_videos(
     return output
 
 
+LEGACY_BACKEND_ENDPOINT_ALIASES = {
+    "vllm-omni": "/v1/chat/completions",
+    "openai": "/v1/images/generations",
+}
+
+
+def normalize_endpoint(value: str) -> str:
+    endpoint = str(value).strip()
+    if not endpoint:
+        raise ValueError("endpoint must not be empty.")
+    endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get(
+        endpoint,
+        LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint),
+    )
+    if not endpoint.startswith("/"):
+        endpoint = f"/{endpoint}"
+    return endpoint
+
+
+def endpoint_filename_token(value: str) -> str:
+    token = normalize_endpoint(value).lstrip("/")
+    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
+        token = token.replace(bad, "_")
+    return token or "endpoint"
+
+
 backends_function_mapping = {
     "2i": {
-        "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
-        "openai": (async_request_openai_images, "/v1/images/generations"),
+        "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
+        "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
     },
     "2v": {
-        "v1/videos": (async_request_v1_videos, "/v1/videos"),
+        "/v1/videos": (async_request_v1_videos, "/v1/videos"),
     },
 }

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -6,31 +6,31 @@
 Benchmark online serving for diffusion models (Image/Video Generation).
 If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
 
-Supports multiple backends:
-    - vllm-omni: Uses /v1/chat/completions endpoint (default)
-    - openai: Uses /v1/images/generations endpoint
-    - v1/videos: Use /v1/videos endpoint
+Supports multiple endpoints:
+    - /v1/chat/completions: OpenAI chat-compatible image requests
+    - /v1/images/generations: OpenAI image generation requests
+    - /v1/videos: Async video jobs
 
 Usage:
-    # Video (v1/videos backend)
+    # Video (/v1/videos endpoint)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset vbench --task t2v --num-prompts 10 \
+        --endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \
         --height 480 --width 640 --fps 16 --num-frames 80
 
     i2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset vbench --task i2v --num-prompts 10
+        --endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10
 
 
-    # Image (vllm-omni backend)
+    # Image (/v1/chat/completions endpoint)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
+        --endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024
 
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset random --task t2i --num-prompts 1 \
+        --endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":512,"height":512,"num_inference_steps":20,"weight":0.15},
@@ -41,18 +41,18 @@ Usage:
 
     i2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2i --num-prompts 10
+        --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
 
-    # Image (openai backend)
+    # Image (/v1/images/generations endpoint)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend openai --dataset vbench --task t2i --num-prompts 10 \
+        --endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024 --port 3000
 
     # Video (v1/videos)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset random --task t2v --num-prompts 1 \
+        --endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1}
@@ -80,7 +80,12 @@ from typing import Any
 import aiohttp
 import numpy as np
 import requests
-from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
+from backends import (
+    RequestFuncInput,
+    RequestFuncOutput,
+    backends_function_mapping,
+    normalize_endpoint,
+)
 from PIL import Image
 from tqdm.asyncio import tqdm
 
@@ -883,6 +888,14 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
         time.sleep(1)
 
 
+def _default_endpoint_for_task(task: str) -> str:
+    if task in {"t2v", "i2v", "ti2v"}:
+        return "/v1/videos"
+    if task in {"t2i", "i2i", "ti2i"}:
+        return "/v1/chat/completions"
+    raise ValueError(f"Unsupported task for endpoint resolution: {task}")
+
+
 async def benchmark(args):
     # Construct base_url if not provided
     if args.base_url is None:
@@ -902,18 +915,23 @@ async def benchmark(args):
             f"Valid image tasks: {sorted(IMAGE_TASKS)}"
         )
 
-    valid_backends = sorted(backends_function_mapping[task_type].keys())
+    raw_endpoint = args.endpoint if args.endpoint is not None else args.backend
+    if raw_endpoint is None:
+        raw_endpoint = _default_endpoint_for_task(args.task)
+    args.endpoint = normalize_endpoint(raw_endpoint)
 
-    if args.backend not in valid_backends:
+    valid_endpoints = sorted(backends_function_mapping[task_type].keys())
+
+    if args.endpoint not in valid_endpoints:
         logger.error(
-            f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n"
-            f"Valid backends for this task type: {valid_backends}\n"
-            f"Example usage: --task {args.task} --backend {valid_backends[0]}"
+            f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n"
+            f"Valid endpoints for this task type: {valid_endpoints}\n"
+            f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}"
         )
-        raise ValueError("Backend validation failed. See log above for valid options.")
+        raise ValueError("Endpoint validation failed. See log above for valid options.")
 
-    # Setup API URL and request function based on backend
-    request_func, api_url = backends_function_mapping[task_type][args.backend]
+    # Setup API URL and request function based on endpoint.
+    request_func, api_url = backends_function_mapping[task_type][args.endpoint]
     api_url = f"{args.base_url}{api_url}"
 
     if args.dataset == "vbench":
@@ -976,15 +994,15 @@ async def benchmark(args):
     metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
 
     # Add configuration info to metrics for JSON output
-    metrics["backend"] = args.backend
+    metrics["endpoint"] = args.endpoint
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
 
-    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
+    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
 
     # Section 1: Configuration
-    print("{:<40} {:<15}".format("Backend:", args.backend))
+    print("{:<40} {:<15}".format("Endpoint:", args.endpoint))
     print("{:<40} {:<15}".format("Model:", args.model))
     print("{:<40} {:<15}".format("Dataset:", args.dataset))
     print("{:<40} {:<15}".format("Task:", args.task))
@@ -1047,12 +1065,17 @@ if __name__ == "__main__":
     parser.add_argument("--host", type=str, default="localhost", help="Server host.")
     parser.add_argument("--port", type=int, default=8091, help="Server port.")
     parser.add_argument("--model", type=str, default="default", help="Model name.")
+    parser.add_argument(
+        "--endpoint",
+        type=str,
+        default=None,
+        help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."),
+    )
     parser.add_argument(
         "--backend",
         type=str,
-        default="vllm-omni",
-        choices=["vllm-omni", "openai", "v1/videos"],
-        help="Backend to target the benchmark to.",
+        default=None,
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "--dataset",

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -58,7 +58,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend vllm-omni \
+    --endpoint /v1/chat/completions \
     --dataset <DATASET_NAME> \
     --task t2i \
     --num-prompts <N> \
@@ -71,7 +71,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--backend`            | Serving backend (use `vllm-omni`) |
+| `--endpoint`           | API endpoint (use `/v1/chat/completions`; leading `/` is optional) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -129,7 +129,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend vllm-omni \
+    --endpoint /v1/chat/completions \
     --dataset random \
     --task t2i \
     --num-prompts 1 \

benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md

@@ -53,7 +53,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend v1/videos \
+    --endpoint /v1/videos \
     --dataset <DATASET_NAME> \
     --task t2v \
     --num-prompts <N> \
@@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--backend`            | Serving backend (use `v1/videos`) |
+| `--endpoint`           | API endpoint (use `/v1/videos`; leading `/` is optional) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2v`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend v1/videos \
+    --endpoint /v1/videos \
     --dataset random \
     --task t2v \
     --num-prompts 1 \

docs/.nav.yml

@@ -6,10 +6,11 @@ nav:
     - getting_started/installation/*
   - Serving:
     - OpenAI-Compatible API:
-      - Diffusion Chat API: serving/diffusion_chat_api.md
+      - Chat Completions API: serving/chat_completions_api.md
       - Image Generation: serving/image_generation_api.md
       - Image Edit: serving/image_edit_api.md
       - Text to Speech: serving/speech_api.md
+      - Videos: serving/videos_api.md
       - Audio Generation: serving/audio_generate_api.md
       - Streaming Video Input: serving/video_stream_api.md
   - Examples:

docs/serving/chat_completions_api.md

@@ -1,4 +1,4 @@
-# Diffusion Chat Completions API
+# Chat Completions API
 
 vLLM-Omni supports generating and editing images via the `/v1/chat/completions`
 endpoint using diffusion models. This page explains how to pass generation

docs/serving/videos_api.md

@@ -0,0 +1,152 @@
+# Videos API
+
+vLLM-Omni provides an OpenAI-compatible video generation API for diffusion
+video models. The API supports asynchronous video jobs through `/v1/videos` and
+a synchronous benchmark-oriented endpoint through `/v1/videos/sync`.
+
+Each server instance runs a single model specified at startup with
+`vllm serve <model> --omni`.
+
+## Quick Start
+
+### Start the Server
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091
+```
+
+### Create a Video Job
+
+```bash
+create_response=$(curl -s http://localhost:8091/v1/videos \
+  -F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16" \
+  -F "num_inference_steps=40")
+
+video_id=$(echo "${create_response}" | jq -r '.id')
+```
+
+### Poll and Download
+
+```bash
+curl -s "http://localhost:8091/v1/videos/${video_id}" | jq .
+curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
+```
+
+## API Reference
+
+### Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v1/videos` | `POST` | Create an asynchronous video generation job |
+| `/v1/videos/sync` | `POST` | Generate a video synchronously and return raw video bytes |
+| `/v1/videos/{video_id}` | `GET` | Retrieve job status and metadata |
+| `/v1/videos` | `GET` | List stored video jobs |
+| `/v1/videos/{video_id}/content` | `GET` | Download generated video content |
+| `/v1/videos/{video_id}` | `DELETE` | Delete a video job and stored output |
+
+### Request Parameters
+
+`POST /v1/videos` and `POST /v1/videos/sync` accept `multipart/form-data`.
+
+#### OpenAI-style fields
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `prompt` | string | **required** | Text prompt for video generation |
+| `model` | string | server's model | Optional model name |
+| `seconds` | string | null | Requested clip duration in seconds |
+| `size` | string | null | Requested output size in `WIDTHxHEIGHT` format |
+| `user` | string | null | Optional user identifier |
+
+#### vLLM-Omni extension fields
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `input_reference` | file | null | Uploaded reference image for image-to-video requests |
+| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` |
+| `width` | integer | model default | Output video width |
+| `height` | integer | model default | Output video height |
+| `num_frames` | integer | model default | Number of generated frames |
+| `fps` | integer | model default | Output frames per second |
+| `num_inference_steps` | integer | model default | Number of diffusion steps |
+| `guidance_scale` | number | null | CFG guidance scale for the low-noise stage |
+| `guidance_scale_2` | number | null | CFG guidance scale for the high-noise stage |
+| `boundary_ratio` | number | null | Boundary split ratio for multi-stage denoising |
+| `flow_shift` | number | null | Scheduler flow-shift value |
+| `true_cfg_scale` | number | null | True CFG scale when supported by the model |
+| `seed` | integer | null | Random seed for reproducibility |
+| `negative_prompt` | string | null | Text describing what to avoid in the generated video |
+| `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation |
+| `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on |
+| `frame_interpolation_scale` | number | null | RIFE inference scale |
+| `frame_interpolation_model_path` | string | null | Local path or Hugging Face repo for the interpolation model |
+| `lora` | string | null | JSON-encoded LoRA configuration object |
+| `extra_params` | string | null | JSON-encoded object for additional model-specific parameters |
+
+### Create Response
+
+`POST /v1/videos` returns a job record:
+
+```json
+{
+  "id": "video-123",
+  "status": "queued",
+  "created_at": 1701234567
+}
+```
+
+The final content is available from `/v1/videos/{video_id}/content` after the
+job status becomes `completed`.
+
+### Synchronous Response
+
+`POST /v1/videos/sync` blocks until generation finishes and returns raw video
+bytes. It is useful for benchmarks and simple scripts that do not need job
+storage or polling.
+
+## Examples
+
+### Image-to-Video
+
+```bash
+curl -s http://localhost:8091/v1/videos \
+  -F "prompt=animate this image with subtle camera movement" \
+  -F "input_reference=@input.png" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16"
+```
+
+### Synchronous Generation
+
+```bash
+curl -X POST http://localhost:8091/v1/videos/sync \
+  -F "prompt=A small robot walking through a neon city" \
+  -F "width=854" \
+  -F "height=480" \
+  -F "num_frames=80" \
+  -F "fps=16" \
+  -o output.mp4
+```
+
+## Storage
+
+Set `VLLM_OMNI_STORAGE_PATH` to control where asynchronous video outputs are
+stored:
+
+```bash
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+```
+
+## Model-Specific Examples
+
+For complete text-to-video and image-to-video walkthroughs, see:
+
+- [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md)
+- [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md)

docs/user_guide/examples/online_serving/glm_image.md

@@ -101,7 +101,7 @@ guides.
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/generations`](../../../serving/image_generation_api.md)
 or [`/v1/images/edits`](../../../serving/image_edit_api.md) endpoints, pass
 the supported generation controls as top-level fields directly. For image

docs/user_guide/examples/online_serving/image_to_image.md

@@ -208,7 +208,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
     ```
 
     For details on how generation parameters are handled across different clients, see the
-    [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md).
+    [Chat Completions API guide](../../../../serving/chat_completions_api.md).
 
 ### Layered Image Generation (Qwen-Image-Layered)
 
@@ -361,7 +361,7 @@ Provide multiple images in `content` (order matters):
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/edits`](../../../../serving/image_edit_api.md)
 endpoint, pass the supported generation controls as top-level form fields
 directly. For image dimensions and count, use `size` and `n` rather than

docs/user_guide/examples/online_serving/text_to_image.md

@@ -216,7 +216,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
     ```
 
     For details on how generation parameters are handled across different clients, see the
-    [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md).
+    [Chat Completions API guide](../../../../serving/chat_completions_api.md).
 
 ### Multimodal Input (Text + Structured Content)
 
@@ -237,7 +237,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/generations`](../../../../serving/image_generation_api.md)
 endpoint, pass the supported generation controls as top-level JSON fields
 directly. For image dimensions and count, use `size` and `n` rather than

recipes/Qwen/Qwen-Image.md

@@ -104,7 +104,7 @@ To replay the batching benefit with matched warmup:
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-  --backend vllm-omni \
+  --endpoint /v1/chat/completions \
   --dataset vbench \
   --task t2i \
   --model Qwen/Qwen-Image \

tests/benchmarks/test_diffusion_backends_metrics.py

@@ -1,6 +1,11 @@
 import pytest
 
-from benchmarks.diffusion.backends import RequestFuncInput, async_request_chat_completions
+from benchmarks.diffusion.backends import (
+    RequestFuncInput,
+    async_request_chat_completions,
+    endpoint_filename_token,
+    normalize_endpoint,
+)
 
 
 class _MockResponse:
@@ -29,6 +34,32 @@ class _MockSession:
         return _MockResponse(self._payload)
 
 
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_normalization_accepts_optional_leading_slash():
+    assert normalize_endpoint("v1/videos") == "/v1/videos"
+    assert normalize_endpoint("/v1/videos") == "/v1/videos"
+    assert normalize_endpoint("v1/chat/completions") == "/v1/chat/completions"
+    assert normalize_endpoint("v1/images/generations") == "/v1/images/generations"
+
+
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_normalization_accepts_legacy_backend_aliases():
+    assert normalize_endpoint("vllm-omni") == "/v1/chat/completions"
+    assert normalize_endpoint("openai") == "/v1/images/generations"
+
+
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_filename_token_drops_leading_slash():
+    assert endpoint_filename_token("/v1/videos") == "v1_videos"
+    assert endpoint_filename_token("v1/chat/completions") == "v1_chat_completions"
+
+
 @pytest.mark.core_model
 @pytest.mark.benchmark
 @pytest.mark.cpu

tests/dfx/perf/scripts/diffusion_result_template.json

@@ -1,7 +1,7 @@
 [
   {
     "test_name": null,
-    "backend": null,
+    "endpoint": null,
     "timestamp": null,
     "server_params": {
       "model": null,
@@ -43,7 +43,7 @@
       "stage_durations_mean": {},
       "stage_durations_p50": {},
       "stage_durations_p99": {},
-      "backend": null,
+      "endpoint": null,
       "model": null,
       "dataset": null,
       "task": null
@@ -51,6 +51,7 @@
     "log_file": null,
     "Model": null,
     "Framework": null,
+    "API Endpoint": null,
     "Hardware": null,
     "Deployment": null,
     "Task": null,

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -5,16 +5,15 @@ This runner separates two concepts:
 
 1. ``server_type``: how the serving process is started.
    Currently only ``vllm-omni`` is supported here.
-2. ``benchmark_backend``: which serving API the benchmark client calls.
-   Examples: ``vllm-omni`` for ``/v1/chat/completions`` and ``v1/videos``
-   for async video jobs.
+2. ``benchmark_endpoint``: which serving API the benchmark client calls.
+   Examples: ``/v1/chat/completions`` and ``/v1/videos``.
 
 A config JSON file is REQUIRED via --test-config-file:
   pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 
 All benchmark results for a session are consolidated into a single JSON file under
 BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
-Each entry in the file contains the test metadata (test_name, backend, benchmark_params,
+Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
 timestamp) together with the raw metrics returned by the benchmark script.
 """
 
@@ -33,6 +32,8 @@ from typing import Any, cast
 import psutil
 import pytest
 
+from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint
+
 pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -400,7 +401,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]]
                 "model": cfg["server_params"]["model"],
                 "serve_args_dict": serve_args_dict,
                 "serve_args": _build_serve_args(serve_args_dict),
-                "benchmark_backend": cfg.get("benchmark_backend"),
+                "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")),
                 "server_params": cfg["server_params"],
             }
         )
@@ -482,7 +483,7 @@ def run_benchmark(
     model: str,
     params: dict[str, Any],
     test_name: str,
-    backend: str = "vllm-omni",
+    endpoint: str = "/v1/chat/completions",
     server_cfg: dict[str, Any] | None = None,
     source_file: str = "",
 ) -> dict[str, Any]:
@@ -490,18 +491,19 @@ def run_benchmark(
 
     The raw metrics are written to a temporary file by the subprocess.  After
     the run completes the metrics are merged with full metadata (test_name,
-    backend, benchmark_params, timestamp, flat reporting fields) and appended
+    endpoint, benchmark_params, timestamp, flat reporting fields) and appended
     to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE).  The
     temporary file is removed afterwards.  Subprocess stdout/stderr are tee'd
     to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in
     the record.
     """
     timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    endpoint = normalize_endpoint(endpoint)
 
     log_dir = BENCHMARK_RESULT_DIR / "logs"
     log_dir.mkdir(parents=True, exist_ok=True)
-    backend_label = backend.replace("/", "_")
-    log_file = log_dir / f"{test_name}_{backend_label}_{timestamp}.log"
+    endpoint_label = endpoint_filename_token(endpoint)
+    log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log"
 
     with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
         tmp_result_file = Path(tmp.name)
@@ -517,8 +519,8 @@ def run_benchmark(
         str(port),
         "--model",
         model,
-        "--backend",
-        backend,
+        "--endpoint",
+        endpoint,
         "--dataset",
         params.get("dataset", "random"),
         "--task",
@@ -545,7 +547,7 @@ def run_benchmark(
     # cause truncated or out-of-order log output when stdout is piped).
     cmd = [cmd[0], "-u"] + cmd[1:]
 
-    print(f"\nRunning benchmark (backend={backend}): {' '.join(cmd)}")
+    print(f"\nRunning benchmark (endpoint={endpoint}): {' '.join(cmd)}")
     print(f"  Log file: {log_file}")
 
     # Redirect stdout + stderr directly to the log file at the OS level
@@ -597,7 +599,7 @@ def run_benchmark(
 
     record: dict[str, Any] = {
         "test_name": test_name,
-        "backend": backend,
+        "endpoint": endpoint,
         "timestamp": timestamp,
         "server_params": server_cfg.get("server_params"),
         "benchmark_params": params,
@@ -605,7 +607,7 @@ def run_benchmark(
         "log_file": str(log_file),
         "Model": model,
         "Framework": server_type,
-        "API Backend": backend,
+        "API Endpoint": endpoint,
         "Hardware": "",
         "Deployment": "",
         "Task": params.get("task", "t2i"),
@@ -669,21 +671,21 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
             assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
 
 
-def _default_benchmark_backend_for_task(task: str) -> str:
-    """Return the default client-side benchmark backend for a diffusion task."""
+def _default_benchmark_endpoint_for_task(task: str) -> str:
+    """Return the default client-side benchmark endpoint for a diffusion task."""
     if task in {"t2v", "i2v", "ti2v"}:
-        return "v1/videos"
+        return "/v1/videos"
     if task in {"t2i", "i2i", "ti2i"}:
-        return "vllm-omni"
-    raise ValueError(f"Unsupported task for benchmark backend resolution: {task}")
+        return "/v1/chat/completions"
+    raise ValueError(f"Unsupported task for benchmark endpoint resolution: {task}")
 
 
-def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any]) -> str:
+def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, Any]) -> str:
     """Resolve which serving API the benchmark client should call."""
-    configured = server_cfg.get("benchmark_backend")
+    configured = server_cfg.get("benchmark_endpoint")
     if configured:
-        return cast(str, configured)
-    return _default_benchmark_backend_for_task(cast(str, params.get("task", "t2i")))
+        return normalize_endpoint(cast(str, configured))
+    return _default_benchmark_endpoint_for_task(cast(str, params.get("task", "t2i")))
 
 
 # ---------------------------------------------------------------------------
@@ -711,7 +713,7 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
     server_cfg = getattr(diffusion_server, "server_cfg", {})
-    backend = _resolve_benchmark_backend(server_cfg, params)
+    endpoint = _resolve_benchmark_endpoint(server_cfg, params)
 
     result = run_benchmark(
         host=diffusion_server.host,
@@ -719,13 +721,13 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
         model=diffusion_server.model,
         params=params,
         test_name=test_name,
-        backend=backend,
+        endpoint=endpoint,
         server_cfg=server_cfg,
         source_file=cast(str, CONFIG_FILE_PATH),
     )
 
     print(f"\n{'=' * 60}")
-    print(f"Results for {test_name} (server={diffusion_server.server_type}, backend={backend}):")
+    print(f"Results for {test_name} (server={diffusion_server.server_type}, endpoint={endpoint}):")
     for key in (
         "throughput_qps",
         "latency_mean",

tests/test_generate_nightly_perf_excel.py

@@ -35,6 +35,8 @@ def test_generate_excel_report_with_perf_templates(tmp_path: Path):
 
     omni_record = json.loads(omni_template_path.read_text(encoding="utf-8"))
     diffusion_records = json.loads(diffusion_template_path.read_text(encoding="utf-8"))
+    diffusion_records[0]["endpoint"] = "/v1/videos"
+    diffusion_records[0]["result"]["endpoint"] = "/v1/videos"
 
     input_dir = tmp_path / "input"
     diffusion_input_dir = tmp_path / "diffusion_input"
@@ -69,3 +71,6 @@ def test_generate_excel_report_with_perf_templates(tmp_path: Path):
     ws_omni_summary = wb["omni_summary"]
     assert _cell_value_by_header(ws_omni_summary, "commit_sha") == "test_commit_sha"
     assert _cell_value_by_header(ws_omni_summary, "build_id") == "test_build_id"
+
+    ws_diff_summary = wb["diffusion_summary"]
+    assert _cell_value_by_header(ws_diff_summary, "endpoint") == "/v1/videos"

tests/test_generate_nightly_perf_html.py

@@ -28,6 +28,8 @@ def test_generate_html_report_with_perf_templates(tmp_path: Path):
 
     omni_record = json.loads(omni_template_path.read_text(encoding="utf-8"))
     diffusion_records = json.loads(diffusion_template_path.read_text(encoding="utf-8"))
+    diffusion_records[0]["endpoint"] = "/v1/videos"
+    diffusion_records[0]["result"]["endpoint"] = "/v1/videos"
 
     input_dir = tmp_path / "input"
     diffusion_input_dir = tmp_path / "diffusion_input"
@@ -52,3 +54,4 @@ def test_generate_html_report_with_perf_templates(tmp_path: Path):
     assert "Omni records <strong>1</strong>" in html
     assert f"Diffusion records <strong>{len(diffusion_records)}</strong>" in html
     assert "const DIFF_DATA =" in html
+    assert '"endpoint": "/v1/videos"' in html

tools/nightly/generate_nightly_perf_excel.py

@@ -18,6 +18,8 @@ from openpyxl import Workbook
 from openpyxl.styles import PatternFill
 from openpyxl.utils import get_column_letter
 
+from benchmarks.diffusion.backends import normalize_endpoint
+
 LOGGER = logging.getLogger(__name__)
 
 GREY_BLOCK_FILL = PatternFill(start_color="D3D3D3", fill_type="solid")
@@ -60,7 +62,7 @@ DIFFUSION_SUMMARY_COLUMNS: tuple[str, ...] = (
     "date",
     "test_name",
     "model",
-    "backend",
+    "endpoint",
     "dataset",
     "task",
     "completed_requests",
@@ -528,6 +530,12 @@ def _process_diffusion_record(record: dict[str, Any]) -> dict[str, Any]:
     """Normalize a diffusion record by merging `result` and flattening stage metrics."""
     flat = record.copy()
     flat.update(flat.pop("result", {}))
+    if flat.get("endpoint"):
+        flat["endpoint"] = normalize_endpoint(flat["endpoint"])
+    elif flat.get("backend"):
+        flat["endpoint"] = normalize_endpoint(flat["backend"])
+    flat.pop("backend", None)
+    flat.pop("API Backend", None)
     flat = _flatten_stage_durations(flat)
     flat.pop("benchmark_params", None)
     flat.pop("server_params", None)
@@ -752,6 +760,7 @@ _OMNI_SUMMARY_WIDTHS = {
 _DIFFUSION_SUMMARY_WIDTHS = {
     "test_name": 30,
     "model": 15,
+    "endpoint": 24,
 }
 
 

tools/nightly/generate_nightly_perf_html.py

@@ -14,6 +14,8 @@ from collections.abc import Iterable, Sequence
 from datetime import datetime, timezone
 from typing import Any
 
+from benchmarks.diffusion.backends import normalize_endpoint
+
 LOGGER = logging.getLogger(__name__)
 
 _RESULT_JSON_PREFIX = "result_test_"
@@ -222,6 +224,12 @@ def _iter_diffusion_json_records(input_dir: str) -> Iterable[dict[str, Any]]:
             result = flat.pop("result", None)
             if isinstance(result, dict):
                 flat.update(result)
+            if flat.get("endpoint"):
+                flat["endpoint"] = normalize_endpoint(flat["endpoint"])
+            elif flat.get("backend"):
+                flat["endpoint"] = normalize_endpoint(flat["backend"])
+            flat.pop("backend", None)
+            flat.pop("API Backend", None)
             if "date" not in flat or not flat.get("date"):
                 flat["date"] = filename_meta.get("date") or datetime.now(
                     timezone.utc,
@@ -1183,7 +1191,7 @@ function filterRows(rows, filters) {{
     ) {{
       return false;
     }}
-    if (filters.backend && String(row.backend || "") !== filters.backend) return false;
+    if (filters.backend && String(row[filters.backendKey] || "") !== filters.backend) return false;
     if (
       filters.extraKey &&
       filters.extraValue &&
@@ -1223,12 +1231,14 @@ function initSection(prefix, columns, data, numericCols, groups) {{
 
   const modelKey = prefix === "diff" ? "model" : "model_id";
   const datasetKey = prefix === "diff" ? "dataset" : "dataset_name";
+  const backendKey = prefix === "diff" ? "endpoint" : "backend";
+  const backendLabel = prefix === "diff" ? "Endpoint" : "Backend";
   const extraKey = prefix === "diff" ? "" : "tokenizer_id";
   const metaKeys = prefix === "diff"
     ? ["test_name"]
     : ["test_name", "max_concurrency", "num_prompts"];
   const configFields = prefix === "diff"
-    ? ["test_name", "model", "backend", "dataset"]
+    ? ["test_name", "model", "endpoint", "dataset"]
     : [
         "endpoint_type",
         "backend",
@@ -1265,6 +1275,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{
       extraKey,
       modelKey,
       datasetKey,
+      backendKey,
     }};
   }}
 
@@ -1296,7 +1307,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{
     }}
     fillSelect(
       backendList,
-      uniqSorted(baseRows.map((row) => row.backend)),
+      uniqSorted(baseRows.map((row) => row[backendKey])),
       backendInput.value,
     );
     if (extraList) {{
@@ -1378,7 +1389,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{
     }});
 
     if (!backendInput.value.trim()) {{
-      backendInput.value = firstValue(rows, "backend");
+      backendInput.value = firstValue(rows, backendKey);
     }}
     rows = filterRows(data, {{
       ...currentFilters(),
@@ -1465,7 +1476,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{
               `<div class="kv-value"><code>${{escapeHtml(filters.numPrompts || "All")}}</code></div></div>`
             )
         }}
-        <div class="kv-row"><div class="kv-label">Backend</div><div class="kv-value"><code>${{escapeHtml(
+        <div class="kv-row"><div class="kv-label">${{backendLabel}}</div><div class="kv-value"><code>${{escapeHtml(
           filters.backend || "All",
         )}}</code></div></div>
         ${{
@@ -1707,7 +1718,7 @@ window.addEventListener("load", () => {{
                 'Diffusion</h2><div class="section-subtitle">Image generation '
                 "benchmark history with throughput and latency trend views.</div>"
                 '</div><div class="section-pills"><div class="pill"><strong>'
-                "Grouping</strong> test_name + model + backend + dataset</div>"
+                "Grouping</strong> test_name + model + endpoint + dataset</div>"
                 '<div class="pill"><strong>Snapshots</strong> single-point series '
                 "stay readable</div></div></div>"
             ),
@@ -1736,7 +1747,7 @@ window.addEventListener("load", () => {{
             "        </div>",
             '        <div class="toolbar-row secondary">',
             (
-                '        <div class="filter-field compact"><label>backend</label>'
+                '        <div class="filter-field compact"><label>endpoint</label>'
                 '<select class="filter-select" id="diff-backend"></select></div>'
             ),
             (
@@ -1803,7 +1814,7 @@ def generate_html_report(input_dir: str, diffusion_input_dir: str, output_file:
         "date",
         "test_name",
         "model",
-        "backend",
+        "endpoint",
         "dataset",
         "task",
         "duration",