[Refactor]Rename diffusion benchmark backend to endpoint (#3137)

Signed-off-by: bjf-frz <frz123db@gmail.com>
Signed-off-by: bjfwhite <baijingfan1@huawei.com>
Co-authored-by: bjfwhite <baijingfan1@huawei.com>
This commit is contained in:
bjf-frzandbjfwhite authored and GitHub committed 2026-05-19 11:30:33 +08:00
1 parent 2c6b1bb0c0
commit 1fa734419e
19 files changed
+352 -87

No files matched your search

+3 -2
View File
@@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
**Notes**
- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
- By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
- If you run the server on another host or port, pass `--base-url` accordingly.
## 2. Supported Datasets
@@ -95,8 +95,9 @@ You can point to your own trace using `--dataset-path`.
### Basic flags
- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.
- `--model`: The OpenAI-compatible `model` field.
- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`.
- `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
- `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
- `--num-prompts`: Number of requests to send.
+30 -4
View File
@@ -151,7 +151,7 @@ async def async_request_chat_completions(
return output
async def async_request_openai_images(
async def async_request_openai_image_generations(
input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: tqdm | None = None,
@@ -348,12 +348,38 @@ async def async_request_v1_videos(
return output
LEGACY_BACKEND_ENDPOINT_ALIASES = {
"vllm-omni": "/v1/chat/completions",
"openai": "/v1/images/generations",
}
def normalize_endpoint(value: str) -> str:
endpoint = str(value).strip()
if not endpoint:
raise ValueError("endpoint must not be empty.")
endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get(
endpoint,
LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint),
)
if not endpoint.startswith("/"):
endpoint = f"/{endpoint}"
return endpoint
def endpoint_filename_token(value: str) -> str:
token = normalize_endpoint(value).lstrip("/")
for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
token = token.replace(bad, "_")
return token or "endpoint"
backends_function_mapping = {
"2i": {
"vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
"openai": (async_request_openai_images, "/v1/images/generations"),
"/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
"/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
},
"2v": {
"v1/videos": (async_request_v1_videos, "/v1/videos"),
"/v1/videos": (async_request_v1_videos, "/v1/videos"),
},
}
@@ -6,31 +6,31 @@
Benchmark online serving for diffusion models (Image/Video Generation).
If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
Supports multiple backends:
- vllm-omni: Uses /v1/chat/completions endpoint (default)
- openai: Uses /v1/images/generations endpoint
- v1/videos: Use /v1/videos endpoint
Supports multiple endpoints:
- /v1/chat/completions: OpenAI chat-compatible image requests
- /v1/images/generations: OpenAI image generation requests
- /v1/videos: Async video jobs
Usage:
# Video (v1/videos backend)
# Video (/v1/videos endpoint)
t2v:
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend v1/videos --dataset vbench --task t2v --num-prompts 10 \
--endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \
--height 480 --width 640 --fps 16 --num-frames 80
i2v:
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend v1/videos --dataset vbench --task i2v --num-prompts 10
--endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10
# Image (vllm-omni backend)
# Image (/v1/chat/completions endpoint)
t2i:
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
--endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \
--height 1024 --width 1024
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend vllm-omni --dataset random --task t2i --num-prompts 1 \
--endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \
--max-concurrency 1 --enable-negative-prompt \
--random-request-config '[
{"width":512,"height":512,"num_inference_steps":20,"weight":0.15},
@@ -41,18 +41,18 @@ Usage:
i2i:
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend vllm-omni --dataset vbench --task i2i --num-prompts 10
--endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
# Image (openai backend)
# Image (/v1/images/generations endpoint)
t2i:
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend openai --dataset vbench --task t2i --num-prompts 10 \
--endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \
--height 1024 --width 1024 --port 3000
# Video (v1/videos)
t2v:
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend v1/videos --dataset random --task t2v --num-prompts 1 \
--endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
--max-concurrency 1 --enable-negative-prompt \
--random-request-config '[
{"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1}
@@ -80,7 +80,12 @@ from typing import Any
import aiohttp
import numpy as np
import requests
from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
from backends import (
RequestFuncInput,
RequestFuncOutput,
backends_function_mapping,
normalize_endpoint,
)
from PIL import Image
from tqdm.asyncio import tqdm
@@ -883,6 +888,14 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
time.sleep(1)
def _default_endpoint_for_task(task: str) -> str:
if task in {"t2v", "i2v", "ti2v"}:
return "/v1/videos"
if task in {"t2i", "i2i", "ti2i"}:
return "/v1/chat/completions"
raise ValueError(f"Unsupported task for endpoint resolution: {task}")
async def benchmark(args):
# Construct base_url if not provided
if args.base_url is None:
@@ -902,18 +915,23 @@ async def benchmark(args):
f"Valid image tasks: {sorted(IMAGE_TASKS)}"
)
valid_backends = sorted(backends_function_mapping[task_type].keys())
raw_endpoint = args.endpoint if args.endpoint is not None else args.backend
if raw_endpoint is None:
raw_endpoint = _default_endpoint_for_task(args.task)
args.endpoint = normalize_endpoint(raw_endpoint)
if args.backend not in valid_backends:
valid_endpoints = sorted(backends_function_mapping[task_type].keys())
if args.endpoint not in valid_endpoints:
logger.error(
f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n"
f"Valid backends for this task type: {valid_backends}\n"
f"Example usage: --task {args.task} --backend {valid_backends[0]}"
f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n"
f"Valid endpoints for this task type: {valid_endpoints}\n"
f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}"
)
raise ValueError("Backend validation failed. See log above for valid options.")
raise ValueError("Endpoint validation failed. See log above for valid options.")
# Setup API URL and request function based on backend
request_func, api_url = backends_function_mapping[task_type][args.backend]
# Setup API URL and request function based on endpoint.
request_func, api_url = backends_function_mapping[task_type][args.endpoint]
api_url = f"{args.base_url}{api_url}"
if args.dataset == "vbench":
@@ -976,15 +994,15 @@ async def benchmark(args):
metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
# Add configuration info to metrics for JSON output
metrics["backend"] = args.backend
metrics["endpoint"] = args.endpoint
metrics["model"] = args.model
metrics["dataset"] = args.dataset
metrics["task"] = args.task
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
# Section 1: Configuration
print("{:<40} {:<15}".format("Backend:", args.backend))
print("{:<40} {:<15}".format("Endpoint:", args.endpoint))
print("{:<40} {:<15}".format("Model:", args.model))
print("{:<40} {:<15}".format("Dataset:", args.dataset))
print("{:<40} {:<15}".format("Task:", args.task))
@@ -1047,12 +1065,17 @@ if __name__ == "__main__":
parser.add_argument("--host", type=str, default="localhost", help="Server host.")
parser.add_argument("--port", type=int, default=8091, help="Server port.")
parser.add_argument("--model", type=str, default="default", help="Model name.")
parser.add_argument(
"--endpoint",
type=str,
default=None,
help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."),
)
parser.add_argument(
"--backend",
type=str,
default="vllm-omni",
choices=["vllm-omni", "openai", "v1/videos"],
help="Backend to target the benchmark to.",
default=None,
help=argparse.SUPPRESS,
)
parser.add_argument(
"--dataset",
@@ -58,7 +58,7 @@ Record these parameters when reporting performance results.
```bash
python benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend vllm-omni \
--endpoint /v1/chat/completions \
--dataset <DATASET_NAME> \
--task t2i \
--num-prompts <N> \
@@ -71,7 +71,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
| Parameter | Description |
| ---------------------- | --------------------------------- |
| `--backend` | Serving backend (use `vllm-omni`) |
| `--endpoint` | API endpoint (use `/v1/chat/completions`; leading `/` is optional) |
| `--dataset` | Dataset name (`random` or custom) |
| `--task` | Task type (e.g., `t2i`) |
| `--num-prompts` | Total number of requests |
@@ -129,7 +129,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
```bash
python benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend vllm-omni \
--endpoint /v1/chat/completions \
--dataset random \
--task t2i \
--num-prompts 1 \
@@ -53,7 +53,7 @@ Record these parameters when reporting performance results.
```bash
python benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend v1/videos \
--endpoint /v1/videos \
--dataset <DATASET_NAME> \
--task t2v \
--num-prompts <N> \
@@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
| Parameter | Description |
| ---------------------- | --------------------------------- |
| `--backend` | Serving backend (use `v1/videos`) |
| `--endpoint` | API endpoint (use `/v1/videos`; leading `/` is optional) |
| `--dataset` | Dataset name (`random` or custom) |
| `--task` | Task type (e.g., `t2v`) |
| `--num-prompts` | Total number of requests |
@@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
```bash
python benchmarks/diffusion/diffusion_benchmark_serving.py \
--backend v1/videos \
--endpoint /v1/videos \
--dataset random \
--task t2v \
--num-prompts 1 \