[Refactor]Rename diffusion benchmark backend to endpoint (#3137)
Signed-off-by: bjf-frz <frz123db@gmail.com> Signed-off-by: bjfwhite <baijingfan1@huawei.com> Co-authored-by: bjfwhite <baijingfan1@huawei.com>
This commit is contained in:
1 parent
2c6b1bb0c0
commit
1fa734419e
19 files changed
+352
-87
No files matched your search
@@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
|
||||
**Notes**
|
||||
|
||||
- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
|
||||
- By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
|
||||
- If you run the server on another host or port, pass `--base-url` accordingly.
|
||||
|
||||
## 2. Supported Datasets
|
||||
@@ -95,8 +95,9 @@ You can point to your own trace using `--dataset-path`.
|
||||
|
||||
### Basic flags
|
||||
|
||||
- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
|
||||
- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.
|
||||
- `--model`: The OpenAI-compatible `model` field.
|
||||
- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`.
|
||||
- `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
|
||||
- `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
|
||||
- `--num-prompts`: Number of requests to send.
|
||||
|
||||
@@ -151,7 +151,7 @@ async def async_request_chat_completions(
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_openai_images(
|
||||
async def async_request_openai_image_generations(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
@@ -348,12 +348,38 @@ async def async_request_v1_videos(
|
||||
return output
|
||||
|
||||
|
||||
LEGACY_BACKEND_ENDPOINT_ALIASES = {
|
||||
"vllm-omni": "/v1/chat/completions",
|
||||
"openai": "/v1/images/generations",
|
||||
}
|
||||
|
||||
|
||||
def normalize_endpoint(value: str) -> str:
|
||||
endpoint = str(value).strip()
|
||||
if not endpoint:
|
||||
raise ValueError("endpoint must not be empty.")
|
||||
endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get(
|
||||
endpoint,
|
||||
LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint),
|
||||
)
|
||||
if not endpoint.startswith("/"):
|
||||
endpoint = f"/{endpoint}"
|
||||
return endpoint
|
||||
|
||||
|
||||
def endpoint_filename_token(value: str) -> str:
|
||||
token = normalize_endpoint(value).lstrip("/")
|
||||
for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
|
||||
token = token.replace(bad, "_")
|
||||
return token or "endpoint"
|
||||
|
||||
|
||||
backends_function_mapping = {
|
||||
"2i": {
|
||||
"vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
|
||||
"openai": (async_request_openai_images, "/v1/images/generations"),
|
||||
"/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
|
||||
"/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
|
||||
},
|
||||
"2v": {
|
||||
"v1/videos": (async_request_v1_videos, "/v1/videos"),
|
||||
"/v1/videos": (async_request_v1_videos, "/v1/videos"),
|
||||
},
|
||||
}
|
||||
@@ -6,31 +6,31 @@
|
||||
Benchmark online serving for diffusion models (Image/Video Generation).
|
||||
If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
|
||||
|
||||
Supports multiple backends:
|
||||
- vllm-omni: Uses /v1/chat/completions endpoint (default)
|
||||
- openai: Uses /v1/images/generations endpoint
|
||||
- v1/videos: Use /v1/videos endpoint
|
||||
Supports multiple endpoints:
|
||||
- /v1/chat/completions: OpenAI chat-compatible image requests
|
||||
- /v1/images/generations: OpenAI image generation requests
|
||||
- /v1/videos: Async video jobs
|
||||
|
||||
Usage:
|
||||
# Video (v1/videos backend)
|
||||
# Video (/v1/videos endpoint)
|
||||
t2v:
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend v1/videos --dataset vbench --task t2v --num-prompts 10 \
|
||||
--endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \
|
||||
--height 480 --width 640 --fps 16 --num-frames 80
|
||||
|
||||
i2v:
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend v1/videos --dataset vbench --task i2v --num-prompts 10
|
||||
--endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10
|
||||
|
||||
|
||||
# Image (vllm-omni backend)
|
||||
# Image (/v1/chat/completions endpoint)
|
||||
t2i:
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
|
||||
--endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \
|
||||
--height 1024 --width 1024
|
||||
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend vllm-omni --dataset random --task t2i --num-prompts 1 \
|
||||
--endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \
|
||||
--max-concurrency 1 --enable-negative-prompt \
|
||||
--random-request-config '[
|
||||
{"width":512,"height":512,"num_inference_steps":20,"weight":0.15},
|
||||
@@ -41,18 +41,18 @@ Usage:
|
||||
|
||||
i2i:
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend vllm-omni --dataset vbench --task i2i --num-prompts 10
|
||||
--endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
|
||||
|
||||
# Image (openai backend)
|
||||
# Image (/v1/images/generations endpoint)
|
||||
t2i:
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend openai --dataset vbench --task t2i --num-prompts 10 \
|
||||
--endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \
|
||||
--height 1024 --width 1024 --port 3000
|
||||
|
||||
# Video (v1/videos)
|
||||
t2v:
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend v1/videos --dataset random --task t2v --num-prompts 1 \
|
||||
--endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
|
||||
--max-concurrency 1 --enable-negative-prompt \
|
||||
--random-request-config '[
|
||||
{"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1}
|
||||
@@ -80,7 +80,12 @@ from typing import Any
|
||||
import aiohttp
|
||||
import numpy as np
|
||||
import requests
|
||||
from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
|
||||
from backends import (
|
||||
RequestFuncInput,
|
||||
RequestFuncOutput,
|
||||
backends_function_mapping,
|
||||
normalize_endpoint,
|
||||
)
|
||||
from PIL import Image
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
@@ -883,6 +888,14 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def _default_endpoint_for_task(task: str) -> str:
|
||||
if task in {"t2v", "i2v", "ti2v"}:
|
||||
return "/v1/videos"
|
||||
if task in {"t2i", "i2i", "ti2i"}:
|
||||
return "/v1/chat/completions"
|
||||
raise ValueError(f"Unsupported task for endpoint resolution: {task}")
|
||||
|
||||
|
||||
async def benchmark(args):
|
||||
# Construct base_url if not provided
|
||||
if args.base_url is None:
|
||||
@@ -902,18 +915,23 @@ async def benchmark(args):
|
||||
f"Valid image tasks: {sorted(IMAGE_TASKS)}"
|
||||
)
|
||||
|
||||
valid_backends = sorted(backends_function_mapping[task_type].keys())
|
||||
raw_endpoint = args.endpoint if args.endpoint is not None else args.backend
|
||||
if raw_endpoint is None:
|
||||
raw_endpoint = _default_endpoint_for_task(args.task)
|
||||
args.endpoint = normalize_endpoint(raw_endpoint)
|
||||
|
||||
if args.backend not in valid_backends:
|
||||
valid_endpoints = sorted(backends_function_mapping[task_type].keys())
|
||||
|
||||
if args.endpoint not in valid_endpoints:
|
||||
logger.error(
|
||||
f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n"
|
||||
f"Valid backends for this task type: {valid_backends}\n"
|
||||
f"Example usage: --task {args.task} --backend {valid_backends[0]}"
|
||||
f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n"
|
||||
f"Valid endpoints for this task type: {valid_endpoints}\n"
|
||||
f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}"
|
||||
)
|
||||
raise ValueError("Backend validation failed. See log above for valid options.")
|
||||
raise ValueError("Endpoint validation failed. See log above for valid options.")
|
||||
|
||||
# Setup API URL and request function based on backend
|
||||
request_func, api_url = backends_function_mapping[task_type][args.backend]
|
||||
# Setup API URL and request function based on endpoint.
|
||||
request_func, api_url = backends_function_mapping[task_type][args.endpoint]
|
||||
api_url = f"{args.base_url}{api_url}"
|
||||
|
||||
if args.dataset == "vbench":
|
||||
@@ -976,15 +994,15 @@ async def benchmark(args):
|
||||
metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
|
||||
|
||||
# Add configuration info to metrics for JSON output
|
||||
metrics["backend"] = args.backend
|
||||
metrics["endpoint"] = args.endpoint
|
||||
metrics["model"] = args.model
|
||||
metrics["dataset"] = args.dataset
|
||||
metrics["task"] = args.task
|
||||
|
||||
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
|
||||
print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
|
||||
|
||||
# Section 1: Configuration
|
||||
print("{:<40} {:<15}".format("Backend:", args.backend))
|
||||
print("{:<40} {:<15}".format("Endpoint:", args.endpoint))
|
||||
print("{:<40} {:<15}".format("Model:", args.model))
|
||||
print("{:<40} {:<15}".format("Dataset:", args.dataset))
|
||||
print("{:<40} {:<15}".format("Task:", args.task))
|
||||
@@ -1047,12 +1065,17 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--host", type=str, default="localhost", help="Server host.")
|
||||
parser.add_argument("--port", type=int, default=8091, help="Server port.")
|
||||
parser.add_argument("--model", type=str, default="default", help="Model name.")
|
||||
parser.add_argument(
|
||||
"--endpoint",
|
||||
type=str,
|
||||
default=None,
|
||||
help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
type=str,
|
||||
default="vllm-omni",
|
||||
choices=["vllm-omni", "openai", "v1/videos"],
|
||||
help="Backend to target the benchmark to.",
|
||||
default=None,
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dataset",
|
||||
|
||||
@@ -58,7 +58,7 @@ Record these parameters when reporting performance results.
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend vllm-omni \
|
||||
--endpoint /v1/chat/completions \
|
||||
--dataset <DATASET_NAME> \
|
||||
--task t2i \
|
||||
--num-prompts <N> \
|
||||
@@ -71,7 +71,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------------- | --------------------------------- |
|
||||
| `--backend` | Serving backend (use `vllm-omni`) |
|
||||
| `--endpoint` | API endpoint (use `/v1/chat/completions`; leading `/` is optional) |
|
||||
| `--dataset` | Dataset name (`random` or custom) |
|
||||
| `--task` | Task type (e.g., `t2i`) |
|
||||
| `--num-prompts` | Total number of requests |
|
||||
@@ -129,7 +129,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend vllm-omni \
|
||||
--endpoint /v1/chat/completions \
|
||||
--dataset random \
|
||||
--task t2i \
|
||||
--num-prompts 1 \
|
||||
|
||||
@@ -53,7 +53,7 @@ Record these parameters when reporting performance results.
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend v1/videos \
|
||||
--endpoint /v1/videos \
|
||||
--dataset <DATASET_NAME> \
|
||||
--task t2v \
|
||||
--num-prompts <N> \
|
||||
@@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
|
||||
| Parameter | Description |
|
||||
| ---------------------- | --------------------------------- |
|
||||
| `--backend` | Serving backend (use `v1/videos`) |
|
||||
| `--endpoint` | API endpoint (use `/v1/videos`; leading `/` is optional) |
|
||||
| `--dataset` | Dataset name (`random` or custom) |
|
||||
| `--task` | Task type (e.g., `t2v`) |
|
||||
| `--num-prompts` | Total number of requests |
|
||||
@@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
|
||||
```bash
|
||||
python benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--backend v1/videos \
|
||||
--endpoint /v1/videos \
|
||||
--dataset random \
|
||||
--task t2v \
|
||||
--num-prompts 1 \
|
||||
|
||||
Reference in new issue
Block a user