From 1fa734419ec6b578537aa5267c0d42f006499201 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Tue, 19 May 2026 11:30:33 +0800 Subject: [PATCH] [Refactor]Rename diffusion benchmark backend to endpoint (#3137) Signed-off-by: bjf-frz Signed-off-by: bjfwhite Co-authored-by: bjfwhite --- benchmarks/diffusion/README.md | 5 +- benchmarks/diffusion/backends.py | 34 +++- .../diffusion/diffusion_benchmark_serving.py | 81 ++++++---- .../qwen_image_serving_performance.md | 6 +- .../wan_2_2_serving_performance.md | 6 +- docs/.nav.yml | 3 +- ...on_chat_api.md => chat_completions_api.md} | 2 +- docs/serving/videos_api.md | 152 ++++++++++++++++++ .../examples/online_serving/glm_image.md | 2 +- .../examples/online_serving/image_to_image.md | 4 +- .../examples/online_serving/text_to_image.md | 4 +- recipes/Qwen/Qwen-Image.md | 2 +- .../test_diffusion_backends_metrics.py | 33 +++- .../scripts/diffusion_result_template.json | 5 +- .../perf/scripts/run_diffusion_benchmark.py | 54 ++++--- tests/test_generate_nightly_perf_excel.py | 5 + tests/test_generate_nightly_perf_html.py | 3 + tools/nightly/generate_nightly_perf_excel.py | 11 +- tools/nightly/generate_nightly_perf_html.py | 27 +++- 19 files changed, 352 insertions(+), 87 deletions(-) rename docs/serving/{diffusion_chat_api.md => chat_completions_api.md} (98%) create mode 100644 docs/serving/videos_api.md diff --git a/benchmarks/diffusion/README.md b/benchmarks/diffusion/README.md index 16a86c88f..54ed64261 100644 --- a/benchmarks/diffusion/README.md +++ b/benchmarks/diffusion/README.md @@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ **Notes** -- The benchmark talks to `http://:/v1/chat/completions`. +- By default, image tasks talk to `http://:/v1/chat/completions`; video tasks talk to `/v1/videos`. - If you run the server on another host or port, pass `--base-url` accordingly. ## 2. Supported Datasets @@ -95,8 +95,9 @@ You can point to your own trace using `--dataset-path`. ### Basic flags -- `--base-url`: Server address (the script calls `.../v1/chat/completions`). +- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL. - `--model`: The OpenAI-compatible `model` field. +- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`. - `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`). - `--dataset`: Dataset mode (`vbench` / `trace` / `random`). - `--num-prompts`: Number of requests to send. diff --git a/benchmarks/diffusion/backends.py b/benchmarks/diffusion/backends.py index d33160f13..b7e533b2d 100644 --- a/benchmarks/diffusion/backends.py +++ b/benchmarks/diffusion/backends.py @@ -151,7 +151,7 @@ async def async_request_chat_completions( return output -async def async_request_openai_images( +async def async_request_openai_image_generations( input: RequestFuncInput, session: aiohttp.ClientSession, pbar: tqdm | None = None, @@ -348,12 +348,38 @@ async def async_request_v1_videos( return output +LEGACY_BACKEND_ENDPOINT_ALIASES = { + "vllm-omni": "/v1/chat/completions", + "openai": "/v1/images/generations", +} + + +def normalize_endpoint(value: str) -> str: + endpoint = str(value).strip() + if not endpoint: + raise ValueError("endpoint must not be empty.") + endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get( + endpoint, + LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint), + ) + if not endpoint.startswith("/"): + endpoint = f"/{endpoint}" + return endpoint + + +def endpoint_filename_token(value: str) -> str: + token = normalize_endpoint(value).lstrip("/") + for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"): + token = token.replace(bad, "_") + return token or "endpoint" + + backends_function_mapping = { "2i": { - "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"), - "openai": (async_request_openai_images, "/v1/images/generations"), + "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"), + "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"), }, "2v": { - "v1/videos": (async_request_v1_videos, "/v1/videos"), + "/v1/videos": (async_request_v1_videos, "/v1/videos"), }, } diff --git a/benchmarks/diffusion/diffusion_benchmark_serving.py b/benchmarks/diffusion/diffusion_benchmark_serving.py index f2b77ff8d..c59844f65 100644 --- a/benchmarks/diffusion/diffusion_benchmark_serving.py +++ b/benchmarks/diffusion/diffusion_benchmark_serving.py @@ -6,31 +6,31 @@ Benchmark online serving for diffusion models (Image/Video Generation). If you want to use i2v, i2i dataset, you should `uv pip install gdown` first -Supports multiple backends: - - vllm-omni: Uses /v1/chat/completions endpoint (default) - - openai: Uses /v1/images/generations endpoint - - v1/videos: Use /v1/videos endpoint +Supports multiple endpoints: + - /v1/chat/completions: OpenAI chat-compatible image requests + - /v1/images/generations: OpenAI image generation requests + - /v1/videos: Async video jobs Usage: - # Video (v1/videos backend) + # Video (/v1/videos endpoint) t2v: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend v1/videos --dataset vbench --task t2v --num-prompts 10 \ + --endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \ --height 480 --width 640 --fps 16 --num-frames 80 i2v: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend v1/videos --dataset vbench --task i2v --num-prompts 10 + --endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10 - # Image (vllm-omni backend) + # Image (/v1/chat/completions endpoint) t2i: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \ + --endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \ --height 1024 --width 1024 python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni --dataset random --task t2i --num-prompts 1 \ + --endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \ --max-concurrency 1 --enable-negative-prompt \ --random-request-config '[ {"width":512,"height":512,"num_inference_steps":20,"weight":0.15}, @@ -41,18 +41,18 @@ Usage: i2i: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni --dataset vbench --task i2i --num-prompts 10 + --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10 - # Image (openai backend) + # Image (/v1/images/generations endpoint) t2i: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend openai --dataset vbench --task t2i --num-prompts 10 \ + --endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \ --height 1024 --width 1024 --port 3000 # Video (v1/videos) t2v: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend v1/videos --dataset random --task t2v --num-prompts 1 \ + --endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \ --max-concurrency 1 --enable-negative-prompt \ --random-request-config '[ {"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1} @@ -80,7 +80,12 @@ from typing import Any import aiohttp import numpy as np import requests -from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping +from backends import ( + RequestFuncInput, + RequestFuncOutput, + backends_function_mapping, + normalize_endpoint, +) from PIL import Image from tqdm.asyncio import tqdm @@ -883,6 +888,14 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None: time.sleep(1) +def _default_endpoint_for_task(task: str) -> str: + if task in {"t2v", "i2v", "ti2v"}: + return "/v1/videos" + if task in {"t2i", "i2i", "ti2i"}: + return "/v1/chat/completions" + raise ValueError(f"Unsupported task for endpoint resolution: {task}") + + async def benchmark(args): # Construct base_url if not provided if args.base_url is None: @@ -902,18 +915,23 @@ async def benchmark(args): f"Valid image tasks: {sorted(IMAGE_TASKS)}" ) - valid_backends = sorted(backends_function_mapping[task_type].keys()) + raw_endpoint = args.endpoint if args.endpoint is not None else args.backend + if raw_endpoint is None: + raw_endpoint = _default_endpoint_for_task(args.task) + args.endpoint = normalize_endpoint(raw_endpoint) - if args.backend not in valid_backends: + valid_endpoints = sorted(backends_function_mapping[task_type].keys()) + + if args.endpoint not in valid_endpoints: logger.error( - f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n" - f"Valid backends for this task type: {valid_backends}\n" - f"Example usage: --task {args.task} --backend {valid_backends[0]}" + f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n" + f"Valid endpoints for this task type: {valid_endpoints}\n" + f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}" ) - raise ValueError("Backend validation failed. See log above for valid options.") + raise ValueError("Endpoint validation failed. See log above for valid options.") - # Setup API URL and request function based on backend - request_func, api_url = backends_function_mapping[task_type][args.backend] + # Setup API URL and request function based on endpoint. + request_func, api_url = backends_function_mapping[task_type][args.endpoint] api_url = f"{args.base_url}{api_url}" if args.dataset == "vbench": @@ -976,15 +994,15 @@ async def benchmark(args): metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo) # Add configuration info to metrics for JSON output - metrics["backend"] = args.backend + metrics["endpoint"] = args.endpoint metrics["model"] = args.model metrics["dataset"] = args.dataset metrics["task"] = args.task - print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="=")) + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) # Section 1: Configuration - print("{:<40} {:<15}".format("Backend:", args.backend)) + print("{:<40} {:<15}".format("Endpoint:", args.endpoint)) print("{:<40} {:<15}".format("Model:", args.model)) print("{:<40} {:<15}".format("Dataset:", args.dataset)) print("{:<40} {:<15}".format("Task:", args.task)) @@ -1047,12 +1065,17 @@ if __name__ == "__main__": parser.add_argument("--host", type=str, default="localhost", help="Server host.") parser.add_argument("--port", type=int, default=8091, help="Server port.") parser.add_argument("--model", type=str, default="default", help="Model name.") + parser.add_argument( + "--endpoint", + type=str, + default=None, + help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."), + ) parser.add_argument( "--backend", type=str, - default="vllm-omni", - choices=["vllm-omni", "openai", "v1/videos"], - help="Backend to target the benchmark to.", + default=None, + help=argparse.SUPPRESS, ) parser.add_argument( "--dataset", diff --git a/benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md b/benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md index 172a8ed05..6d4a3c405 100644 --- a/benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md +++ b/benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md @@ -58,7 +58,7 @@ Record these parameters when reporting performance results. ```bash python benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni \ + --endpoint /v1/chat/completions \ --dataset \ --task t2i \ --num-prompts \ @@ -71,7 +71,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \ | Parameter | Description | | ---------------------- | --------------------------------- | -| `--backend` | Serving backend (use `vllm-omni`) | +| `--endpoint` | API endpoint (use `/v1/chat/completions`; leading `/` is optional) | | `--dataset` | Dataset name (`random` or custom) | | `--task` | Task type (e.g., `t2i`) | | `--num-prompts` | Total number of requests | @@ -129,7 +129,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \ ```bash python benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni \ + --endpoint /v1/chat/completions \ --dataset random \ --task t2i \ --num-prompts 1 \ diff --git a/benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md b/benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md index 9d6c40ece..d9b77fd99 100644 --- a/benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md +++ b/benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md @@ -53,7 +53,7 @@ Record these parameters when reporting performance results. ```bash python benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend v1/videos \ + --endpoint /v1/videos \ --dataset \ --task t2v \ --num-prompts \ @@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \ | Parameter | Description | | ---------------------- | --------------------------------- | -| `--backend` | Serving backend (use `v1/videos`) | +| `--endpoint` | API endpoint (use `/v1/videos`; leading `/` is optional) | | `--dataset` | Dataset name (`random` or custom) | | `--task` | Task type (e.g., `t2v`) | | `--num-prompts` | Total number of requests | @@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \ ```bash python benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend v1/videos \ + --endpoint /v1/videos \ --dataset random \ --task t2v \ --num-prompts 1 \ diff --git a/docs/.nav.yml b/docs/.nav.yml index 55283f0e8..a60a4368d 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -6,10 +6,11 @@ nav: - getting_started/installation/* - Serving: - OpenAI-Compatible API: - - Diffusion Chat API: serving/diffusion_chat_api.md + - Chat Completions API: serving/chat_completions_api.md - Image Generation: serving/image_generation_api.md - Image Edit: serving/image_edit_api.md - Text to Speech: serving/speech_api.md + - Videos: serving/videos_api.md - Audio Generation: serving/audio_generate_api.md - Streaming Video Input: serving/video_stream_api.md - Examples: diff --git a/docs/serving/diffusion_chat_api.md b/docs/serving/chat_completions_api.md similarity index 98% rename from docs/serving/diffusion_chat_api.md rename to docs/serving/chat_completions_api.md index d0e2990ad..d8f852f99 100644 --- a/docs/serving/diffusion_chat_api.md +++ b/docs/serving/chat_completions_api.md @@ -1,4 +1,4 @@ -# Diffusion Chat Completions API +# Chat Completions API vLLM-Omni supports generating and editing images via the `/v1/chat/completions` endpoint using diffusion models. This page explains how to pass generation diff --git a/docs/serving/videos_api.md b/docs/serving/videos_api.md new file mode 100644 index 000000000..a8caee013 --- /dev/null +++ b/docs/serving/videos_api.md @@ -0,0 +1,152 @@ +# Videos API + +vLLM-Omni provides an OpenAI-compatible video generation API for diffusion +video models. The API supports asynchronous video jobs through `/v1/videos` and +a synchronous benchmark-oriented endpoint through `/v1/videos/sync`. + +Each server instance runs a single model specified at startup with +`vllm serve --omni`. + +## Quick Start + +### Start the Server + +```bash +vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091 +``` + +### Create a Video Job + +```bash +create_response=$(curl -s http://localhost:8091/v1/videos \ + -F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \ + -F "width=1280" \ + -F "height=720" \ + -F "num_frames=80" \ + -F "fps=16" \ + -F "num_inference_steps=40") + +video_id=$(echo "${create_response}" | jq -r '.id') +``` + +### Poll and Download + +```bash +curl -s "http://localhost:8091/v1/videos/${video_id}" | jq . +curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4 +``` + +## API Reference + +### Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/v1/videos` | `POST` | Create an asynchronous video generation job | +| `/v1/videos/sync` | `POST` | Generate a video synchronously and return raw video bytes | +| `/v1/videos/{video_id}` | `GET` | Retrieve job status and metadata | +| `/v1/videos` | `GET` | List stored video jobs | +| `/v1/videos/{video_id}/content` | `GET` | Download generated video content | +| `/v1/videos/{video_id}` | `DELETE` | Delete a video job and stored output | + +### Request Parameters + +`POST /v1/videos` and `POST /v1/videos/sync` accept `multipart/form-data`. + +#### OpenAI-style fields + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `prompt` | string | **required** | Text prompt for video generation | +| `model` | string | server's model | Optional model name | +| `seconds` | string | null | Requested clip duration in seconds | +| `size` | string | null | Requested output size in `WIDTHxHEIGHT` format | +| `user` | string | null | Optional user identifier | + +#### vLLM-Omni extension fields + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input_reference` | file | null | Uploaded reference image for image-to-video requests | +| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` | +| `width` | integer | model default | Output video width | +| `height` | integer | model default | Output video height | +| `num_frames` | integer | model default | Number of generated frames | +| `fps` | integer | model default | Output frames per second | +| `num_inference_steps` | integer | model default | Number of diffusion steps | +| `guidance_scale` | number | null | CFG guidance scale for the low-noise stage | +| `guidance_scale_2` | number | null | CFG guidance scale for the high-noise stage | +| `boundary_ratio` | number | null | Boundary split ratio for multi-stage denoising | +| `flow_shift` | number | null | Scheduler flow-shift value | +| `true_cfg_scale` | number | null | True CFG scale when supported by the model | +| `seed` | integer | null | Random seed for reproducibility | +| `negative_prompt` | string | null | Text describing what to avoid in the generated video | +| `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation | +| `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on | +| `frame_interpolation_scale` | number | null | RIFE inference scale | +| `frame_interpolation_model_path` | string | null | Local path or Hugging Face repo for the interpolation model | +| `lora` | string | null | JSON-encoded LoRA configuration object | +| `extra_params` | string | null | JSON-encoded object for additional model-specific parameters | + +### Create Response + +`POST /v1/videos` returns a job record: + +```json +{ + "id": "video-123", + "status": "queued", + "created_at": 1701234567 +} +``` + +The final content is available from `/v1/videos/{video_id}/content` after the +job status becomes `completed`. + +### Synchronous Response + +`POST /v1/videos/sync` blocks until generation finishes and returns raw video +bytes. It is useful for benchmarks and simple scripts that do not need job +storage or polling. + +## Examples + +### Image-to-Video + +```bash +curl -s http://localhost:8091/v1/videos \ + -F "prompt=animate this image with subtle camera movement" \ + -F "input_reference=@input.png" \ + -F "width=1280" \ + -F "height=720" \ + -F "num_frames=80" \ + -F "fps=16" +``` + +### Synchronous Generation + +```bash +curl -X POST http://localhost:8091/v1/videos/sync \ + -F "prompt=A small robot walking through a neon city" \ + -F "width=854" \ + -F "height=480" \ + -F "num_frames=80" \ + -F "fps=16" \ + -o output.mp4 +``` + +## Storage + +Set `VLLM_OMNI_STORAGE_PATH` to control where asynchronous video outputs are +stored: + +```bash +export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos +``` + +## Model-Specific Examples + +For complete text-to-video and image-to-video walkthroughs, see: + +- [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md) +- [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md) diff --git a/docs/user_guide/examples/online_serving/glm_image.md b/docs/user_guide/examples/online_serving/glm_image.md index 4cc49e846..410b1180a 100644 --- a/docs/user_guide/examples/online_serving/glm_image.md +++ b/docs/user_guide/examples/online_serving/glm_image.md @@ -101,7 +101,7 @@ guides. When using `/v1/chat/completions`, pass these inside `extra_body` in the curl JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the -[Diffusion Chat API guide](../../../serving/diffusion_chat_api.md)). +[Chat Completions API guide](../../../serving/chat_completions_api.md)). When using the dedicated [`/v1/images/generations`](../../../serving/image_generation_api.md) or [`/v1/images/edits`](../../../serving/image_edit_api.md) endpoints, pass the supported generation controls as top-level fields directly. For image diff --git a/docs/user_guide/examples/online_serving/image_to_image.md b/docs/user_guide/examples/online_serving/image_to_image.md index f40173c4f..89048ae71 100644 --- a/docs/user_guide/examples/online_serving/image_to_image.md +++ b/docs/user_guide/examples/online_serving/image_to_image.md @@ -208,7 +208,7 @@ Wrap generation parameters inside `extra_body` in the request JSON: ``` For details on how generation parameters are handled across different clients, see the - [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md). + [Chat Completions API guide](../../../../serving/chat_completions_api.md). ### Layered Image Generation (Qwen-Image-Layered) @@ -361,7 +361,7 @@ Provide multiple images in `content` (order matters): When using `/v1/chat/completions`, pass these inside `extra_body` in the curl JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the -[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)). +[Chat Completions API guide](../../../../serving/chat_completions_api.md)). When using the dedicated [`/v1/images/edits`](../../../../serving/image_edit_api.md) endpoint, pass the supported generation controls as top-level form fields directly. For image dimensions and count, use `size` and `n` rather than diff --git a/docs/user_guide/examples/online_serving/text_to_image.md b/docs/user_guide/examples/online_serving/text_to_image.md index 69c1480e3..43fb96594 100644 --- a/docs/user_guide/examples/online_serving/text_to_image.md +++ b/docs/user_guide/examples/online_serving/text_to_image.md @@ -216,7 +216,7 @@ Wrap generation parameters inside `extra_body` in the request JSON: ``` For details on how generation parameters are handled across different clients, see the - [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md). + [Chat Completions API guide](../../../../serving/chat_completions_api.md). ### Multimodal Input (Text + Structured Content) @@ -237,7 +237,7 @@ Wrap generation parameters inside `extra_body` in the request JSON: When using `/v1/chat/completions`, pass these inside `extra_body` in the curl JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the -[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)). +[Chat Completions API guide](../../../../serving/chat_completions_api.md)). When using the dedicated [`/v1/images/generations`](../../../../serving/image_generation_api.md) endpoint, pass the supported generation controls as top-level JSON fields directly. For image dimensions and count, use `size` and `n` rather than diff --git a/recipes/Qwen/Qwen-Image.md b/recipes/Qwen/Qwen-Image.md index d9d60bf0d..327160973 100644 --- a/recipes/Qwen/Qwen-Image.md +++ b/recipes/Qwen/Qwen-Image.md @@ -104,7 +104,7 @@ To replay the batching benefit with matched warmup: ```bash python benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni \ + --endpoint /v1/chat/completions \ --dataset vbench \ --task t2i \ --model Qwen/Qwen-Image \ diff --git a/tests/benchmarks/test_diffusion_backends_metrics.py b/tests/benchmarks/test_diffusion_backends_metrics.py index 2d51d0f1d..a89ab06be 100644 --- a/tests/benchmarks/test_diffusion_backends_metrics.py +++ b/tests/benchmarks/test_diffusion_backends_metrics.py @@ -1,6 +1,11 @@ import pytest -from benchmarks.diffusion.backends import RequestFuncInput, async_request_chat_completions +from benchmarks.diffusion.backends import ( + RequestFuncInput, + async_request_chat_completions, + endpoint_filename_token, + normalize_endpoint, +) class _MockResponse: @@ -29,6 +34,32 @@ class _MockSession: return _MockResponse(self._payload) +@pytest.mark.core_model +@pytest.mark.benchmark +@pytest.mark.cpu +def test_endpoint_normalization_accepts_optional_leading_slash(): + assert normalize_endpoint("v1/videos") == "/v1/videos" + assert normalize_endpoint("/v1/videos") == "/v1/videos" + assert normalize_endpoint("v1/chat/completions") == "/v1/chat/completions" + assert normalize_endpoint("v1/images/generations") == "/v1/images/generations" + + +@pytest.mark.core_model +@pytest.mark.benchmark +@pytest.mark.cpu +def test_endpoint_normalization_accepts_legacy_backend_aliases(): + assert normalize_endpoint("vllm-omni") == "/v1/chat/completions" + assert normalize_endpoint("openai") == "/v1/images/generations" + + +@pytest.mark.core_model +@pytest.mark.benchmark +@pytest.mark.cpu +def test_endpoint_filename_token_drops_leading_slash(): + assert endpoint_filename_token("/v1/videos") == "v1_videos" + assert endpoint_filename_token("v1/chat/completions") == "v1_chat_completions" + + @pytest.mark.core_model @pytest.mark.benchmark @pytest.mark.cpu diff --git a/tests/dfx/perf/scripts/diffusion_result_template.json b/tests/dfx/perf/scripts/diffusion_result_template.json index 86bdf1bc7..ba16a993d 100644 --- a/tests/dfx/perf/scripts/diffusion_result_template.json +++ b/tests/dfx/perf/scripts/diffusion_result_template.json @@ -1,7 +1,7 @@ [ { "test_name": null, - "backend": null, + "endpoint": null, "timestamp": null, "server_params": { "model": null, @@ -43,7 +43,7 @@ "stage_durations_mean": {}, "stage_durations_p50": {}, "stage_durations_p99": {}, - "backend": null, + "endpoint": null, "model": null, "dataset": null, "task": null @@ -51,6 +51,7 @@ "log_file": null, "Model": null, "Framework": null, + "API Endpoint": null, "Hardware": null, "Deployment": null, "Task": null, diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 639f03623..9059904e4 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -5,16 +5,15 @@ This runner separates two concepts: 1. ``server_type``: how the serving process is started. Currently only ``vllm-omni`` is supported here. -2. ``benchmark_backend``: which serving API the benchmark client calls. - Examples: ``vllm-omni`` for ``/v1/chat/completions`` and ``v1/videos`` - for async video jobs. +2. ``benchmark_endpoint``: which serving API the benchmark client calls. + Examples: ``/v1/chat/completions`` and ``/v1/videos``. A config JSON file is REQUIRED via --test-config-file: pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json All benchmark results for a session are consolidated into a single JSON file under BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable). -Each entry in the file contains the test metadata (test_name, backend, benchmark_params, +Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params, timestamp) together with the raw metrics returned by the benchmark script. """ @@ -33,6 +32,8 @@ from typing import Any, cast import psutil import pytest +from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint + pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -400,7 +401,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]] "model": cfg["server_params"]["model"], "serve_args_dict": serve_args_dict, "serve_args": _build_serve_args(serve_args_dict), - "benchmark_backend": cfg.get("benchmark_backend"), + "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")), "server_params": cfg["server_params"], } ) @@ -482,7 +483,7 @@ def run_benchmark( model: str, params: dict[str, Any], test_name: str, - backend: str = "vllm-omni", + endpoint: str = "/v1/chat/completions", server_cfg: dict[str, Any] | None = None, source_file: str = "", ) -> dict[str, Any]: @@ -490,18 +491,19 @@ def run_benchmark( The raw metrics are written to a temporary file by the subprocess. After the run completes the metrics are merged with full metadata (test_name, - backend, benchmark_params, timestamp, flat reporting fields) and appended + endpoint, benchmark_params, timestamp, flat reporting fields) and appended to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE). The temporary file is removed afterwards. Subprocess stdout/stderr are tee'd to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in the record. """ timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + endpoint = normalize_endpoint(endpoint) log_dir = BENCHMARK_RESULT_DIR / "logs" log_dir.mkdir(parents=True, exist_ok=True) - backend_label = backend.replace("/", "_") - log_file = log_dir / f"{test_name}_{backend_label}_{timestamp}.log" + endpoint_label = endpoint_filename_token(endpoint) + log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log" with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) @@ -517,8 +519,8 @@ def run_benchmark( str(port), "--model", model, - "--backend", - backend, + "--endpoint", + endpoint, "--dataset", params.get("dataset", "random"), "--task", @@ -545,7 +547,7 @@ def run_benchmark( # cause truncated or out-of-order log output when stdout is piped). cmd = [cmd[0], "-u"] + cmd[1:] - print(f"\nRunning benchmark (backend={backend}): {' '.join(cmd)}") + print(f"\nRunning benchmark (endpoint={endpoint}): {' '.join(cmd)}") print(f" Log file: {log_file}") # Redirect stdout + stderr directly to the log file at the OS level @@ -597,7 +599,7 @@ def run_benchmark( record: dict[str, Any] = { "test_name": test_name, - "backend": backend, + "endpoint": endpoint, "timestamp": timestamp, "server_params": server_cfg.get("server_params"), "benchmark_params": params, @@ -605,7 +607,7 @@ def run_benchmark( "log_file": str(log_file), "Model": model, "Framework": server_type, - "API Backend": backend, + "API Endpoint": endpoint, "Hardware": "", "Deployment": "", "Task": params.get("task", "t2i"), @@ -669,21 +671,21 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None: assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}" -def _default_benchmark_backend_for_task(task: str) -> str: - """Return the default client-side benchmark backend for a diffusion task.""" +def _default_benchmark_endpoint_for_task(task: str) -> str: + """Return the default client-side benchmark endpoint for a diffusion task.""" if task in {"t2v", "i2v", "ti2v"}: - return "v1/videos" + return "/v1/videos" if task in {"t2i", "i2i", "ti2i"}: - return "vllm-omni" - raise ValueError(f"Unsupported task for benchmark backend resolution: {task}") + return "/v1/chat/completions" + raise ValueError(f"Unsupported task for benchmark endpoint resolution: {task}") -def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any]) -> str: +def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, Any]) -> str: """Resolve which serving API the benchmark client should call.""" - configured = server_cfg.get("benchmark_backend") + configured = server_cfg.get("benchmark_endpoint") if configured: - return cast(str, configured) - return _default_benchmark_backend_for_task(cast(str, params.get("task", "t2i"))) + return normalize_endpoint(cast(str, configured)) + return _default_benchmark_endpoint_for_task(cast(str, params.get("task", "t2i"))) # --------------------------------------------------------------------------- @@ -711,7 +713,7 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): test_name = benchmark_params["test_name"] params = benchmark_params["params"] server_cfg = getattr(diffusion_server, "server_cfg", {}) - backend = _resolve_benchmark_backend(server_cfg, params) + endpoint = _resolve_benchmark_endpoint(server_cfg, params) result = run_benchmark( host=diffusion_server.host, @@ -719,13 +721,13 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): model=diffusion_server.model, params=params, test_name=test_name, - backend=backend, + endpoint=endpoint, server_cfg=server_cfg, source_file=cast(str, CONFIG_FILE_PATH), ) print(f"\n{'=' * 60}") - print(f"Results for {test_name} (server={diffusion_server.server_type}, backend={backend}):") + print(f"Results for {test_name} (server={diffusion_server.server_type}, endpoint={endpoint}):") for key in ( "throughput_qps", "latency_mean", diff --git a/tests/test_generate_nightly_perf_excel.py b/tests/test_generate_nightly_perf_excel.py index 9b05d6de0..6658b89b8 100644 --- a/tests/test_generate_nightly_perf_excel.py +++ b/tests/test_generate_nightly_perf_excel.py @@ -35,6 +35,8 @@ def test_generate_excel_report_with_perf_templates(tmp_path: Path): omni_record = json.loads(omni_template_path.read_text(encoding="utf-8")) diffusion_records = json.loads(diffusion_template_path.read_text(encoding="utf-8")) + diffusion_records[0]["endpoint"] = "/v1/videos" + diffusion_records[0]["result"]["endpoint"] = "/v1/videos" input_dir = tmp_path / "input" diffusion_input_dir = tmp_path / "diffusion_input" @@ -69,3 +71,6 @@ def test_generate_excel_report_with_perf_templates(tmp_path: Path): ws_omni_summary = wb["omni_summary"] assert _cell_value_by_header(ws_omni_summary, "commit_sha") == "test_commit_sha" assert _cell_value_by_header(ws_omni_summary, "build_id") == "test_build_id" + + ws_diff_summary = wb["diffusion_summary"] + assert _cell_value_by_header(ws_diff_summary, "endpoint") == "/v1/videos" diff --git a/tests/test_generate_nightly_perf_html.py b/tests/test_generate_nightly_perf_html.py index 4e77eb3ad..7a05165da 100644 --- a/tests/test_generate_nightly_perf_html.py +++ b/tests/test_generate_nightly_perf_html.py @@ -28,6 +28,8 @@ def test_generate_html_report_with_perf_templates(tmp_path: Path): omni_record = json.loads(omni_template_path.read_text(encoding="utf-8")) diffusion_records = json.loads(diffusion_template_path.read_text(encoding="utf-8")) + diffusion_records[0]["endpoint"] = "/v1/videos" + diffusion_records[0]["result"]["endpoint"] = "/v1/videos" input_dir = tmp_path / "input" diffusion_input_dir = tmp_path / "diffusion_input" @@ -52,3 +54,4 @@ def test_generate_html_report_with_perf_templates(tmp_path: Path): assert "Omni records 1" in html assert f"Diffusion records {len(diffusion_records)}" in html assert "const DIFF_DATA =" in html + assert '"endpoint": "/v1/videos"' in html diff --git a/tools/nightly/generate_nightly_perf_excel.py b/tools/nightly/generate_nightly_perf_excel.py index 6ba1d1eef..e76dc4346 100644 --- a/tools/nightly/generate_nightly_perf_excel.py +++ b/tools/nightly/generate_nightly_perf_excel.py @@ -18,6 +18,8 @@ from openpyxl import Workbook from openpyxl.styles import PatternFill from openpyxl.utils import get_column_letter +from benchmarks.diffusion.backends import normalize_endpoint + LOGGER = logging.getLogger(__name__) GREY_BLOCK_FILL = PatternFill(start_color="D3D3D3", fill_type="solid") @@ -60,7 +62,7 @@ DIFFUSION_SUMMARY_COLUMNS: tuple[str, ...] = ( "date", "test_name", "model", - "backend", + "endpoint", "dataset", "task", "completed_requests", @@ -528,6 +530,12 @@ def _process_diffusion_record(record: dict[str, Any]) -> dict[str, Any]: """Normalize a diffusion record by merging `result` and flattening stage metrics.""" flat = record.copy() flat.update(flat.pop("result", {})) + if flat.get("endpoint"): + flat["endpoint"] = normalize_endpoint(flat["endpoint"]) + elif flat.get("backend"): + flat["endpoint"] = normalize_endpoint(flat["backend"]) + flat.pop("backend", None) + flat.pop("API Backend", None) flat = _flatten_stage_durations(flat) flat.pop("benchmark_params", None) flat.pop("server_params", None) @@ -752,6 +760,7 @@ _OMNI_SUMMARY_WIDTHS = { _DIFFUSION_SUMMARY_WIDTHS = { "test_name": 30, "model": 15, + "endpoint": 24, } diff --git a/tools/nightly/generate_nightly_perf_html.py b/tools/nightly/generate_nightly_perf_html.py index a50a46255..73e245ab1 100644 --- a/tools/nightly/generate_nightly_perf_html.py +++ b/tools/nightly/generate_nightly_perf_html.py @@ -14,6 +14,8 @@ from collections.abc import Iterable, Sequence from datetime import datetime, timezone from typing import Any +from benchmarks.diffusion.backends import normalize_endpoint + LOGGER = logging.getLogger(__name__) _RESULT_JSON_PREFIX = "result_test_" @@ -222,6 +224,12 @@ def _iter_diffusion_json_records(input_dir: str) -> Iterable[dict[str, Any]]: result = flat.pop("result", None) if isinstance(result, dict): flat.update(result) + if flat.get("endpoint"): + flat["endpoint"] = normalize_endpoint(flat["endpoint"]) + elif flat.get("backend"): + flat["endpoint"] = normalize_endpoint(flat["backend"]) + flat.pop("backend", None) + flat.pop("API Backend", None) if "date" not in flat or not flat.get("date"): flat["date"] = filename_meta.get("date") or datetime.now( timezone.utc, @@ -1183,7 +1191,7 @@ function filterRows(rows, filters) {{ ) {{ return false; }} - if (filters.backend && String(row.backend || "") !== filters.backend) return false; + if (filters.backend && String(row[filters.backendKey] || "") !== filters.backend) return false; if ( filters.extraKey && filters.extraValue && @@ -1223,12 +1231,14 @@ function initSection(prefix, columns, data, numericCols, groups) {{ const modelKey = prefix === "diff" ? "model" : "model_id"; const datasetKey = prefix === "diff" ? "dataset" : "dataset_name"; + const backendKey = prefix === "diff" ? "endpoint" : "backend"; + const backendLabel = prefix === "diff" ? "Endpoint" : "Backend"; const extraKey = prefix === "diff" ? "" : "tokenizer_id"; const metaKeys = prefix === "diff" ? ["test_name"] : ["test_name", "max_concurrency", "num_prompts"]; const configFields = prefix === "diff" - ? ["test_name", "model", "backend", "dataset"] + ? ["test_name", "model", "endpoint", "dataset"] : [ "endpoint_type", "backend", @@ -1265,6 +1275,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{ extraKey, modelKey, datasetKey, + backendKey, }}; }} @@ -1296,7 +1307,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{ }} fillSelect( backendList, - uniqSorted(baseRows.map((row) => row.backend)), + uniqSorted(baseRows.map((row) => row[backendKey])), backendInput.value, ); if (extraList) {{ @@ -1378,7 +1389,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{ }}); if (!backendInput.value.trim()) {{ - backendInput.value = firstValue(rows, "backend"); + backendInput.value = firstValue(rows, backendKey); }} rows = filterRows(data, {{ ...currentFilters(), @@ -1465,7 +1476,7 @@ function initSection(prefix, columns, data, numericCols, groups) {{ `
${{escapeHtml(filters.numPrompts || "All")}}
` ) }} -
Backend
${{escapeHtml( +
${{backendLabel}}
${{escapeHtml( filters.backend || "All", )}}
${{ @@ -1707,7 +1718,7 @@ window.addEventListener("load", () => {{ 'Diffusion
Image generation ' "benchmark history with throughput and latency trend views.
" '
' - "Grouping test_name + model + backend + dataset
" + "Grouping test_name + model + endpoint + dataset
" '
Snapshots single-point series ' "stay readable
" ), @@ -1736,7 +1747,7 @@ window.addEventListener("load", () => {{ " ", '
', ( - '
' + '
' '
' ), ( @@ -1803,7 +1814,7 @@ def generate_html_report(input_dir: str, diffusion_input_dir: str, output_file: "date", "test_name", "model", - "backend", + "endpoint", "dataset", "task", "duration",