diff --git a/benchmarks/diffusion/backends.py b/benchmarks/diffusion/backends.py index b7e533b2d..368dc5dcb 100644 --- a/benchmarks/diffusion/backends.py +++ b/benchmarks/diffusion/backends.py @@ -10,6 +10,8 @@ from typing import Any import aiohttp from tqdm import tqdm +DEFAULT_EDITS_BOT_TASK = "think" + @dataclass class RequestFuncInput: @@ -27,6 +29,7 @@ class RequestFuncInput: extra_body: dict[str, Any] = field(default_factory=dict) image_paths: list[str] | None = None request_id: str = field(default_factory=lambda: str(uuid.uuid4())) + default_bot_task: str | None = DEFAULT_EDITS_BOT_TASK @dataclass @@ -53,6 +56,95 @@ def _encode_image_as_data_url(path: str) -> str: return f"data:{mime};base64,{encoded}" +async def async_request_image_edits( + input: RequestFuncInput, + session: aiohttp.ClientSession, + pbar: tqdm | None = None, + enable_diffusion_pipeline_profiler: bool = False, +) -> RequestFuncOutput: + """POST /v1/images/edits (multipart).""" + del enable_diffusion_pipeline_profiler + output = RequestFuncOutput() + output.start_time = time.perf_counter() + + extra_body = dict(input.extra_body) + width = input.width or extra_body.get("width") or 1024 + height = input.height or extra_body.get("height") or 1024 + edits_url = input.api_url + + form = aiohttp.FormData() + form.add_field("model", input.model) + form.add_field("prompt", input.prompt) + form.add_field("size", f"{width}x{height}") + form.add_field("response_format", "b64_json") + + if input.num_inference_steps is not None: + form.add_field("num_inference_steps", str(input.num_inference_steps)) + elif extra_body.get("num_inference_steps") is not None: + form.add_field("num_inference_steps", str(extra_body["num_inference_steps"])) + + if input.seed is not None: + form.add_field("seed", str(input.seed)) + elif extra_body.get("seed") is not None: + form.add_field("seed", str(extra_body["seed"])) + + if extra_body.get("guidance_scale") is not None: + form.add_field("guidance_scale", str(extra_body["guidance_scale"])) + if extra_body.get("negative_prompt") is not None: + form.add_field("negative_prompt", str(extra_body["negative_prompt"])) + if extra_body.get("true_cfg_scale") is not None: + form.add_field("true_cfg_scale", str(extra_body["true_cfg_scale"])) + if extra_body.get("sys_type") is not None: + form.add_field("sys_type", str(extra_body["sys_type"])) + if extra_body.get("system_prompt") is not None: + form.add_field("system_prompt", str(extra_body["system_prompt"])) + + bot_task = extra_body.get("bot_task") + if bot_task is None and input.default_bot_task is not None: + bot_task = input.default_bot_task + if bot_task is not None: + form.add_field("bot_task", str(bot_task)) + + assert input.image_paths is not None + for img_path in input.image_paths: + if not os.path.exists(img_path): + output.error = f"Image file not found: {img_path}" + output.success = False + if pbar: + pbar.update(1) + return output + with open(img_path, "rb") as img_f: + image_bytes = img_f.read() + form.add_field( + "image", + image_bytes, + filename=os.path.basename(img_path), + content_type=_guess_mime_type(img_path), + ) + + try: + async with session.post(edits_url, data=form) as response: + if response.status == 200: + resp_json = await response.json() + output.response_body = resp_json + output.success = True + else: + output.error = f"HTTP {response.status}: {await response.text()}" + output.success = False + except Exception as e: + output.error = str(e) + output.success = False + + output.latency = time.perf_counter() - output.start_time + + if output.success and input.slo_ms is not None: + output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms) + + if pbar: + pbar.update(1) + return output + + async def async_request_chat_completions( input: RequestFuncInput, session: aiohttp.ClientSession, @@ -378,6 +470,7 @@ backends_function_mapping = { "2i": { "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"), "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"), + "/v1/images/edits": (async_request_image_edits, "/v1/images/edits"), }, "2v": { "/v1/videos": (async_request_v1_videos, "/v1/videos"), diff --git a/benchmarks/diffusion/diffusion_benchmark_serving.py b/benchmarks/diffusion/diffusion_benchmark_serving.py index c59844f65..7e8ea97e1 100644 --- a/benchmarks/diffusion/diffusion_benchmark_serving.py +++ b/benchmarks/diffusion/diffusion_benchmark_serving.py @@ -7,10 +7,13 @@ Benchmark online serving for diffusion models (Image/Video Generation). If you want to use i2v, i2i dataset, you should `uv pip install gdown` first Supports multiple endpoints: - - /v1/chat/completions: OpenAI chat-compatible image requests + - /v1/chat/completions: OpenAI chat-compatible image requests (e.g. t2i, Qwen i2i) + - /v1/images/edits: OpenAI image edit / IT2I (multipart; e.g. Hunyuan --bot-task think) - /v1/images/generations: OpenAI image generation requests - /v1/videos: Async video jobs +Legacy --backend vllm-omni and openai are aliases for chat/completions and images/generations. + Usage: # Video (/v1/videos endpoint) t2v: @@ -39,7 +42,12 @@ Usage: {"width":1536,"height":1536,"num_inference_steps":35,"weight":0.15} ]' - i2i: + ti2i (Hunyuan / OpenAI image edit API): + python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ + --endpoint /v1/images/edits --dataset random --task ti2i --num-prompts 10 \ + --bot-task think + + i2i (chat-based models such as Qwen-Image-Edit): python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10 @@ -132,7 +140,7 @@ class VBenchDataset(BaseDataset): def _load_data(self) -> list[dict[str, Any]]: if self.args.task == "t2v": return self._load_t2v_prompts() - elif self.args.task in ["i2v", "ti2v", "ti2i", "i2i"]: + elif self.args.task in ["i2v", "ti2v", "ti2i", "i2i", "it2i"]: return self._load_i2v_data() else: return self._load_t2v_prompts() @@ -524,7 +532,7 @@ class TraceDataset(BaseDataset): single = row.get("image_path") image_paths = [single] if single else None - if not image_paths and self.args.task in ["i2v", "i2i", "ti2v", "ti2i"]: + if not image_paths and self.args.task in ["i2v", "i2i", "ti2v", "ti2i", "it2i"]: raise ValueError( f"Task {self.args.task} requires image input, but no image_path or image_paths found in trace row." ) @@ -585,7 +593,7 @@ class RandomDataset(BaseDataset): self._sampled_requests = None # Random image generate - if self.args.task in ["i2v", "ti2v", "ti2i", "i2i"]: + if self.args.task in ["i2v", "ti2v", "ti2i", "i2i", "it2i"]: self._random_image_path = self._generate_random_image_paths() else: self._random_image_path = None @@ -891,7 +899,9 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None: def _default_endpoint_for_task(task: str) -> str: if task in {"t2v", "i2v", "ti2v"}: return "/v1/videos" - if task in {"t2i", "i2i", "ti2i"}: + if task in {"i2i", "ti2i", "it2i"}: + return "/v1/images/edits" + if task == "t2i": return "/v1/chat/completions" raise ValueError(f"Unsupported task for endpoint resolution: {task}") @@ -902,7 +912,7 @@ async def benchmark(args): args.base_url = f"http://{args.host}:{args.port}" VIDEO_TASKS = {"t2v", "i2v", "ti2v"} - IMAGE_TASKS = {"t2i", "i2i", "ti2i"} + IMAGE_TASKS = {"t2i", "i2i", "ti2i", "it2i"} if args.task in VIDEO_TASKS: task_type = "2v" @@ -947,6 +957,10 @@ async def benchmark(args): requests_list = dataset.get_requests() print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.") + if args.endpoint == "/v1/images/edits": + for req in requests_list: + req.default_bot_task = args.bot_task + # Limit concurrency if args.max_concurrency is not None: semaphore = asyncio.Semaphore(args.max_concurrency) @@ -998,6 +1012,8 @@ async def benchmark(args): metrics["model"] = args.model metrics["dataset"] = args.dataset metrics["task"] = args.task + if args.endpoint == "/v1/images/edits": + metrics["bot_task"] = args.bot_task print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) @@ -1088,7 +1104,7 @@ if __name__ == "__main__": "--task", type=str, default="t2v", - choices=["t2v", "i2v", "ti2v", "ti2i", "i2i", "t2i"], + choices=["t2v", "i2v", "ti2v", "ti2i", "i2i", "it2i", "t2i"], help="Task type.", ) parser.add_argument( @@ -1195,9 +1211,15 @@ if __name__ == "__main__": default=1, help=( "Number of synthetic input images to attach for image-conditioned tasks " - "(i2v, ti2v, ti2i, i2i) when using random dataset." + "(i2v, ti2v, ti2i, i2i, it2i) when using random dataset." ), ) + parser.add_argument( + "--bot-task", + type=str, + default="think", + help=("bot_task form field for --endpoint /v1/images/edits (think, recaption, think_recaption, vanilla)."), + ) args = parser.parse_args() asyncio.run(benchmark(args))