[Perf] Qwen-Image Performance Nightly CI test (#1805)

Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
Didan Deng authored and GitHub committed 2026-03-24 00:19:20 +08:00
1 parent df93b3a965
commit 1f5eca5686
13 files changed
+1291 -4

No files matched your search

+254 -3
View File
@@ -1,5 +1,6 @@
import asyncio
import base64
import json
import mimetypes
import os
import time
@@ -109,8 +110,6 @@ async def async_request_chat_completions(
resp_json = await response.json()
output.response_body = resp_json
output.success = True
if "peak_memory_mb" in resp_json:
output.peak_memory_mb = resp_json["peak_memory_mb"]
try:
choices = resp_json.get("choices", [])
if choices and isinstance(choices, list):
@@ -120,7 +119,8 @@ async def async_request_chat_completions(
if content and isinstance(content, list) and len(content) > 0:
first_item = content[0]
if isinstance(first_item, dict):
output.stage_durations = first_item.get("stage_durations")
output.stage_durations = first_item.get("stage_durations") or {}
output.peak_memory_mb = first_item.get("peak_memory_mb", 0.0)
except (IndexError, TypeError, AttributeError):
pass
else:
@@ -335,12 +335,263 @@ async def async_request_v1_videos(
return output
async def async_request_image_sglang(
input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
output = RequestFuncOutput()
output.start_time = time.perf_counter()
# Check if we need to use multipart (for image edits with input images)
if input.image_paths and len(input.image_paths) > 0:
# Use multipart/form-data for image edits
data = aiohttp.FormData()
data.add_field("model", input.model)
data.add_field("prompt", input.prompt)
data.add_field("response_format", "b64_json")
if input.width and input.height:
data.add_field("size", f"{input.width}x{input.height}")
# Merge extra parameters
for key, value in input.extra_body.items():
data.add_field(key, str(value))
# Add image file(s)
for idx, img_path in enumerate(input.image_paths):
if os.path.exists(img_path):
data.add_field(
"image",
open(img_path, "rb"),
filename=os.path.basename(img_path),
content_type="application/octet-stream",
)
else:
output.error = f"Image file not found: {img_path}"
output.success = False
if pbar:
pbar.update(1)
return output
try:
async with session.post(input.api_url, data=data) as response:
if response.status == 200:
resp_json = await response.json()
output.response_body = resp_json
output.success = True
if "peak_memory_mb" in resp_json:
output.peak_memory_mb = resp_json["peak_memory_mb"]
else:
output.error = f"HTTP {response.status}: {await response.text()}"
output.success = False
except Exception as e:
output.error = str(e)
output.success = False
else:
# Use JSON for text-to-image generation
payload = {
"model": input.model,
"prompt": input.prompt,
"n": 1,
"response_format": "b64_json",
}
if input.width and input.height:
payload["size"] = f"{input.width}x{input.height}"
if input.num_inference_steps:
payload["num_inference_steps"] = input.num_inference_steps
payload.update(input.extra_body)
try:
async with session.post(input.api_url, json=payload) as response:
if response.status == 200:
resp_json = await response.json()
output.response_body = resp_json
output.success = True
if "peak_memory_mb" in resp_json:
output.peak_memory_mb = resp_json["peak_memory_mb"]
else:
output.error = f"HTTP {response.status}: {await response.text()}"
output.success = False
except Exception as e:
output.error = str(e)
output.success = False
output.latency = time.perf_counter() - output.start_time
# Check SLO if defined
if input.slo_ms is not None and output.success:
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
if pbar:
pbar.update(1)
return output
async def async_request_video_sglang(
input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
output = RequestFuncOutput()
output.start_time = time.perf_counter()
# 1. Submit Job
job_id = None
# Check if we need to upload images (Multipart) or just send JSON
if input.image_paths and len(input.image_paths) > 0:
# Use multipart/form-data
data = aiohttp.FormData()
data.add_field("model", input.model)
data.add_field("prompt", input.prompt)
if input.width and input.height:
data.add_field("size", f"{input.width}x{input.height}")
# Add extra body fields to form data if possible, or assume simple key-values
# Note: Nested dicts in extra_body might need JSON serialization if API expects it stringified
if input.extra_body:
data.add_field("extra_body", json.dumps(input.extra_body))
# Explicitly add fps/num_frames if they are not in extra_body (bench_serving logic overrides)
if input.num_frames:
data.add_field("num_frames", str(input.num_frames))
if input.fps:
data.add_field("fps", str(input.fps))
# Add image file
# Currently only support single image upload as 'input_reference' per API spec
img_path = input.image_paths[0]
if os.path.exists(img_path):
data.add_field(
"input_reference",
open(img_path, "rb"),
filename=os.path.basename(img_path),
content_type="application/octet-stream",
)
else:
output.error = f"Image file not found: {img_path}"
output.success = False
if pbar:
pbar.update(1)
return output
try:
async with session.post(input.api_url, data=data) as response:
if response.status == 200:
resp_json = await response.json()
job_id = resp_json.get("id")
else:
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
output.success = False
if pbar:
pbar.update(1)
return output
except Exception as e:
output.error = f"Submit exception: {str(e)}"
output.success = False
if pbar:
pbar.update(1)
return output
else:
# Use JSON
payload: dict[str, Any] = {
"model": input.model,
"prompt": input.prompt,
}
if input.width and input.height:
payload["size"] = f"{input.width}x{input.height}"
if input.num_frames:
payload["num_frames"] = input.num_frames
if input.fps:
payload["fps"] = input.fps
if input.num_inference_steps:
payload["num_inference_steps"] = input.num_inference_steps
payload.update(input.extra_body)
try:
async with session.post(input.api_url, json=payload) as response:
if response.status == 200:
resp_json = await response.json()
job_id = resp_json.get("id")
else:
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
output.success = False
if pbar:
pbar.update(1)
return output
except Exception as e:
output.error = f"Submit exception: {str(e)}"
output.success = False
if pbar:
pbar.update(1)
return output
if not job_id:
output.error = "No job_id returned"
output.success = False
if pbar:
pbar.update(1)
return output
# 2. Poll for completion
# Assuming the API returns a 'status' field.
# We construct the check URL. Assuming api_url is like .../v1/videos
# The check url should be .../v1/videos/{id}
check_url = f"{input.api_url}/{job_id}"
while True:
try:
async with session.get(check_url) as response:
if response.status == 200:
status_data = await response.json()
status = status_data.get("status")
if status == "completed":
output.success = True
output.response_body = status_data
if "peak_memory_mb" in status_data:
output.peak_memory_mb = status_data["peak_memory_mb"]
break
elif status == "failed":
output.success = False
output.error = f"Job failed: {status_data.get('error')}"
break
else:
# queued or processing
await asyncio.sleep(1.0)
else:
output.success = False
output.error = f"Poll failed HTTP {response.status}: {await response.text()}"
break
except Exception as e:
output.success = False
output.error = f"Poll exception: {str(e)}"
break
output.latency = time.perf_counter() - output.start_time
# Check SLO if defined
if input.slo_ms is not None and output.success:
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
if pbar:
pbar.update(1)
return output
backends_function_mapping = {
"2i": {
"vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
"openai": (async_request_openai_images, "/v1/images/generations"),
"sglang": (async_request_image_sglang, "/v1/images/generations"),
},
"2v": {
"v1/videos": (async_request_v1_videos, "/v1/videos"),
"sglang": (async_request_video_sglang, "/v1/videos"),
},
}