[Perf] Qwen-Image Performance Nightly CI test (#1805)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
df93b3a965
commit
1f5eca5686
13 files changed
+1291
-4
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
@@ -109,8 +110,6 @@ async def async_request_chat_completions(
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
try:
|
||||
choices = resp_json.get("choices", [])
|
||||
if choices and isinstance(choices, list):
|
||||
@@ -120,7 +119,8 @@ async def async_request_chat_completions(
|
||||
if content and isinstance(content, list) and len(content) > 0:
|
||||
first_item = content[0]
|
||||
if isinstance(first_item, dict):
|
||||
output.stage_durations = first_item.get("stage_durations")
|
||||
output.stage_durations = first_item.get("stage_durations") or {}
|
||||
output.peak_memory_mb = first_item.get("peak_memory_mb", 0.0)
|
||||
except (IndexError, TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
@@ -335,12 +335,263 @@ async def async_request_v1_videos(
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_image_sglang(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output.start_time = time.perf_counter()
|
||||
|
||||
# Check if we need to use multipart (for image edits with input images)
|
||||
if input.image_paths and len(input.image_paths) > 0:
|
||||
# Use multipart/form-data for image edits
|
||||
data = aiohttp.FormData()
|
||||
data.add_field("model", input.model)
|
||||
data.add_field("prompt", input.prompt)
|
||||
data.add_field("response_format", "b64_json")
|
||||
|
||||
if input.width and input.height:
|
||||
data.add_field("size", f"{input.width}x{input.height}")
|
||||
|
||||
# Merge extra parameters
|
||||
for key, value in input.extra_body.items():
|
||||
data.add_field(key, str(value))
|
||||
|
||||
# Add image file(s)
|
||||
for idx, img_path in enumerate(input.image_paths):
|
||||
if os.path.exists(img_path):
|
||||
data.add_field(
|
||||
"image",
|
||||
open(img_path, "rb"),
|
||||
filename=os.path.basename(img_path),
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
else:
|
||||
output.error = f"Image file not found: {img_path}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception as e:
|
||||
output.error = str(e)
|
||||
output.success = False
|
||||
else:
|
||||
# Use JSON for text-to-image generation
|
||||
payload = {
|
||||
"model": input.model,
|
||||
"prompt": input.prompt,
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
}
|
||||
|
||||
if input.width and input.height:
|
||||
payload["size"] = f"{input.width}x{input.height}"
|
||||
|
||||
if input.num_inference_steps:
|
||||
payload["num_inference_steps"] = input.num_inference_steps
|
||||
|
||||
payload.update(input.extra_body)
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, json=payload) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception as e:
|
||||
output.error = str(e)
|
||||
output.success = False
|
||||
|
||||
output.latency = time.perf_counter() - output.start_time
|
||||
|
||||
# Check SLO if defined
|
||||
if input.slo_ms is not None and output.success:
|
||||
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_video_sglang(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output.start_time = time.perf_counter()
|
||||
|
||||
# 1. Submit Job
|
||||
job_id = None
|
||||
# Check if we need to upload images (Multipart) or just send JSON
|
||||
if input.image_paths and len(input.image_paths) > 0:
|
||||
# Use multipart/form-data
|
||||
data = aiohttp.FormData()
|
||||
data.add_field("model", input.model)
|
||||
data.add_field("prompt", input.prompt)
|
||||
|
||||
if input.width and input.height:
|
||||
data.add_field("size", f"{input.width}x{input.height}")
|
||||
|
||||
# Add extra body fields to form data if possible, or assume simple key-values
|
||||
# Note: Nested dicts in extra_body might need JSON serialization if API expects it stringified
|
||||
if input.extra_body:
|
||||
data.add_field("extra_body", json.dumps(input.extra_body))
|
||||
|
||||
# Explicitly add fps/num_frames if they are not in extra_body (bench_serving logic overrides)
|
||||
if input.num_frames:
|
||||
data.add_field("num_frames", str(input.num_frames))
|
||||
if input.fps:
|
||||
data.add_field("fps", str(input.fps))
|
||||
|
||||
# Add image file
|
||||
# Currently only support single image upload as 'input_reference' per API spec
|
||||
img_path = input.image_paths[0]
|
||||
if os.path.exists(img_path):
|
||||
data.add_field(
|
||||
"input_reference",
|
||||
open(img_path, "rb"),
|
||||
filename=os.path.basename(img_path),
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
else:
|
||||
output.error = f"Image file not found: {img_path}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
job_id = resp_json.get("id")
|
||||
else:
|
||||
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
except Exception as e:
|
||||
output.error = f"Submit exception: {str(e)}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
else:
|
||||
# Use JSON
|
||||
payload: dict[str, Any] = {
|
||||
"model": input.model,
|
||||
"prompt": input.prompt,
|
||||
}
|
||||
if input.width and input.height:
|
||||
payload["size"] = f"{input.width}x{input.height}"
|
||||
if input.num_frames:
|
||||
payload["num_frames"] = input.num_frames
|
||||
if input.fps:
|
||||
payload["fps"] = input.fps
|
||||
if input.num_inference_steps:
|
||||
payload["num_inference_steps"] = input.num_inference_steps
|
||||
|
||||
payload.update(input.extra_body)
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, json=payload) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
job_id = resp_json.get("id")
|
||||
else:
|
||||
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
except Exception as e:
|
||||
output.error = f"Submit exception: {str(e)}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
if not job_id:
|
||||
output.error = "No job_id returned"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
# 2. Poll for completion
|
||||
# Assuming the API returns a 'status' field.
|
||||
# We construct the check URL. Assuming api_url is like .../v1/videos
|
||||
# The check url should be .../v1/videos/{id}
|
||||
check_url = f"{input.api_url}/{job_id}"
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with session.get(check_url) as response:
|
||||
if response.status == 200:
|
||||
status_data = await response.json()
|
||||
status = status_data.get("status")
|
||||
if status == "completed":
|
||||
output.success = True
|
||||
output.response_body = status_data
|
||||
if "peak_memory_mb" in status_data:
|
||||
output.peak_memory_mb = status_data["peak_memory_mb"]
|
||||
break
|
||||
elif status == "failed":
|
||||
output.success = False
|
||||
output.error = f"Job failed: {status_data.get('error')}"
|
||||
break
|
||||
else:
|
||||
# queued or processing
|
||||
await asyncio.sleep(1.0)
|
||||
else:
|
||||
output.success = False
|
||||
output.error = f"Poll failed HTTP {response.status}: {await response.text()}"
|
||||
break
|
||||
except Exception as e:
|
||||
output.success = False
|
||||
output.error = f"Poll exception: {str(e)}"
|
||||
break
|
||||
|
||||
output.latency = time.perf_counter() - output.start_time
|
||||
|
||||
# Check SLO if defined
|
||||
if input.slo_ms is not None and output.success:
|
||||
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
backends_function_mapping = {
|
||||
"2i": {
|
||||
"vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
|
||||
"openai": (async_request_openai_images, "/v1/images/generations"),
|
||||
"sglang": (async_request_image_sglang, "/v1/images/generations"),
|
||||
},
|
||||
"2v": {
|
||||
"v1/videos": (async_request_v1_videos, "/v1/videos"),
|
||||
"sglang": (async_request_video_sglang, "/v1/videos"),
|
||||
},
|
||||
}
|
||||
Reference in new issue
Block a user