[Perf] Qwen-Image Performance Nightly CI test (#1805)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
df93b3a965
commit
1f5eca5686
13 files changed
+1291
-4
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
@@ -109,8 +110,6 @@ async def async_request_chat_completions(
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
try:
|
||||
choices = resp_json.get("choices", [])
|
||||
if choices and isinstance(choices, list):
|
||||
@@ -120,7 +119,8 @@ async def async_request_chat_completions(
|
||||
if content and isinstance(content, list) and len(content) > 0:
|
||||
first_item = content[0]
|
||||
if isinstance(first_item, dict):
|
||||
output.stage_durations = first_item.get("stage_durations")
|
||||
output.stage_durations = first_item.get("stage_durations") or {}
|
||||
output.peak_memory_mb = first_item.get("peak_memory_mb", 0.0)
|
||||
except (IndexError, TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
@@ -335,12 +335,263 @@ async def async_request_v1_videos(
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_image_sglang(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output.start_time = time.perf_counter()
|
||||
|
||||
# Check if we need to use multipart (for image edits with input images)
|
||||
if input.image_paths and len(input.image_paths) > 0:
|
||||
# Use multipart/form-data for image edits
|
||||
data = aiohttp.FormData()
|
||||
data.add_field("model", input.model)
|
||||
data.add_field("prompt", input.prompt)
|
||||
data.add_field("response_format", "b64_json")
|
||||
|
||||
if input.width and input.height:
|
||||
data.add_field("size", f"{input.width}x{input.height}")
|
||||
|
||||
# Merge extra parameters
|
||||
for key, value in input.extra_body.items():
|
||||
data.add_field(key, str(value))
|
||||
|
||||
# Add image file(s)
|
||||
for idx, img_path in enumerate(input.image_paths):
|
||||
if os.path.exists(img_path):
|
||||
data.add_field(
|
||||
"image",
|
||||
open(img_path, "rb"),
|
||||
filename=os.path.basename(img_path),
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
else:
|
||||
output.error = f"Image file not found: {img_path}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception as e:
|
||||
output.error = str(e)
|
||||
output.success = False
|
||||
else:
|
||||
# Use JSON for text-to-image generation
|
||||
payload = {
|
||||
"model": input.model,
|
||||
"prompt": input.prompt,
|
||||
"n": 1,
|
||||
"response_format": "b64_json",
|
||||
}
|
||||
|
||||
if input.width and input.height:
|
||||
payload["size"] = f"{input.width}x{input.height}"
|
||||
|
||||
if input.num_inference_steps:
|
||||
payload["num_inference_steps"] = input.num_inference_steps
|
||||
|
||||
payload.update(input.extra_body)
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, json=payload) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
output.response_body = resp_json
|
||||
output.success = True
|
||||
if "peak_memory_mb" in resp_json:
|
||||
output.peak_memory_mb = resp_json["peak_memory_mb"]
|
||||
else:
|
||||
output.error = f"HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
except Exception as e:
|
||||
output.error = str(e)
|
||||
output.success = False
|
||||
|
||||
output.latency = time.perf_counter() - output.start_time
|
||||
|
||||
# Check SLO if defined
|
||||
if input.slo_ms is not None and output.success:
|
||||
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_video_sglang(
|
||||
input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output.start_time = time.perf_counter()
|
||||
|
||||
# 1. Submit Job
|
||||
job_id = None
|
||||
# Check if we need to upload images (Multipart) or just send JSON
|
||||
if input.image_paths and len(input.image_paths) > 0:
|
||||
# Use multipart/form-data
|
||||
data = aiohttp.FormData()
|
||||
data.add_field("model", input.model)
|
||||
data.add_field("prompt", input.prompt)
|
||||
|
||||
if input.width and input.height:
|
||||
data.add_field("size", f"{input.width}x{input.height}")
|
||||
|
||||
# Add extra body fields to form data if possible, or assume simple key-values
|
||||
# Note: Nested dicts in extra_body might need JSON serialization if API expects it stringified
|
||||
if input.extra_body:
|
||||
data.add_field("extra_body", json.dumps(input.extra_body))
|
||||
|
||||
# Explicitly add fps/num_frames if they are not in extra_body (bench_serving logic overrides)
|
||||
if input.num_frames:
|
||||
data.add_field("num_frames", str(input.num_frames))
|
||||
if input.fps:
|
||||
data.add_field("fps", str(input.fps))
|
||||
|
||||
# Add image file
|
||||
# Currently only support single image upload as 'input_reference' per API spec
|
||||
img_path = input.image_paths[0]
|
||||
if os.path.exists(img_path):
|
||||
data.add_field(
|
||||
"input_reference",
|
||||
open(img_path, "rb"),
|
||||
filename=os.path.basename(img_path),
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
else:
|
||||
output.error = f"Image file not found: {img_path}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, data=data) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
job_id = resp_json.get("id")
|
||||
else:
|
||||
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
except Exception as e:
|
||||
output.error = f"Submit exception: {str(e)}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
else:
|
||||
# Use JSON
|
||||
payload: dict[str, Any] = {
|
||||
"model": input.model,
|
||||
"prompt": input.prompt,
|
||||
}
|
||||
if input.width and input.height:
|
||||
payload["size"] = f"{input.width}x{input.height}"
|
||||
if input.num_frames:
|
||||
payload["num_frames"] = input.num_frames
|
||||
if input.fps:
|
||||
payload["fps"] = input.fps
|
||||
if input.num_inference_steps:
|
||||
payload["num_inference_steps"] = input.num_inference_steps
|
||||
|
||||
payload.update(input.extra_body)
|
||||
|
||||
try:
|
||||
async with session.post(input.api_url, json=payload) as response:
|
||||
if response.status == 200:
|
||||
resp_json = await response.json()
|
||||
job_id = resp_json.get("id")
|
||||
else:
|
||||
output.error = f"Submit failed HTTP {response.status}: {await response.text()}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
except Exception as e:
|
||||
output.error = f"Submit exception: {str(e)}"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
if not job_id:
|
||||
output.error = "No job_id returned"
|
||||
output.success = False
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
# 2. Poll for completion
|
||||
# Assuming the API returns a 'status' field.
|
||||
# We construct the check URL. Assuming api_url is like .../v1/videos
|
||||
# The check url should be .../v1/videos/{id}
|
||||
check_url = f"{input.api_url}/{job_id}"
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with session.get(check_url) as response:
|
||||
if response.status == 200:
|
||||
status_data = await response.json()
|
||||
status = status_data.get("status")
|
||||
if status == "completed":
|
||||
output.success = True
|
||||
output.response_body = status_data
|
||||
if "peak_memory_mb" in status_data:
|
||||
output.peak_memory_mb = status_data["peak_memory_mb"]
|
||||
break
|
||||
elif status == "failed":
|
||||
output.success = False
|
||||
output.error = f"Job failed: {status_data.get('error')}"
|
||||
break
|
||||
else:
|
||||
# queued or processing
|
||||
await asyncio.sleep(1.0)
|
||||
else:
|
||||
output.success = False
|
||||
output.error = f"Poll failed HTTP {response.status}: {await response.text()}"
|
||||
break
|
||||
except Exception as e:
|
||||
output.success = False
|
||||
output.error = f"Poll exception: {str(e)}"
|
||||
break
|
||||
|
||||
output.latency = time.perf_counter() - output.start_time
|
||||
|
||||
# Check SLO if defined
|
||||
if input.slo_ms is not None and output.success:
|
||||
output.slo_achieved = (output.latency * 1000.0) <= input.slo_ms
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
backends_function_mapping = {
|
||||
"2i": {
|
||||
"vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
|
||||
"openai": (async_request_openai_images, "/v1/images/generations"),
|
||||
"sglang": (async_request_image_sglang, "/v1/images/generations"),
|
||||
},
|
||||
"2v": {
|
||||
"v1/videos": (async_request_v1_videos, "/v1/videos"),
|
||||
"sglang": (async_request_video_sglang, "/v1/videos"),
|
||||
},
|
||||
}
|
||||
@@ -745,6 +745,15 @@ def calculate_metrics(
|
||||
latencies = [o.latency for o in success_outputs]
|
||||
peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]
|
||||
|
||||
# Aggregate per-stage durations across all successful requests that reported them.
|
||||
stage_duration_lists: dict[str, list[float]] = {}
|
||||
for o in success_outputs:
|
||||
for stage, duration in (o.stage_durations or {}).items():
|
||||
stage_duration_lists.setdefault(stage, []).append(duration)
|
||||
stage_durations_mean = {s: float(np.mean(v)) for s, v in stage_duration_lists.items()}
|
||||
stage_durations_p50 = {s: float(np.percentile(v, 50)) for s, v in stage_duration_lists.items()}
|
||||
stage_durations_p99 = {s: float(np.percentile(v, 99)) for s, v in stage_duration_lists.items()}
|
||||
|
||||
metrics = {
|
||||
"duration": total_duration,
|
||||
"completed_requests": num_success,
|
||||
@@ -758,6 +767,9 @@ def calculate_metrics(
|
||||
"peak_memory_mb_max": max(peak_memories) if peak_memories else 0,
|
||||
"peak_memory_mb_mean": np.mean(peak_memories) if peak_memories else 0,
|
||||
"peak_memory_mb_median": np.median(peak_memories) if peak_memories else 0,
|
||||
"stage_durations_mean": stage_durations_mean,
|
||||
"stage_durations_p50": stage_durations_p50,
|
||||
"stage_durations_p99": stage_durations_p99,
|
||||
}
|
||||
|
||||
if slo_enabled:
|
||||
@@ -953,6 +965,12 @@ async def benchmark(args):
|
||||
print("{:<40} {:<15.2f}".format("Peak Memory Mean (MB):", metrics["peak_memory_mb_mean"]))
|
||||
print("{:<40} {:<15.2f}".format("Peak Memory Median (MB):", metrics["peak_memory_mb_median"]))
|
||||
|
||||
if metrics["stage_durations_mean"]:
|
||||
print(f"{'-' * 50}")
|
||||
print("Stage Durations Mean (s):")
|
||||
for stage, val in metrics["stage_durations_mean"].items():
|
||||
print("{:<40} {:<15.4f}".format(f" {stage}:", val))
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
if args.output_file:
|
||||
@@ -976,7 +994,7 @@ if __name__ == "__main__":
|
||||
"--backend",
|
||||
type=str,
|
||||
default="vllm-omni",
|
||||
choices=["vllm-omni", "openai", "v1/videos"],
|
||||
choices=["vllm-omni", "openai", "sglang", "v1/videos"],
|
||||
help="Backend to target the benchmark to.",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
Reference in new issue
Block a user