From 9f3a73df17757eb9c198b4ce230f77ed092439f4 Mon Sep 17 00:00:00 2001 From: Inesh Reddy Chappidi Date: Mon, 27 Jul 2026 01:14:26 -0400 Subject: [PATCH] test: make Qwen-Image diffusers-backend latency check fair and stable (#5108) (#5132) Signed-off-by: Inesh Reddy --- .../test_diffusers_backend_similarity.py | 94 ++++++++++++------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/tests/e2e/accuracy/test_diffusers_backend_similarity.py b/tests/e2e/accuracy/test_diffusers_backend_similarity.py index 529152290..9e5485361 100644 --- a/tests/e2e/accuracy/test_diffusers_backend_similarity.py +++ b/tests/e2e/accuracy/test_diffusers_backend_similarity.py @@ -56,6 +56,10 @@ SEED = 42 SSIM_THRESHOLD = 0.97 PSNR_THRESHOLD = 30.0 +# A single-shot latency measurement on this benchmark is dominated by CI variance +# (see issue #5108). Warm up once, then report the median of several timed runs. +NUM_TIMED_RUNS = 2 + VIDEO_PROMPT = "The bear in the image dances happily" VIDEO_WIDTH = 832 VIDEO_HEIGHT = 480 @@ -181,25 +185,38 @@ def _run_vllm_omni_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I # "--enable-diffusion-pipeline-profiler", ] with OmniServer(model, server_args, use_omni=True) as omni_server: - start_time = time.perf_counter() - response = requests.post( - f"http://{omni_server.host}:{omni_server.port}/v1/images/generations", - json={ - "model": omni_server.model, - "prompt": PROMPT, - "size": f"{WIDTH}x{HEIGHT}", - "n": 1, - "response_format": "b64_json", - "negative_prompt": NEGATIVE_PROMPT, - "num_inference_steps": NUM_INFERENCE_STEPS, - "true_cfg_scale": TRUE_CFG_SCALE, - "seed": SEED, - }, - timeout=600, - ) - end_time = time.perf_counter() - e2e_latency = end_time - start_time - response.raise_for_status() + url = f"http://{omni_server.host}:{omni_server.port}/v1/images/generations" + request_json = { + "model": omni_server.model, + "prompt": PROMPT, + "size": f"{WIDTH}x{HEIGHT}", + "n": 1, + "response_format": "b64_json", + "negative_prompt": NEGATIVE_PROMPT, + "num_inference_steps": NUM_INFERENCE_STEPS, + "true_cfg_scale": TRUE_CFG_SCALE, + "seed": SEED, + } + + def _timed_request() -> tuple[requests.Response, float]: + start_time = time.perf_counter() + response = requests.post(url, json=request_json, timeout=600) + end_time = time.perf_counter() + response.raise_for_status() + return response, end_time - start_time + + # Warm up the client request path so first-request lazy init is not charged + # to the measurement, mirroring _diffusers_dummy_run on the diffusers side. + _timed_request() + + latencies: list[float] = [] + response: requests.Response | None = None + for _ in range(NUM_TIMED_RUNS): + response, latency = _timed_request() + latencies.append(latency) + e2e_latency = float(np.median(latencies)) + + assert response is not None payload = response.json() assert len(payload["data"]) == 1 image_bytes = base64.b64decode(payload["data"][0]["b64_json"]) @@ -221,20 +238,25 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> tuple[Image.I pipe.transformer.set_attention_backend("_flash_3_hub") _diffusers_dummy_run(pipe) - generator = torch.Generator(device="cuda").manual_seed(SEED) + latencies: list[float] = [] + result = None with torch.inference_mode(): - start_time = time.perf_counter() - result = pipe( # pyright: ignore[reportCallIssue] - prompt=PROMPT, - negative_prompt=NEGATIVE_PROMPT, - width=WIDTH, - height=HEIGHT, - num_inference_steps=NUM_INFERENCE_STEPS, - true_cfg_scale=TRUE_CFG_SCALE, - generator=generator, - ) - end_time = time.perf_counter() - e2e_latency = end_time - start_time + for _ in range(NUM_TIMED_RUNS): + generator = torch.Generator(device="cuda").manual_seed(SEED) + start_time = time.perf_counter() + result = pipe( # pyright: ignore[reportCallIssue] + prompt=PROMPT, + negative_prompt=NEGATIVE_PROMPT, + width=WIDTH, + height=HEIGHT, + num_inference_steps=NUM_INFERENCE_STEPS, + true_cfg_scale=TRUE_CFG_SCALE, + generator=generator, + ) + end_time = time.perf_counter() + latencies.append(end_time - start_time) + e2e_latency = float(np.median(latencies)) + assert result is not None output_image = result.images[0].convert("RGB") output_image.save(output_path) return output_image, e2e_latency @@ -276,8 +298,14 @@ def test_diffusers_backend_t2i_matches_diffusers(model_id: str, accuracy_artifac ssim_threshold=SSIM_THRESHOLD, psnr_threshold=PSNR_THRESHOLD, ) + # Latency is checked against a warmed median of NUM_TIMED_RUNS runs (see the + # _run_* helpers), so the assertion reflects steady-state rather than a single + # cold shot -- which is what previously made this check flaky (issue #5108). + # Correctness is additionally gated by the SSIM/PSNR assertions above. assert vllm_latency <= latency_threshold, ( - f"VLLM latency ({vllm_latency:.2f}ms) is greater than {latency_threshold_factor * 100}% more than Diffusers latency ({diffusers_latency:.2f}ms)." + f"VLLM latency ({vllm_latency:.2f}ms) is greater than " + f"{latency_threshold_factor * 100}% more than Diffusers latency " + f"({diffusers_latency:.2f}ms)." )