anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI] Rectify L2~L4 Qwen Image Edit series tests (#3901)

Browse Source
This commit is contained in:
Zeyu Huang | 黃澤宇 authored and GitHub committed 2026-05-28 18:33:31 +08:00
1 parent 1c876f0cd1
commit 74258b18b7
6 files changed
+18 -115
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/test-nightly.yml
+5 -5
View File
@@ -843,8 +843,8 @@ steps:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
@@ -852,7 +852,7 @@ steps:
- export CACHE_DIT_VERSION=1.3.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -1288,7 +1288,7 @@ steps:
- nightly-tts-performance
- nightly-diffusion-x2iat-performance-qwen-image
- nightly-diffusion-x2iat-performance-qwen-image-edit
- nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- nightly-diffusion-x2iat-performance-qwen-image-edit-2511
- nightly-diffusion-x2iat-performance-qwen-image-layered
- nightly-diffusion-x2iat-performance-bagel
- nightly-diffusion-x2v-performance
@@ -1305,7 +1305,7 @@ steps:
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
-18
View File
@@ -63,24 +63,6 @@
}
},
"benchmark_params": [
{
"name": "512x512_steps20_i2i_2img",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.15,
"latency_mean": 10,
"peak_memory_mb_max": 55000,
"peak_memory_mb_mean": 55000
}
},
{
"name": "1536x1536_steps35_i2i_2img",
"dataset": "random",
tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
-17
View File
@@ -61,23 +61,6 @@
}
},
"benchmark_params": [
{
"name": "512x512_steps20_i2i",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.21,
"latency_mean": 10,
"peak_memory_mb_max": 56000,
"peak_memory_mb_mean": 56000
}
},
{
"name": "1536x1536_steps35_i2i",
"dataset": "random",
tests/e2e/accuracy/test_qwen_image_edit.py
-62
View File
@@ -253,37 +253,6 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image
)
def _vllm_omni_output_multiple_image(
accuracy_artifact_root: Path,
qwen_bear_image: Image.Image,
rabbit_image: Image.Image,
) -> Image.Image:
output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL)
output_path = output_dir / "vllm_omni_multiple.png"
with OmniServer(model=MULTIPLE_MODEL, serve_args=SERVER_ARGS) as server:
output = _run_vllm_omni_image_edit(
omni_server=server,
prompt=PROMPT_MULTIPLE_IMAGE,
input_images=[qwen_bear_image, rabbit_image],
output_path=output_path,
)
return output
def _diffusers_output_multiple_image(
accuracy_artifact_root: Path, qwen_bear_image: Image.Image, rabbit_image: Image.Image
) -> Image.Image:
output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL)
output_path = output_dir / "diffusers_multiple.png"
return _run_diffusers_image_edit(
model=MULTIPLE_MODEL,
pipeline_class=QwenImageEditPlusPipeline,
prompt=PROMPT_MULTIPLE_IMAGE,
input_images=[qwen_bear_image, rabbit_image],
output_path=output_path,
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_qwen_image_edit_single_matches_diffusers(
@@ -309,37 +278,6 @@ def test_qwen_image_edit_single_matches_diffusers(
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.skip(
reason="Skipping as the second image seems to be ignored by the API. Will come back to this later after #2772 is merged."
)
def test_qwen_image_edit_multiple_matches_diffusers(
accuracy_artifact_root: Path,
qwen_bear_image: Image.Image,
rabbit_image: Image.Image,
) -> None:
vllm_image = _vllm_omni_output_multiple_image(
accuracy_artifact_root=accuracy_artifact_root,
qwen_bear_image=qwen_bear_image,
rabbit_image=rabbit_image,
)
diffusers_image = _diffusers_output_multiple_image(
accuracy_artifact_root=accuracy_artifact_root,
qwen_bear_image=qwen_bear_image,
rabbit_image=rabbit_image,
)
assert_similarity(
model_name=MULTIPLE_MODEL,
vllm_image=vllm_image,
diffusers_image=diffusers_image,
width=WIDTH,
height=HEIGHT,
ssim_threshold=SSIM_THRESHOLD,
psnr_threshold=PSNR_THRESHOLD,
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:
tests/e2e/online_serving/test_qwen_image_edit.py
+6 -6
View File
@@ -5,8 +5,8 @@
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2509`` — two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2509`` only, ``advanced_model`` — mixed input
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
@@ -53,7 +53,6 @@ def _get_diffusion_feature_cases(model: str):
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
@@ -83,15 +82,16 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA
openai_client.send_diffusion_request(request_config)
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
indirect=True,
)
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2509``."""
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511``."""
image_data_url_list = [f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" for _ in range(2)]
@@ -118,7 +118,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
indirect=True,
)
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+7 -7
View File
@@ -2,7 +2,7 @@
Comprehensive tests of diffusion features that are available in online serving mode
and are supported by the following models:
- Qwen-Image-Edit: single image input
- Qwen-Image-Edit-2509: single image input and two image inputs
- Qwen-Image-Edit-2511: single image input and two image inputs
"""
import pytest
@@ -14,7 +14,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
SINGLE_EDIT_PROMPT_2509 = "Restyle this image into a Vincent van Gogh style impressionist painting."
SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
"Transform the second image into a Vincent van Gogh style painting. "
@@ -34,7 +34,7 @@ def _get_diffusion_feature_cases(model: str):
model=model,
server_args=[
"--cache-backend",
"tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together.
"cache_dit",
"--enable-layerwise-offload",
],
),
@@ -59,7 +59,7 @@ def _get_diffusion_feature_cases(model: str):
model=model,
server_args=[
"--cache-backend",
"cache_dit",
"tea_cache",
"--ring",
"2",
],
@@ -145,7 +145,7 @@ def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHan
indirect=True,
)
def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test Qwen-Image-Edit-2509 with a single image input.
"""Test Qwen-Image-Edit-2511 with a single image input.
Regression: with tea_cache enabled and zero_cond_t=True, the TeaCache
postprocess closure used the doubled temb (shape 2*batch) without halving
@@ -156,7 +156,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
"""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2509)
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511)
request_config = {
"model": omni_server.model,
@@ -180,7 +180,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
indirect=True,
)
def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test Qwen-Image-Edit-2509 with two image inputs."""
"""Test Qwen-Image-Edit-2511 with two image inputs."""
image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"