From 74258b18b7b91df66d99be4161c9d487cff379f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Thu, 28 May 2026 18:33:31 +0800 Subject: [PATCH] [CI] Rectify L2~L4 Qwen Image Edit series tests (#3901) --- .buildkite/test-nightly.yml | 10 +-- .../test_qwen_image_edit_2511_vllm_omni.json | 18 ------ .../tests/test_qwen_image_edit_vllm_omni.json | 17 ----- tests/e2e/accuracy/test_qwen_image_edit.py | 62 ------------------- .../online_serving/test_qwen_image_edit.py | 12 ++-- .../test_qwen_image_edit_expansion.py | 14 ++--- 6 files changed, 18 insertions(+), 115 deletions(-) diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 099b1b978..8a43539d4 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -843,8 +843,8 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509" - key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509 + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" + key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 timeout_in_minutes: 180 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results @@ -852,7 +852,7 @@ steps: - export CACHE_DIT_VERSION=1.3.0 - | set +e - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" @@ -1288,7 +1288,7 @@ steps: - nightly-tts-performance - nightly-diffusion-x2iat-performance-qwen-image - nightly-diffusion-x2iat-performance-qwen-image-edit - - nightly-diffusion-x2iat-performance-qwen-image-edit-2509 + - nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - nightly-diffusion-x2iat-performance-qwen-image-layered - nightly-diffusion-x2iat-performance-bagel - nightly-diffusion-x2v-performance @@ -1305,7 +1305,7 @@ steps: - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit - - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509 + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511 - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 82c2ca27d..1903bc670 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -63,24 +63,6 @@ } }, "benchmark_params": [ - { - "name": "512x512_steps20_i2i_2img", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "num-input-images": 2, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.15, - "latency_mean": 10, - "peak_memory_mb_max": 55000, - "peak_memory_mb_mean": 55000 - } - }, { "name": "1536x1536_steps35_i2i_2img", "dataset": "random", diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index 8ae370df6..21695b752 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -61,23 +61,6 @@ } }, "benchmark_params": [ - { - "name": "512x512_steps20_i2i", - "dataset": "random", - "task": "i2i", - "width": 512, - "height": 512, - "num-inference-steps": 20, - "num-prompts": 10, - "max-concurrency": 1, - "enable-negative-prompt": true, - "baseline": { - "throughput_qps": 0.21, - "latency_mean": 10, - "peak_memory_mb_max": 56000, - "peak_memory_mb_mean": 56000 - } - }, { "name": "1536x1536_steps35_i2i", "dataset": "random", diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index 540444d42..338758cc0 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -253,37 +253,6 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image ) -def _vllm_omni_output_multiple_image( - accuracy_artifact_root: Path, - qwen_bear_image: Image.Image, - rabbit_image: Image.Image, -) -> Image.Image: - output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL) - output_path = output_dir / "vllm_omni_multiple.png" - with OmniServer(model=MULTIPLE_MODEL, serve_args=SERVER_ARGS) as server: - output = _run_vllm_omni_image_edit( - omni_server=server, - prompt=PROMPT_MULTIPLE_IMAGE, - input_images=[qwen_bear_image, rabbit_image], - output_path=output_path, - ) - return output - - -def _diffusers_output_multiple_image( - accuracy_artifact_root: Path, qwen_bear_image: Image.Image, rabbit_image: Image.Image -) -> Image.Image: - output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL) - output_path = output_dir / "diffusers_multiple.png" - return _run_diffusers_image_edit( - model=MULTIPLE_MODEL, - pipeline_class=QwenImageEditPlusPipeline, - prompt=PROMPT_MULTIPLE_IMAGE, - input_images=[qwen_bear_image, rabbit_image], - output_path=output_path, - ) - - @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_single_matches_diffusers( @@ -309,37 +278,6 @@ def test_qwen_image_edit_single_matches_diffusers( ) -@pytest.mark.benchmark -@hardware_test(res={"cuda": "H100"}, num_cards=1) -@pytest.mark.skip( - reason="Skipping as the second image seems to be ignored by the API. Will come back to this later after #2772 is merged." -) -def test_qwen_image_edit_multiple_matches_diffusers( - accuracy_artifact_root: Path, - qwen_bear_image: Image.Image, - rabbit_image: Image.Image, -) -> None: - vllm_image = _vllm_omni_output_multiple_image( - accuracy_artifact_root=accuracy_artifact_root, - qwen_bear_image=qwen_bear_image, - rabbit_image=rabbit_image, - ) - diffusers_image = _diffusers_output_multiple_image( - accuracy_artifact_root=accuracy_artifact_root, - qwen_bear_image=qwen_bear_image, - rabbit_image=rabbit_image, - ) - assert_similarity( - model_name=MULTIPLE_MODEL, - vllm_image=vllm_image, - diffusers_image=diffusers_image, - width=WIDTH, - height=HEIGHT, - ssim_threshold=SSIM_THRESHOLD, - psnr_threshold=PSNR_THRESHOLD, - ) - - @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None: diff --git a/tests/e2e/online_serving/test_qwen_image_edit.py b/tests/e2e/online_serving/test_qwen_image_edit.py index bdf8c9122..738a2e50c 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit.py +++ b/tests/e2e/online_serving/test_qwen_image_edit.py @@ -5,8 +5,8 @@ Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions). - ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512. -- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2509`` — two reference images, fixed 512×512. -- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2509`` only, ``advanced_model`` — mixed input +- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512. +- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``). @@ -53,7 +53,6 @@ def _get_diffusion_feature_cases(model: str): @pytest.mark.advanced_model -@pytest.mark.core_model @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", @@ -83,15 +82,16 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA openai_client.send_diffusion_request(request_config) +@pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"), + _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), indirect=True, ) def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2509``.""" + """Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511``.""" image_data_url_list = [f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" for _ in range(2)] @@ -118,7 +118,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA @pytest.mark.diffusion @pytest.mark.parametrize( "omni_server", - _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"), + _get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"), indirect=True, ) def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler): diff --git a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py index 632f75231..1457ad636 100644 --- a/tests/e2e/online_serving/test_qwen_image_edit_expansion.py +++ b/tests/e2e/online_serving/test_qwen_image_edit_expansion.py @@ -2,7 +2,7 @@ Comprehensive tests of diffusion features that are available in online serving mode and are supported by the following models: - Qwen-Image-Edit: single image input -- Qwen-Image-Edit-2509: single image input and two image inputs +- Qwen-Image-Edit-2511: single image input and two image inputs """ import pytest @@ -14,7 +14,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting." -SINGLE_EDIT_PROMPT_2509 = "Restyle this image into a Vincent van Gogh style impressionist painting." +SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting." MULTI_EDIT_PROMPT = ( "Transform the first image into a Dadaism collage art. " "Transform the second image into a Vincent van Gogh style painting. " @@ -34,7 +34,7 @@ def _get_diffusion_feature_cases(model: str): model=model, server_args=[ "--cache-backend", - "tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together. + "cache_dit", "--enable-layerwise-offload", ], ), @@ -59,7 +59,7 @@ def _get_diffusion_feature_cases(model: str): model=model, server_args=[ "--cache-backend", - "cache_dit", + "tea_cache", "--ring", "2", ], @@ -145,7 +145,7 @@ def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHan indirect=True, ) def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Test Qwen-Image-Edit-2509 with a single image input. + """Test Qwen-Image-Edit-2511 with a single image input. Regression: with tea_cache enabled and zero_cond_t=True, the TeaCache postprocess closure used the doubled temb (shape 2*batch) without halving @@ -156,7 +156,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien """ image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" - messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2509) + messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511) request_config = { "model": omni_server.model, @@ -180,7 +180,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien indirect=True, ) def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler): - """Test Qwen-Image-Edit-2509 with two image inputs.""" + """Test Qwen-Image-Edit-2511 with two image inputs.""" image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"