Compare commits

...
7 changed files with 69 additions and 124 deletions

No files matched your search

+5 -5
View File
@@ -843,8 +843,8 @@ steps:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
@@ -852,7 +852,7 @@ steps:
- export CACHE_DIT_VERSION=1.3.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -1288,7 +1288,7 @@ steps:
- nightly-tts-performance
- nightly-diffusion-x2iat-performance-qwen-image
- nightly-diffusion-x2iat-performance-qwen-image-edit
- nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- nightly-diffusion-x2iat-performance-qwen-image-edit-2511
- nightly-diffusion-x2iat-performance-qwen-image-layered
- nightly-diffusion-x2iat-performance-bagel
- nightly-diffusion-x2v-performance
@@ -1305,7 +1305,7 @@ steps:
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
@@ -63,24 +63,6 @@
}
},
"benchmark_params": [
{
"name": "512x512_steps20_i2i_2img",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.15,
"latency_mean": 10,
"peak_memory_mb_max": 55000,
"peak_memory_mb_mean": 55000
}
},
{
"name": "1536x1536_steps35_i2i_2img",
"dataset": "random",
@@ -61,23 +61,6 @@
}
},
"benchmark_params": [
{
"name": "512x512_steps20_i2i",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.21,
"latency_mean": 10,
"peak_memory_mb_max": 56000,
"peak_memory_mb_mean": 56000
}
},
{
"name": "1536x1536_steps35_i2i",
"dataset": "random",
@@ -71,7 +71,7 @@ class QualityTestConfig:
prompt: str # generation prompt
max_lpips: float # fail threshold — higher = more lenient
model: str | None = None # HF model name
quantization: str | None = None # quantization method, e.g. "fp8"
quantization: str | dict[str, object] | None = None # quantization method/config, e.g. "fp8"
baseline_model: str | None = None # explicit BF16/local baseline path
quantized_model: str | None = None # explicit quantized/local model path
height: int = 1024
@@ -118,9 +118,57 @@ QUALITY_CONFIGS = [
QualityTestConfig(
id="fp8_z_image",
model="Tongyi-MAI/Z-Image-Turbo",
quantization="fp8",
quantization={
"method": "fp8",
"ignored_layers": [
"img_mlp",
*[
f"layers.{layer_id}.{suffix}"
for layer_id in range(15, 30)
for suffix in (
"attention.to_qkv",
"attention.to_out.0",
"feed_forward.w13",
"feed_forward.w2",
)
],
*[
f"model.layers.{layer_id}.{suffix}"
for layer_id in range(28, 36)
for suffix in (
"self_attn.q_proj",
"self_attn.k_proj",
"self_attn.v_proj",
"self_attn.o_proj",
"mlp.gate_proj",
"mlp.up_proj",
"mlp.down_proj",
)
],
],
},
task="t2i",
prompt="a cup of coffee on a wooden table, morning light",
prompt=(
"A breathtaking twilight scene atop a floating archipelago of crystalline islands suspended in an "
"endless nebula-drenched sky, where waterfalls of liquid starlight cascade from the edges of each "
"island into the cosmic abyss below. The largest island hosts an ancient, overgrown observatory "
"crafted from pearlescent white stone and living silverwood trees whose bioluminescent leaves pulse "
"with soft cerulean and violet light. At the observatory's center stands a solitary figure—a young "
"astronomer in flowing robes woven from woven moonlight and deep-space fabric, their face illuminated "
"by the glow of a holographic star chart hovering above an altar of polished obsidian. The sky swirls "
"with the birth of a new galaxy: ribbons of magenta and gold gas clouds twist around nascent stars, "
"while distant supernovae bloom like cosmic flowers in the far reaches of the void. Below, the abyss "
"reveals faint echoes of forgotten civilizations—ghostly silhouettes of submerged cities shimmering "
"in layers of atmospheric haze. Cinematic volumetric lighting cuts through the scene as twin moons—one "
"copper-hued, one opalescent—rise on opposite horizons, casting long, intersecting shadows across "
"moss-covered ruins and crystalline flora that refract light into prismatic halos. Hyper-detailed, "
"photorealistic rendering with the atmospheric depth of Roger Deakins' cinematography, the "
"architectural grandeur of Zaha Hadid, and the cosmic wonder of James Jean's illustrations. Shot on "
"a mythical 150mm lens with shallow depth of field, 8K resolution, Unreal Engine 5 realism, subsurface "
"scattering on organic elements, and ray-traced reflections dancing across every water droplet in the "
"starlight waterfalls. Ethereal, melancholic, and transcendent mood—like a moment of quiet revelation "
"at the edge of existence."
),
max_lpips=0.15,
num_inference_steps=20,
),
@@ -391,12 +439,6 @@ _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUA
def _quality_param(c: QualityTestConfig):
marks = list(_marks)
if c.id == "fp8_z_image":
marks.append(
pytest.mark.skip(
reason="Z-Image FP8 quality gate temporarily disabled: https://github.com/vllm-project/vllm-omni/issues/3531"
)
)
if c.id == "fp8_qwen_image":
marks.append(
pytest.mark.skip(reason="Qwen-Image FP8 quality gate temporarily disabled (see CI / issue tracker).")
@@ -253,37 +253,6 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image
)
def _vllm_omni_output_multiple_image(
accuracy_artifact_root: Path,
qwen_bear_image: Image.Image,
rabbit_image: Image.Image,
) -> Image.Image:
output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL)
output_path = output_dir / "vllm_omni_multiple.png"
with OmniServer(model=MULTIPLE_MODEL, serve_args=SERVER_ARGS) as server:
output = _run_vllm_omni_image_edit(
omni_server=server,
prompt=PROMPT_MULTIPLE_IMAGE,
input_images=[qwen_bear_image, rabbit_image],
output_path=output_path,
)
return output
def _diffusers_output_multiple_image(
accuracy_artifact_root: Path, qwen_bear_image: Image.Image, rabbit_image: Image.Image
) -> Image.Image:
output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL)
output_path = output_dir / "diffusers_multiple.png"
return _run_diffusers_image_edit(
model=MULTIPLE_MODEL,
pipeline_class=QwenImageEditPlusPipeline,
prompt=PROMPT_MULTIPLE_IMAGE,
input_images=[qwen_bear_image, rabbit_image],
output_path=output_path,
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_qwen_image_edit_single_matches_diffusers(
@@ -309,37 +278,6 @@ def test_qwen_image_edit_single_matches_diffusers(
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.skip(
reason="Skipping as the second image seems to be ignored by the API. Will come back to this later after #2772 is merged."
)
def test_qwen_image_edit_multiple_matches_diffusers(
accuracy_artifact_root: Path,
qwen_bear_image: Image.Image,
rabbit_image: Image.Image,
) -> None:
vllm_image = _vllm_omni_output_multiple_image(
accuracy_artifact_root=accuracy_artifact_root,
qwen_bear_image=qwen_bear_image,
rabbit_image=rabbit_image,
)
diffusers_image = _diffusers_output_multiple_image(
accuracy_artifact_root=accuracy_artifact_root,
qwen_bear_image=qwen_bear_image,
rabbit_image=rabbit_image,
)
assert_similarity(
model_name=MULTIPLE_MODEL,
vllm_image=vllm_image,
diffusers_image=diffusers_image,
width=WIDTH,
height=HEIGHT,
ssim_threshold=SSIM_THRESHOLD,
psnr_threshold=PSNR_THRESHOLD,
)
@pytest.mark.benchmark
@hardware_test(res={"cuda": "H100"}, num_cards=1)
def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:
@@ -5,8 +5,8 @@
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2509`` — two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2509`` only, ``advanced_model`` — mixed input
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512.
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
@@ -53,7 +53,6 @@ def _get_diffusion_feature_cases(model: str):
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
@@ -83,15 +82,16 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA
openai_client.send_diffusion_request(request_config)
@pytest.mark.core_model
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
indirect=True,
)
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2509``."""
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511``."""
image_data_url_list = [f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" for _ in range(2)]
@@ -118,7 +118,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA
@pytest.mark.diffusion
@pytest.mark.parametrize(
"omni_server",
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
indirect=True,
)
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
@@ -2,7 +2,7 @@
Comprehensive tests of diffusion features that are available in online serving mode
and are supported by the following models:
- Qwen-Image-Edit: single image input
- Qwen-Image-Edit-2509: single image input and two image inputs
- Qwen-Image-Edit-2511: single image input and two image inputs
"""
import pytest
@@ -14,7 +14,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
SINGLE_EDIT_PROMPT_2509 = "Restyle this image into a Vincent van Gogh style impressionist painting."
SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting."
MULTI_EDIT_PROMPT = (
"Transform the first image into a Dadaism collage art. "
"Transform the second image into a Vincent van Gogh style painting. "
@@ -34,7 +34,7 @@ def _get_diffusion_feature_cases(model: str):
model=model,
server_args=[
"--cache-backend",
"tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together.
"cache_dit",
"--enable-layerwise-offload",
],
),
@@ -59,7 +59,7 @@ def _get_diffusion_feature_cases(model: str):
model=model,
server_args=[
"--cache-backend",
"cache_dit",
"tea_cache",
"--ring",
"2",
],
@@ -145,7 +145,7 @@ def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHan
indirect=True,
)
def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test Qwen-Image-Edit-2509 with a single image input.
"""Test Qwen-Image-Edit-2511 with a single image input.
Regression: with tea_cache enabled and zero_cond_t=True, the TeaCache
postprocess closure used the doubled temb (shape 2*batch) without halving
@@ -156,7 +156,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
"""
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2509)
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511)
request_config = {
"model": omni_server.model,
@@ -180,7 +180,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
indirect=True,
)
def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler):
"""Test Qwen-Image-Edit-2509 with two image inputs."""
"""Test Qwen-Image-Edit-2511 with two image inputs."""
image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"