Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bbe81308b | ||
|
|
74258b18b7 | ||
|
|
5ab2932ce6 | ||
|
|
2b4a76502f | ||
|
|
414d14f37e | ||
|
|
8593781a07 | ||
|
|
eb0c4b1de4 | ||
|
|
f035107ba0 | ||
|
|
65784f8199 |
No files matched your search
@@ -843,8 +843,8 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
@@ -852,7 +852,7 @@ steps:
|
||||
- export CACHE_DIT_VERSION=1.3.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
@@ -1288,7 +1288,7 @@ steps:
|
||||
- nightly-tts-performance
|
||||
- nightly-diffusion-x2iat-performance-qwen-image
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-edit
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-edit-2509
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-layered
|
||||
- nightly-diffusion-x2iat-performance-bagel
|
||||
- nightly-diffusion-x2v-performance
|
||||
@@ -1305,7 +1305,7 @@ steps:
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
|
||||
|
||||
@@ -63,24 +63,6 @@
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i_2img",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.15,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 55000,
|
||||
"peak_memory_mb_mean": 55000
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i_2img",
|
||||
"dataset": "random",
|
||||
|
||||
@@ -61,23 +61,6 @@
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "512x512_steps20_i2i",
|
||||
"dataset": "random",
|
||||
"task": "i2i",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 10,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"throughput_qps": 0.21,
|
||||
"latency_mean": 10,
|
||||
"peak_memory_mb_max": 56000,
|
||||
"peak_memory_mb_mean": 56000
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "1536x1536_steps35_i2i",
|
||||
"dataset": "random",
|
||||
|
||||
@@ -71,7 +71,7 @@ class QualityTestConfig:
|
||||
prompt: str # generation prompt
|
||||
max_lpips: float # fail threshold — higher = more lenient
|
||||
model: str | None = None # HF model name
|
||||
quantization: str | None = None # quantization method, e.g. "fp8"
|
||||
quantization: str | dict[str, object] | None = None # quantization method/config, e.g. "fp8"
|
||||
baseline_model: str | None = None # explicit BF16/local baseline path
|
||||
quantized_model: str | None = None # explicit quantized/local model path
|
||||
height: int = 1024
|
||||
@@ -118,9 +118,57 @@ QUALITY_CONFIGS = [
|
||||
QualityTestConfig(
|
||||
id="fp8_z_image",
|
||||
model="Tongyi-MAI/Z-Image-Turbo",
|
||||
quantization="fp8",
|
||||
quantization={
|
||||
"method": "fp8",
|
||||
"ignored_layers": [
|
||||
"img_mlp",
|
||||
*[
|
||||
f"layers.{layer_id}.{suffix}"
|
||||
for layer_id in range(15, 30)
|
||||
for suffix in (
|
||||
"attention.to_qkv",
|
||||
"attention.to_out.0",
|
||||
"feed_forward.w13",
|
||||
"feed_forward.w2",
|
||||
)
|
||||
],
|
||||
*[
|
||||
f"model.layers.{layer_id}.{suffix}"
|
||||
for layer_id in range(28, 36)
|
||||
for suffix in (
|
||||
"self_attn.q_proj",
|
||||
"self_attn.k_proj",
|
||||
"self_attn.v_proj",
|
||||
"self_attn.o_proj",
|
||||
"mlp.gate_proj",
|
||||
"mlp.up_proj",
|
||||
"mlp.down_proj",
|
||||
)
|
||||
],
|
||||
],
|
||||
},
|
||||
task="t2i",
|
||||
prompt="a cup of coffee on a wooden table, morning light",
|
||||
prompt=(
|
||||
"A breathtaking twilight scene atop a floating archipelago of crystalline islands suspended in an "
|
||||
"endless nebula-drenched sky, where waterfalls of liquid starlight cascade from the edges of each "
|
||||
"island into the cosmic abyss below. The largest island hosts an ancient, overgrown observatory "
|
||||
"crafted from pearlescent white stone and living silverwood trees whose bioluminescent leaves pulse "
|
||||
"with soft cerulean and violet light. At the observatory's center stands a solitary figure—a young "
|
||||
"astronomer in flowing robes woven from woven moonlight and deep-space fabric, their face illuminated "
|
||||
"by the glow of a holographic star chart hovering above an altar of polished obsidian. The sky swirls "
|
||||
"with the birth of a new galaxy: ribbons of magenta and gold gas clouds twist around nascent stars, "
|
||||
"while distant supernovae bloom like cosmic flowers in the far reaches of the void. Below, the abyss "
|
||||
"reveals faint echoes of forgotten civilizations—ghostly silhouettes of submerged cities shimmering "
|
||||
"in layers of atmospheric haze. Cinematic volumetric lighting cuts through the scene as twin moons—one "
|
||||
"copper-hued, one opalescent—rise on opposite horizons, casting long, intersecting shadows across "
|
||||
"moss-covered ruins and crystalline flora that refract light into prismatic halos. Hyper-detailed, "
|
||||
"photorealistic rendering with the atmospheric depth of Roger Deakins' cinematography, the "
|
||||
"architectural grandeur of Zaha Hadid, and the cosmic wonder of James Jean's illustrations. Shot on "
|
||||
"a mythical 150mm lens with shallow depth of field, 8K resolution, Unreal Engine 5 realism, subsurface "
|
||||
"scattering on organic elements, and ray-traced reflections dancing across every water droplet in the "
|
||||
"starlight waterfalls. Ethereal, melancholic, and transcendent mood—like a moment of quiet revelation "
|
||||
"at the edge of existence."
|
||||
),
|
||||
max_lpips=0.15,
|
||||
num_inference_steps=20,
|
||||
),
|
||||
@@ -391,12 +439,6 @@ _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUA
|
||||
|
||||
def _quality_param(c: QualityTestConfig):
|
||||
marks = list(_marks)
|
||||
if c.id == "fp8_z_image":
|
||||
marks.append(
|
||||
pytest.mark.skip(
|
||||
reason="Z-Image FP8 quality gate temporarily disabled: https://github.com/vllm-project/vllm-omni/issues/3531"
|
||||
)
|
||||
)
|
||||
if c.id == "fp8_qwen_image":
|
||||
marks.append(
|
||||
pytest.mark.skip(reason="Qwen-Image FP8 quality gate temporarily disabled (see CI / issue tracker).")
|
||||
|
||||
@@ -253,37 +253,6 @@ def _diffusers_output_single_image(accuracy_artifact_root: Path, qwen_bear_image
|
||||
)
|
||||
|
||||
|
||||
def _vllm_omni_output_multiple_image(
|
||||
accuracy_artifact_root: Path,
|
||||
qwen_bear_image: Image.Image,
|
||||
rabbit_image: Image.Image,
|
||||
) -> Image.Image:
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL)
|
||||
output_path = output_dir / "vllm_omni_multiple.png"
|
||||
with OmniServer(model=MULTIPLE_MODEL, serve_args=SERVER_ARGS) as server:
|
||||
output = _run_vllm_omni_image_edit(
|
||||
omni_server=server,
|
||||
prompt=PROMPT_MULTIPLE_IMAGE,
|
||||
input_images=[qwen_bear_image, rabbit_image],
|
||||
output_path=output_path,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _diffusers_output_multiple_image(
|
||||
accuracy_artifact_root: Path, qwen_bear_image: Image.Image, rabbit_image: Image.Image
|
||||
) -> Image.Image:
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MULTIPLE_MODEL)
|
||||
output_path = output_dir / "diffusers_multiple.png"
|
||||
return _run_diffusers_image_edit(
|
||||
model=MULTIPLE_MODEL,
|
||||
pipeline_class=QwenImageEditPlusPipeline,
|
||||
prompt=PROMPT_MULTIPLE_IMAGE,
|
||||
input_images=[qwen_bear_image, rabbit_image],
|
||||
output_path=output_path,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_qwen_image_edit_single_matches_diffusers(
|
||||
@@ -309,37 +278,6 @@ def test_qwen_image_edit_single_matches_diffusers(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.skip(
|
||||
reason="Skipping as the second image seems to be ignored by the API. Will come back to this later after #2772 is merged."
|
||||
)
|
||||
def test_qwen_image_edit_multiple_matches_diffusers(
|
||||
accuracy_artifact_root: Path,
|
||||
qwen_bear_image: Image.Image,
|
||||
rabbit_image: Image.Image,
|
||||
) -> None:
|
||||
vllm_image = _vllm_omni_output_multiple_image(
|
||||
accuracy_artifact_root=accuracy_artifact_root,
|
||||
qwen_bear_image=qwen_bear_image,
|
||||
rabbit_image=rabbit_image,
|
||||
)
|
||||
diffusers_image = _diffusers_output_multiple_image(
|
||||
accuracy_artifact_root=accuracy_artifact_root,
|
||||
qwen_bear_image=qwen_bear_image,
|
||||
rabbit_image=rabbit_image,
|
||||
)
|
||||
assert_similarity(
|
||||
model_name=MULTIPLE_MODEL,
|
||||
vllm_image=vllm_image,
|
||||
diffusers_image=diffusers_image,
|
||||
width=WIDTH,
|
||||
height=HEIGHT,
|
||||
ssim_threshold=SSIM_THRESHOLD,
|
||||
psnr_threshold=PSNR_THRESHOLD,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
Online serving tests for the Qwen-Image-Edit family (image-to-image via chat completions).
|
||||
|
||||
- ``test_single_image_to_image_001``: ``Qwen/Qwen-Image-Edit`` — one reference image, fixed 512×512.
|
||||
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2509`` — two reference images, fixed 512×512.
|
||||
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2509`` only, ``advanced_model`` — mixed input
|
||||
- ``test_multi_images_to_image_001``: ``Qwen/Qwen-Image-Edit-2511`` — two reference images, fixed 512×512.
|
||||
- ``test_different_sizes_001``: ``Qwen/Qwen-Image-Edit-2511`` only, ``advanced_model`` — mixed input
|
||||
resolutions; ``extra_body`` uses per-output ``width``/``height`` lists; the test client sends one
|
||||
scalar-size request per list index in parallel and merges images (see ``OpenAIClientHandler.send_diffusion_request``).
|
||||
|
||||
@@ -53,7 +53,6 @@ def _get_diffusion_feature_cases(model: str):
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
@@ -83,15 +82,16 @@ def test_single_image_to_image_001(omni_server: OmniServer, openai_client: OpenA
|
||||
openai_client.send_diffusion_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2509``."""
|
||||
"""Two-reference edit smoke for ``Qwen/Qwen-Image-Edit-2511``."""
|
||||
|
||||
image_data_url_list = [f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}" for _ in range(2)]
|
||||
|
||||
@@ -118,7 +118,7 @@ def test_multi_images_to_image_001(omni_server: OmniServer, openai_client: OpenA
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize(
|
||||
"omni_server",
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2509"),
|
||||
_get_diffusion_feature_cases("Qwen/Qwen-Image-Edit-2511"),
|
||||
indirect=True,
|
||||
)
|
||||
def test_different_sizes_001(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
Comprehensive tests of diffusion features that are available in online serving mode
|
||||
and are supported by the following models:
|
||||
- Qwen-Image-Edit: single image input
|
||||
- Qwen-Image-Edit-2509: single image input and two image inputs
|
||||
- Qwen-Image-Edit-2511: single image input and two image inputs
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -14,7 +14,7 @@ from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHand
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
EDIT_PROMPT = "Transform this modern, geometrist image into a Vincent van Gogh style impressionist painting."
|
||||
SINGLE_EDIT_PROMPT_2509 = "Restyle this image into a Vincent van Gogh style impressionist painting."
|
||||
SINGLE_EDIT_PROMPT_2511 = "Restyle this image into a Vincent van Gogh style impressionist painting."
|
||||
MULTI_EDIT_PROMPT = (
|
||||
"Transform the first image into a Dadaism collage art. "
|
||||
"Transform the second image into a Vincent van Gogh style painting. "
|
||||
@@ -34,7 +34,7 @@ def _get_diffusion_feature_cases(model: str):
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"tea_cache", # [TODO] may consider changing to cache_dit after #1779 is resolved. Currently cache_dit and layerwise offload cannot work together.
|
||||
"cache_dit",
|
||||
"--enable-layerwise-offload",
|
||||
],
|
||||
),
|
||||
@@ -59,7 +59,7 @@ def _get_diffusion_feature_cases(model: str):
|
||||
model=model,
|
||||
server_args=[
|
||||
"--cache-backend",
|
||||
"cache_dit",
|
||||
"tea_cache",
|
||||
"--ring",
|
||||
"2",
|
||||
],
|
||||
@@ -145,7 +145,7 @@ def test_qwen_image_edit(omni_server: OmniServer, openai_client: OpenAIClientHan
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test Qwen-Image-Edit-2509 with a single image input.
|
||||
"""Test Qwen-Image-Edit-2511 with a single image input.
|
||||
|
||||
Regression: with tea_cache enabled and zero_cond_t=True, the TeaCache
|
||||
postprocess closure used the doubled temb (shape 2*batch) without halving
|
||||
@@ -156,7 +156,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
|
||||
"""
|
||||
image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2509)
|
||||
messages = dummy_messages_from_mix_data(image_data_url=image_data_url, content_text=SINGLE_EDIT_PROMPT_2511)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
@@ -180,7 +180,7 @@ def test_qwen_image_edit_2511_single_image(omni_server: OmniServer, openai_clien
|
||||
indirect=True,
|
||||
)
|
||||
def test_qwen_image_edit_2511_two_images(omni_server: OmniServer, openai_client: OpenAIClientHandler):
|
||||
"""Test Qwen-Image-Edit-2509 with two image inputs."""
|
||||
"""Test Qwen-Image-Edit-2511 with two image inputs."""
|
||||
image_data_url_1 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
image_data_url_2 = f"data:image/jpeg;base64,{generate_synthetic_image(512, 512)['base64']}"
|
||||
|
||||
|
||||
Reference in new issue
Block a user