diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 779e09efc..1b61044af 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -660,87 +660,6 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate - - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1 - - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test" - timeout_in_minutes: 60 - commands: - - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1 - - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv" - - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json" - agents: - queue: "mithril-h100-pool" - plugins: - - kubernetes: - podSpec: - containers: - - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - resources: - limits: - nvidia.com/gpu: 1 - volumeMounts: - - name: devshm - mountPath: /dev/shm - - name: hf-cache - mountPath: /root/.cache/huggingface - env: - - name: HF_HOME - value: /root/.cache/huggingface - - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE - value: "120" - - name: HF_TOKEN - valueFrom: - secretKeyRef: - name: hf-token-secret - key: token - nodeSelector: - node.kubernetes.io/instance-type: gpu-h100-sxm - volumes: - - name: devshm - emptyDir: - medium: Memory - - name: hf-cache - hostPath: - path: /mnt/hf-cache - type: DirectoryOrCreate - - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" timeout_in_minutes: 180 commands: diff --git a/tests/e2e/accuracy/assets/qwen_image_edit_2511_test1.png b/tests/e2e/accuracy/assets/qwen_image_edit_2511_test1.png new file mode 100644 index 000000000..5cbf63ec0 Binary files /dev/null and b/tests/e2e/accuracy/assets/qwen_image_edit_2511_test1.png differ diff --git a/tests/e2e/accuracy/assets/qwen_image_edit_2511_test2.png b/tests/e2e/accuracy/assets/qwen_image_edit_2511_test2.png new file mode 100644 index 000000000..b507b18f1 Binary files /dev/null and b/tests/e2e/accuracy/assets/qwen_image_edit_2511_test2.png differ diff --git a/tests/e2e/accuracy/helpers.py b/tests/e2e/accuracy/helpers.py index a5044db76..38b045dce 100644 --- a/tests/e2e/accuracy/helpers.py +++ b/tests/e2e/accuracy/helpers.py @@ -127,6 +127,74 @@ def assert_image_sequence_similarity( ) +def assert_images_pixel_close( + *, + model_name: str, + vllm_image: Image.Image, + diffusers_image: Image.Image, + mean_threshold: float, + p99_threshold: float, + compare_mode: str = "RGB", +) -> None: + """Assert full-image pixel closeness between online serving and diffusers output. + + Match the threshold style used by diffusion parallelism tests: convert both + images to float32 RGB tensors in the [0, 1] range, then compare mean and p99 + absolute channel differences. + + Accuracy improves as both values move lower: + - ``mean_abs_diff`` tracks global average drift across the whole image. + - ``p99_abs_diff`` tracks tail drift while ignoring the noisiest 1% of + channel samples. + + The printed mismatch ratios are diagnostics only. ``pixel_ratio`` counts + pixels where any channel exceeds a tolerance, while ``channel_ratio`` counts + individual channel samples above that tolerance. + """ + assert vllm_image.size == diffusers_image.size, ( + f"Online and diffusers output sizes mismatch: online={vllm_image.size}, diffusers={diffusers_image.size}" + ) + + vllm_array = np.asarray(vllm_image.convert(compare_mode), dtype=np.float32) / 255.0 + diffusers_array = np.asarray(diffusers_image.convert(compare_mode), dtype=np.float32) / 255.0 + channel_abs_diff = np.abs(vllm_array - diffusers_array) + mean_abs_diff = float(channel_abs_diff.mean()) + p99_abs_diff = float(np.quantile(channel_abs_diff, 0.99)) + percentiles = { + percentile: float(np.quantile(channel_abs_diff, percentile / 100.0)) for percentile in (50, 90, 95, 99, 99.9) + } + + print(f"{model_name} pixel metrics:") + print( + f" mean_abs_diff: value={mean_abs_diff:.6e}, threshold<={mean_threshold:.6e}, " + "range=[0, 1], lower_is_better=True" + ) + print( + f" p99_abs_diff: value={p99_abs_diff:.6e}, threshold<={p99_threshold:.6e}, range=[0, 1], lower_is_better=True" + ) + print(" abs_diff_percentiles:") + for percentile, value in percentiles.items(): + print(f" p{percentile}: value={value:.6e}, range=[0, 1], lower_is_better=True") + print(" mismatch_ratios_by_channel_threshold:") + for tolerance in (0, 1, 2, 4, 8, 16, 32, 64, 128): + normalized_tolerance = tolerance / 255.0 + pixel_mismatch = np.any(channel_abs_diff > normalized_tolerance, axis=-1) + pixel_mismatch_ratio = float(np.count_nonzero(pixel_mismatch) / pixel_mismatch.size) + channel_mismatch_ratio = float( + np.count_nonzero(channel_abs_diff > normalized_tolerance) / channel_abs_diff.size + ) + print( + f" threshold>{normalized_tolerance:.6e} ({tolerance}/255): pixel_ratio={pixel_mismatch_ratio:.8f}, " + f"channel_ratio={channel_mismatch_ratio:.8f}" + ) + + assert mean_abs_diff <= mean_threshold and p99_abs_diff <= p99_threshold, ( + f"Image diff exceeded threshold for {model_name}: mean_abs_diff={mean_abs_diff:.6e}, " + f"p99_abs_diff={p99_abs_diff:.6e} (thresholds: mean<={mean_threshold:.6e}, " + f"p99<={p99_threshold:.6e})" + ) + + def compute_image_ssim_psnr( *, prediction: Image.Image, diff --git a/tests/e2e/accuracy/test_qwen_image.py b/tests/e2e/accuracy/test_qwen_image.py index 97f721139..c9a09e6de 100644 --- a/tests/e2e/accuracy/test_qwen_image.py +++ b/tests/e2e/accuracy/test_qwen_image.py @@ -12,7 +12,7 @@ import torch from diffusers.pipelines.pipeline_utils import DiffusionPipeline from PIL import Image -from tests.e2e.accuracy.helpers import assert_similarity, model_output_dir +from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer @@ -32,11 +32,38 @@ SEED = 42 SSIM_THRESHOLD = 0.97 PSNR_THRESHOLD = 30.0 +MODEL_2512_ID = "Qwen/Qwen-Image-2512" +MODEL_2512_ENV_VAR = "QWEN_IMAGE_2512_MODEL" +PROMPT_2512 = ( + "A 20-year-old East Asian girl with delicate, charming features and large, bright brown eyes—expressive and " + "lively, with a cheerful or subtly smiling expression. Her naturally wavy long hair is either loose or tied in " + "twin ponytails. She has fair skin and light makeup accentuating her youthful freshness. She wears a modern, " + "cute dress or relaxed outfit in bright, soft colors—lightweight fabric, minimalist cut. She stands indoors at " + "an anime convention, surrounded by banners, posters, or stalls. Lighting is typical indoor illumination—no " + "staged lighting—and the image resembles a casual iPhone snapshot: unpretentious composition, yet brimming " + "with vivid, fresh, youthful charm." +) +NEGATIVE_PROMPT_2512 = ( + "低分辨率，低画质，肢体畸形，手指畸形，画面过饱和，蜡像感，人脸无细节，过度光滑，画面具有AI感。" + "构图混乱。文字模糊，扭曲。" +) +WIDTH_2512 = 1664 +HEIGHT_2512 = 928 +NUM_INFERENCE_STEPS_2512 = 50 +TRUE_CFG_SCALE_2512 = 4.0 +SEED_2512 = 42 +MEAN_ABS_DIFF_THRESHOLD_2512 = 3e-2 +P99_ABS_DIFF_THRESHOLD_2512 = 4e-1 + def _model_name() -> str: return os.environ.get(MODEL_ENV_VAR, MODEL_ID) +def _model_2512_name() -> str: + return os.environ.get(MODEL_2512_ENV_VAR, MODEL_2512_ID) + + def _local_files_only(model: str) -> bool: return Path(model).exists() @@ -102,6 +129,67 @@ def _run_diffusers_qwen_image(*, model: str, output_path: Path) -> Image.Image: run_post_test_cleanup() +def _run_vllm_omni_qwen_image_2512(*, model: str, output_path: Path) -> Image.Image: + server_args = ["--num-gpus", "1", "--stage-init-timeout", "300", "--init-timeout", "900"] + with OmniServer(model, server_args, use_omni=True) as omni_server: + response = requests.post( + f"http://{omni_server.host}:{omni_server.port}/v1/images/generations", + json={ + "model": omni_server.model, + "prompt": PROMPT_2512, + "size": f"{WIDTH_2512}x{HEIGHT_2512}", + "n": 1, + "response_format": "b64_json", + "negative_prompt": NEGATIVE_PROMPT_2512, + "num_inference_steps": NUM_INFERENCE_STEPS_2512, + "true_cfg_scale": TRUE_CFG_SCALE_2512, + "seed": SEED_2512, + }, + timeout=1200, + ) + response.raise_for_status() + payload = response.json() + assert len(payload["data"]) == 1 + image_bytes = base64.b64decode(payload["data"][0]["b64_json"]) + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + image.load() + image.save(output_path) + return image + + +def _run_diffusers_qwen_image_2512(*, model: str, output_path: Path) -> Image.Image: + run_pre_test_cleanup() + pipe: DiffusionPipeline | None = None + try: + pipe = DiffusionPipeline.from_pretrained( + model, + torch_dtype=torch.bfloat16, + trust_remote_code=True, + local_files_only=_local_files_only(model), + ).to("cuda") + generator = torch.Generator(device="cuda").manual_seed(SEED_2512) + result = pipe( # pyright: ignore[reportCallIssue] + prompt=PROMPT_2512, + negative_prompt=NEGATIVE_PROMPT_2512, + width=WIDTH_2512, + height=HEIGHT_2512, + num_inference_steps=NUM_INFERENCE_STEPS_2512, + true_cfg_scale=TRUE_CFG_SCALE_2512, + generator=generator, + ) + output_image = result.images[0].convert("RGB") + output_image.save(output_path) + return output_image + finally: + if pipe is not None and hasattr(pipe, "maybe_free_model_hooks"): + pipe.maybe_free_model_hooks() + del pipe + gc.collect() + if torch.cuda.is_available(): + torch.accelerator.empty_cache() + run_post_test_cleanup() + + @pytest.mark.benchmark @hardware_test(res={"cuda": "H100"}, num_cards=1) def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None: @@ -120,3 +208,27 @@ def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None: ssim_threshold=SSIM_THRESHOLD, psnr_threshold=PSNR_THRESHOLD, ) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_qwen_image_2512_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None: + model = _model_2512_name() + output_dir = model_output_dir(accuracy_artifact_root, MODEL_2512_ID) + vllm_output_path = output_dir / "vllm_omni_2512.png" + diffusers_output_path = output_dir / "diffusers_2512.png" + + vllm_output = _run_vllm_omni_qwen_image_2512(model=model, output_path=vllm_output_path) + diffusers_output = _run_diffusers_qwen_image_2512(model=model, output_path=diffusers_output_path) + + print(f"{MODEL_2512_ID} generated images:") + print(f" vllm_omni: {vllm_output_path}") + print(f" diffusers: {diffusers_output_path}") + + assert_images_pixel_close( + model_name=MODEL_2512_ID, + vllm_image=vllm_output, + diffusers_image=diffusers_output, + mean_threshold=MEAN_ABS_DIFF_THRESHOLD_2512, + p99_threshold=P99_ABS_DIFF_THRESHOLD_2512, + ) diff --git a/tests/e2e/accuracy/test_qwen_image_edit.py b/tests/e2e/accuracy/test_qwen_image_edit.py index a910ed5ba..ea4a9ff1b 100644 --- a/tests/e2e/accuracy/test_qwen_image_edit.py +++ b/tests/e2e/accuracy/test_qwen_image_edit.py @@ -1,6 +1,7 @@ from __future__ import annotations import gc +import os from pathlib import Path import pytest @@ -10,7 +11,7 @@ from diffusers import QwenImageEditPipeline, QwenImageEditPlusPipeline from PIL import Image from benchmarks.accuracy.common import decode_base64_image, pil_to_png_bytes -from tests.e2e.accuracy.helpers import assert_similarity, model_output_dir +from tests.e2e.accuracy.helpers import assert_images_pixel_close, assert_similarity, model_output_dir from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test from tests.helpers.runtime import OmniServer @@ -28,12 +29,43 @@ SEED = 42 SSIM_THRESHOLD = 0.94 PSNR_THRESHOLD = 28.0 +EDIT_2511_MODEL = "Qwen/Qwen-Image-Edit-2511" +EDIT_2511_MODEL_ENV_VAR = "QWEN_IMAGE_EDIT_2511_MODEL" +EDIT_2511_WIDTH = 1190 +EDIT_2511_HEIGHT = 2032 +EDIT_2511_NUM_INFERENCE_STEPS = 40 +EDIT_2511_TRUE_CFG_SCALE = 4.0 +EDIT_2511_GUIDANCE_SCALE = 1.0 +EDIT_2511_SEED = 42 +EDIT_2511_MEAN_ABS_DIFF_THRESHOLD = 3e-2 +EDIT_2511_P99_ABS_DIFF_THRESHOLD = 4e-1 +EDIT_2511_NEGATIVE_PROMPT = " " +EDIT_2511_PROMPT = ( + "将第二张图中人脸/猫狗脸转换为3D卡通形象，质量要求：毛发纹理自然电影级色彩校准，注意保留原图的外貌、" + "肤色、发型特征，仅替换第一张财神爷/猫狗财神的用绿框框住的头部区域，耳朵样式和原图的3D卡通形象一致， " + "（帽子紧贴头顶，未贴合部分可调整帽子摆放角度达到完全贴合）， 头部与身体过渡必须自然，无拼接痕迹，" + "无额外元素出现在头部四周，纯白背景，不要出现绿色圆圈" +) +EDIT_2511_ASSET_DIR = Path(__file__).resolve().parent / "assets" +EDIT_2511_IMAGE_PATHS = [ + EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test1.png", + EDIT_2511_ASSET_DIR / "qwen_image_edit_2511_test2.png", +] + PROMPT_SINGLE_IMAGE = "The input is a 2D cartoon bear mascot. Restyle it into a painterly oil artwork with warm colors while preserving the main structure." PROMPT_MULTIPLE_IMAGE = "Put the cartoon bear mascot and the furry rabbit into one coherent scene with a painterly oil artwork style and consistent lighting." NEGATIVE_PROMPT = "low quality, blurry, artifacts, distortion" SERVER_ARGS = ["--num-gpus", "1", "--stage-init-timeout", "300", "--init-timeout", "900"] +def _edit_2511_model_name() -> str: + return os.environ.get(EDIT_2511_MODEL_ENV_VAR, EDIT_2511_MODEL) + + +def _local_files_only(model: str) -> bool: + return Path(model).exists() + + def _run_vllm_omni_image_edit( *, omni_server: OmniServer, @@ -79,17 +111,15 @@ def _run_diffusers_image_edit( ) -> Image.Image: run_pre_test_cleanup() pipe: QwenImageEditPipeline | QwenImageEditPlusPipeline | None = None - device = torch.device("cuda:0") - torch.cuda.set_device(device) try: images = input_images[0] if len(input_images) == 1 else input_images pipe = pipeline_class.from_pretrained( model, torch_dtype=torch.bfloat16, trust_remote_code=True, - ).to(device) + ).to("cuda") pipe.set_progress_bar_config(disable=False) - generator = torch.Generator(device=device).manual_seed(SEED) + generator = torch.Generator(device="cuda").manual_seed(SEED) result = pipe( # pyright: ignore[reportCallIssue] prompt=prompt, image=images, @@ -113,6 +143,88 @@ def _run_diffusers_image_edit( run_post_test_cleanup() +def _load_edit_2511_input_images() -> list[Image.Image]: + return [Image.open(path).convert("RGB") for path in EDIT_2511_IMAGE_PATHS] + + +def _run_vllm_omni_image_edit_2511( + *, + model: str, + input_images: list[Image.Image], + output_path: Path, +) -> Image.Image: + with OmniServer(model=model, serve_args=SERVER_ARGS) as server: + response = requests.post( + f"http://{server.host}:{server.port}/v1/images/edits", + data={ + "model": server.model, + "prompt": EDIT_2511_PROMPT, + "size": f"{EDIT_2511_WIDTH}x{EDIT_2511_HEIGHT}", + "n": 1, + "response_format": "b64_json", + "negative_prompt": EDIT_2511_NEGATIVE_PROMPT, + "num_inference_steps": EDIT_2511_NUM_INFERENCE_STEPS, + "true_cfg_scale": EDIT_2511_TRUE_CFG_SCALE, + "guidance_scale": EDIT_2511_GUIDANCE_SCALE, + "seed": EDIT_2511_SEED, + }, + files=[ + ("image", (f"image_{index}.png", pil_to_png_bytes(image), "image/png")) + for index, image in enumerate(input_images) + ], + timeout=1200, + ) + response.raise_for_status() + payload = response.json() + assert len(payload["data"]) == 1 + image = decode_base64_image(payload["data"][0]["b64_json"]) + image.load() + image.save(output_path) + return image + + +def _run_diffusers_image_edit_2511( + *, + model: str, + input_images: list[Image.Image], + output_path: Path, +) -> Image.Image: + run_pre_test_cleanup() + pipe: QwenImageEditPlusPipeline | None = None + try: + pipe = QwenImageEditPlusPipeline.from_pretrained( + model, + torch_dtype=torch.bfloat16, + trust_remote_code=True, + local_files_only=_local_files_only(model), + ).to("cuda") + pipe.set_progress_bar_config(disable=False) + generator = torch.Generator(device="cuda").manual_seed(EDIT_2511_SEED) + result = pipe( # pyright: ignore[reportCallIssue] + image=input_images, + prompt=EDIT_2511_PROMPT, + generator=generator, + true_cfg_scale=EDIT_2511_TRUE_CFG_SCALE, + negative_prompt=EDIT_2511_NEGATIVE_PROMPT, + num_inference_steps=EDIT_2511_NUM_INFERENCE_STEPS, + guidance_scale=EDIT_2511_GUIDANCE_SCALE, + num_images_per_prompt=1, + width=EDIT_2511_WIDTH, + height=EDIT_2511_HEIGHT, + ) + output_image = result.images[0].convert("RGB") # pyright: ignore[reportAttributeAccessIssue] + output_image.save(output_path) + return output_image + finally: + if pipe is not None and hasattr(pipe, "maybe_free_model_hooks"): + pipe.maybe_free_model_hooks() + del pipe + gc.collect() + if torch.cuda.is_available(): + torch.accelerator.empty_cache() + run_post_test_cleanup() + + def _vllm_omni_output_single_image( accuracy_artifact_root: Path, qwen_bear_image: Image.Image, @@ -226,3 +338,43 @@ def test_qwen_image_edit_multiple_matches_diffusers( ssim_threshold=SSIM_THRESHOLD, psnr_threshold=PSNR_THRESHOLD, ) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_qwen_image_edit_2511_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None: + model = _edit_2511_model_name() + output_dir = model_output_dir(accuracy_artifact_root, EDIT_2511_MODEL) + input_images = _load_edit_2511_input_images() + input_paths: list[Path] = [] + for index, image in enumerate(input_images, start=1): + input_path = output_dir / f"edit_2511_input_{index}.png" + image.save(input_path) + input_paths.append(input_path) + vllm_output_path = output_dir / "vllm_omni_edit_2511.png" + diffusers_output_path = output_dir / "diffusers_edit_2511.png" + + vllm_image = _run_vllm_omni_image_edit_2511( + model=model, + input_images=input_images, + output_path=vllm_output_path, + ) + diffusers_image = _run_diffusers_image_edit_2511( + model=model, + input_images=input_images, + output_path=diffusers_output_path, + ) + + print(f"{EDIT_2511_MODEL} generated images:") + for index, input_path in enumerate(input_paths, start=1): + print(f" input_{index}: {input_path}") + print(f" vllm_omni: {vllm_output_path}") + print(f" diffusers: {diffusers_output_path}") + + assert_images_pixel_close( + model_name=EDIT_2511_MODEL, + vllm_image=vllm_image, + diffusers_image=diffusers_image, + mean_threshold=EDIT_2511_MEAN_ABS_DIFF_THRESHOLD, + p99_threshold=EDIT_2511_P99_ABS_DIFF_THRESHOLD, + )