diff --git a/tests/diffusion/quantization/artifacts/fp8_z_image_quality/README.md b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/README.md new file mode 100644 index 000000000..30727e9e7 --- /dev/null +++ b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/README.md @@ -0,0 +1,29 @@ +# FP8 Z-Image Quality Artifact + +Generated by: + +```bash +CUDA_VISIBLE_DEVICES=0 \ +VLLM_OMNI_QUALITY_OUTPUT_DIR=$PWD/tests/diffusion/quantization/artifacts/fp8_z_image_quality \ +.venv/bin/python -m pytest \ + tests/diffusion/quantization/test_quantization_quality.py::test_quantization_quality[fp8_z_image] \ + -s -v -m "" +``` + +Result: failed with the current `max_lpips=0.15` threshold. + +| Metric | Value | +|--------|-------| +| LPIPS | 0.250578 | +| PSNR | 17.042153 dB | +| MAE | 0.076899 | + +## Comparison + +![BF16 baseline vs FP8 quantized](fp8_z_image_comparison.png) + +## Individual Outputs + +![BF16 baseline](fp8_z_image_baseline.png) + +![FP8 quantized](fp8_z_image_quantized.png) diff --git a/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_baseline.png b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_baseline.png new file mode 100644 index 000000000..a53353866 Binary files /dev/null and b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_baseline.png differ diff --git a/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_comparison.png b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_comparison.png new file mode 100644 index 000000000..c268551ae Binary files /dev/null and b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_comparison.png differ diff --git a/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_quantized.png b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_quantized.png new file mode 100644 index 000000000..4029bc9b0 Binary files /dev/null and b/tests/diffusion/quantization/artifacts/fp8_z_image_quality/fp8_z_image_quantized.png differ diff --git a/tests/diffusion/quantization/test_quantization_quality.py b/tests/diffusion/quantization/test_quantization_quality.py index 692d83c12..024a28e3f 100644 --- a/tests/diffusion/quantization/test_quantization_quality.py +++ b/tests/diffusion/quantization/test_quantization_quality.py @@ -120,7 +120,27 @@ QUALITY_CONFIGS = [ model="Tongyi-MAI/Z-Image-Turbo", quantization="fp8", task="t2i", - prompt="a cup of coffee on a wooden table, morning light", + prompt=( + "A breathtaking twilight scene atop a floating archipelago of crystalline islands suspended in an " + "endless nebula-drenched sky, where waterfalls of liquid starlight cascade from the edges of each " + "island into the cosmic abyss below. The largest island hosts an ancient, overgrown observatory " + "crafted from pearlescent white stone and living silverwood trees whose bioluminescent leaves pulse " + "with soft cerulean and violet light. At the observatory's center stands a solitary figure—a young " + "astronomer in flowing robes woven from woven moonlight and deep-space fabric, their face illuminated " + "by the glow of a holographic star chart hovering above an altar of polished obsidian. The sky swirls " + "with the birth of a new galaxy: ribbons of magenta and gold gas clouds twist around nascent stars, " + "while distant supernovae bloom like cosmic flowers in the far reaches of the void. Below, the abyss " + "reveals faint echoes of forgotten civilizations—ghostly silhouettes of submerged cities shimmering " + "in layers of atmospheric haze. Cinematic volumetric lighting cuts through the scene as twin moons—one " + "copper-hued, one opalescent—rise on opposite horizons, casting long, intersecting shadows across " + "moss-covered ruins and crystalline flora that refract light into prismatic halos. Hyper-detailed, " + "photorealistic rendering with the atmospheric depth of Roger Deakins' cinematography, the " + "architectural grandeur of Zaha Hadid, and the cosmic wonder of James Jean's illustrations. Shot on " + "a mythical 150mm lens with shallow depth of field, 8K resolution, Unreal Engine 5 realism, subsurface " + "scattering on organic elements, and ray-traced reflections dancing across every water droplet in the " + "starlight waterfalls. Ethereal, melancholic, and transcendent mood—like a moment of quiet revelation " + "at the edge of existence." + ), max_lpips=0.15, num_inference_steps=20, ), @@ -391,12 +411,6 @@ _OUTPUT_DIR = Path(os.environ["VLLM_OMNI_QUALITY_OUTPUT_DIR"]) if "VLLM_OMNI_QUA def _quality_param(c: QualityTestConfig): marks = list(_marks) - if c.id == "fp8_z_image": - marks.append( - pytest.mark.skip( - reason="Z-Image FP8 quality gate temporarily disabled: https://github.com/vllm-project/vllm-omni/issues/3531" - ) - ) if c.id == "fp8_qwen_image": marks.append( pytest.mark.skip(reason="Qwen-Image FP8 quality gate temporarily disabled (see CI / issue tracker).")