From 9aed0e4eb8337f31a2081484fe4a3cbdb4bcd137 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Thu, 9 Jul 2026 02:39:14 +0800 Subject: [PATCH] test(accuracy): re-baseline HunyuanImage3 SSIM/PSNR for the vLLM rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HunyuanImage-3 (expert-parallel MoE) accuracy fails deterministically on dev/vllm-align (SSIM 0.9490 vs 0.97, PSNR 26.21 vs 30 dB) while passing on main's nightly. The align branch changes no hunyuan code; the drift comes from the newer upstream vLLM (65a7b4628409, 585 commits past the v0.24.0 tag main tracks) shifting MoE kernel numerics. The image still renders correctly and the mean/p99 pixel-diff checks pass — only the structural metrics dip below a strict bar. Per owner decision, re-baseline to the new stack: relax SSIM 0.97->0.94 and PSNR 30.0->25.0 (below the deterministic 0.9490 / 26.21, with margin). The HunyuanImage CI job uploads no image artifact, so the reference PNG cannot be auto-regenerated here; relaxing thresholds is the equivalent re-baseline. main keeps the stricter bar. Co-Authored-By: Claude --- .../accuracy/test_hunyuan_image3_pixel_accuracy.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 6ac930f46..157983caa 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -27,8 +27,15 @@ WIDTH = 1024 PROMPT = "A brown and white dog is running on the grass." MEAN_THRESHOLD = 3e-2 P99_THRESHOLD = 3e-1 -SSIM_THRESHOLD = 0.97 -PSNR_THRESHOLD = 30.0 +# NOTE: re-baselined for the vLLM rebase (dev/vllm-align). HunyuanImage-3 is an +# expert-parallel MoE model; the newer upstream vLLM (65a7b4628409, 585 commits +# past the v0.24.0 tag that main tracks) shifted MoE kernel numerics, so the +# structural-similarity metrics against the committed baseline drift to a +# deterministic SSIM 0.9490 / PSNR 26.21 dB (the mean/p99 pixel-diff checks above +# still pass — the image renders correctly). Thresholds relaxed to accept this +# drift as the new bar. On main (older vLLM) the stricter 0.97 / 30.0 hold. +SSIM_THRESHOLD = 0.94 +PSNR_THRESHOLD = 25.0 _REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent BASELINE_PATH = _REPO_ROOT / "tests" / "assets" / "hunyuan" / "hunyuan_baseline.png"