[CI] add qwen image and layered accuracy test (#2772)

Signed-off-by: david6666666 <530634352@qq.com>
This commit is contained in:
WeiQing Chen authored and GitHub committed 2026-04-16 14:52:53 +08:00
1 parent b43c6c6663
commit 24e61f4d7b
3 files changed
+313 -9

No files matched your search

+38 -9
View File
@@ -21,13 +21,14 @@ def assert_similarity(
model_name: str,
vllm_image: Image.Image,
diffusers_image: Image.Image,
width: int,
height: int,
ssim_threshold: float,
psnr_threshold: float,
width: int | None = None,
height: int | None = None,
compare_mode: str = "RGB",
) -> None:
requested_size = (width, height)
if diffusers_image.size != requested_size:
requested_size = (width, height) if width is not None and height is not None else None
if requested_size is not None and diffusers_image.size != requested_size:
pytest.skip(
"Skipping as diffusers baseline output is corrupt and not comparable: "
f"dimensions do not match requested size; requested={requested_size}, got={diffusers_image.size}."
@@ -37,7 +38,11 @@ def assert_similarity(
f"Online and diffusers output sizes mismatch: online={vllm_image.size}, diffusers={diffusers_image.size}"
)
ssim_score, psnr_score = compute_image_ssim_psnr(prediction=vllm_image, reference=diffusers_image)
ssim_score, psnr_score = compute_image_ssim_psnr(
prediction=vllm_image,
reference=diffusers_image,
compare_mode=compare_mode,
)
print(f"{model_name} similarity metrics:")
print(f" SSIM: value={ssim_score:.6f}, threshold>={ssim_threshold:.6f}, range=[-1, 1], higher_is_better=True")
print(
@@ -52,13 +57,37 @@ def assert_similarity(
)
def assert_image_sequence_similarity(
*,
model_name: str,
vllm_images: list[Image.Image],
diffusers_images: list[Image.Image],
ssim_threshold: float,
psnr_threshold: float,
compare_mode: str = "RGB",
) -> None:
assert len(vllm_images) == len(diffusers_images), (
f"Output image count mismatch for {model_name}: online={len(vllm_images)}, diffusers={len(diffusers_images)}"
)
for index, (vllm_image, diffusers_image) in enumerate(zip(vllm_images, diffusers_images, strict=True), start=1):
assert_similarity(
model_name=f"{model_name}[layer={index}]",
vllm_image=vllm_image,
diffusers_image=diffusers_image,
ssim_threshold=ssim_threshold,
psnr_threshold=psnr_threshold,
compare_mode=compare_mode,
)
def compute_image_ssim_psnr(
*,
prediction: Image.Image,
reference: Image.Image,
compare_mode: str = "RGB",
) -> tuple[float, float]:
pred_tensor = _pil_to_batched_tensor(prediction)
ref_tensor = _pil_to_batched_tensor(reference)
pred_tensor = _pil_to_batched_tensor(prediction, compare_mode=compare_mode)
ref_tensor = _pil_to_batched_tensor(reference, compare_mode=compare_mode)
ssim_metric = StructuralSimilarityIndexMeasure(data_range=1.0)
psnr_metric = PeakSignalNoiseRatio(data_range=1.0)
@@ -68,7 +97,7 @@ def compute_image_ssim_psnr(
return ssim_value, psnr_value
def _pil_to_batched_tensor(image: Image.Image) -> torch.Tensor:
array = np.asarray(image.convert("RGB"), dtype=np.float32) / 255.0
def _pil_to_batched_tensor(image: Image.Image, *, compare_mode: str) -> torch.Tensor:
array = np.asarray(image.convert(compare_mode), dtype=np.float32) / 255.0
tensor = torch.from_numpy(array).permute(2, 0, 1).unsqueeze(0)
return tensor