(Phase 2)Add ModelOpt mixed FP8/NVFP4 support for image generation (#3570)

Signed-off-by: roG0d <baonudesifeizhai@gmail.com>
Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
This commit is contained in:
baonudesifeizhai authored and GitHub committed 2026-05-22 10:39:10 +08:00
1 parent 7d8816b46b
commit 076c34abfb
16 files changed
+768 -68

No files matched your search

+234 -2
View File
@@ -6,9 +6,11 @@ from __future__ import annotations
import copy
import gc
import importlib
import json
import os
import tempfile
import time
from dataclasses import dataclass
from pathlib import Path
import pytest
@@ -19,16 +21,17 @@ from PIL import Image
from tests.e2e.accuracy.helpers import (
CLIPScorer,
SemanticSimilarityScorer,
assert_similarity,
compute_image_ssim_psnr,
download_images,
model_output_dir,
)
from tests.helpers.mark import hardware_marks
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import build_prompt_tokens, resolve_stop_token_ids
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion, *hardware_marks(res={"cuda": "H100"}, num_cards=8)]
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion]
# ============================================================================
# Configurable Parameters
@@ -76,6 +79,18 @@ THRESHOLDS = {
"ssim": 0.20, # Structural similarity
"psnr": 11.0, # Peak signal-to-noise ratio (dB)
}
QUANT_PROMPT = "A brown and white dog is running on the grass."
QUANT_HEIGHT, QUANT_WIDTH = 1024, 1024
QUANT_PSNR_THRESHOLD = 10.0
QUANT_SSIM_THRESHOLD = 0.20
QUANT_CLIP_SCORE_THRESHOLD = 20.0
QUANT_CLIP_SCORE_DROP_THRESHOLD = float(os.environ.get("HUNYUAN_IMAGE3_QUANT_CLIP_SCORE_DROP_THRESHOLD", "5.0"))
QUANT_RUN_ENV = "HUNYUAN_IMAGE3_RUN_QUANT_ACCURACY"
QUANT_BF16_ENV = "HUNYUAN_IMAGE3_BF16_MODEL"
QUANT_FP8_ENV = "HUNYUAN_IMAGE3_FP8_MODEL"
QUANT_NVFP4_ENV = "HUNYUAN_IMAGE3_NVFP4_MODEL"
_TRUE_ENV_VALUES = {"1", "true", "yes", "on"}
# fmt: off
_BASE_CONFIG = {
"stage_args": [
@@ -124,6 +139,64 @@ _BASE_CONFIG = {
}
# fmt: on
_QUANT_DIT_CONFIG = {
"stage_args": [
{
"stage_id": 0,
"stage_type": "diffusion",
"runtime": {
"devices": "0,1",
"max_batch_size": 1,
},
"engine_args": {
"model_stage": "dit",
"enforce_eager": True,
"trust_remote_code": True,
"distributed_executor_backend": "mp",
"force_cutlass_fp8": True,
"moe_backend": "cutlass",
"parallel_config": {
"tensor_parallel_size": 2,
"enable_expert_parallel": True,
},
"omni_kv_config": {"need_recv_cache": True},
},
"final_output": True,
"final_output_type": "image",
"is_comprehension": False,
"default_sampling_params": {"seed": SEED},
}
],
"runtime": {"enabled": True},
}
@dataclass(frozen=True)
class _QuantAccuracyCase:
name: str
model_env: str
nvfp4_backend: str | None = None
def _quant_accuracy_cases() -> list[pytest.ParameterSet]:
cases = [
_QuantAccuracyCase(name="fp8", model_env=QUANT_FP8_ENV),
_QuantAccuracyCase(name="mixed_nvfp4", model_env=QUANT_NVFP4_ENV, nvfp4_backend="cutlass"),
]
params: list[pytest.ParameterSet] = []
run_quant_accuracy = os.environ.get(QUANT_RUN_ENV, "").lower() in _TRUE_ENV_VALUES
for case in cases:
marks = []
if not run_quant_accuracy:
marks.append(pytest.mark.skip(reason=f"Set {QUANT_RUN_ENV}=1 to run HunyuanImage3 quant accuracy."))
if not os.environ.get(QUANT_BF16_ENV):
marks.append(pytest.mark.skip(reason=f"Set {QUANT_BF16_ENV} to run HunyuanImage3 quant accuracy."))
if not os.environ.get(case.model_env):
marks.append(pytest.mark.skip(reason=f"Set {case.model_env} to run this quant accuracy case."))
params.append(pytest.param(case, id=case.name, marks=marks))
return params
# fmt: off
COT_REF = ("首先,我分析所有输入图像:图像1是一个圆形的logo,设计现代且抽象。它由不同色调的蓝色(深蓝、中蓝、浅蓝)和白色构成,这些色块以流畅的曲线相互交织,形成一个动态的、类似旋涡或波浪的图案。整个logo是扁平化的矢量图形,背景为纯黑色。图像2展示了四个并排摆放的卡通动物造型冰箱贴,"
"它们被放置在灰色的织物背景上。这些冰箱贴的关键特征是其材质:它们具有光滑、高光的珐琅或烤漆质感,边缘有明显的金属包边,整体呈现出一种立体的、有厚度的实体感。用户的指令是“基于图一的logo,参考图二中冰箱贴的材质,制作一个新的冰箱贴”。这个指令要求将一个二维的平面设计(logo)"
@@ -144,6 +217,21 @@ def _make_config(enable_kv_reuse: bool, path: Path) -> None:
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
def _quant_devices() -> str:
return os.environ.get("HUNYUAN_IMAGE3_QUANT_DEVICES", "0,1")
def _quant_tensor_parallel_size() -> int:
return int(os.environ.get("HUNYUAN_IMAGE3_QUANT_TP", str(len(_quant_devices().split(",")))))
def _make_quant_dit_config(path: Path) -> None:
config = copy.deepcopy(_QUANT_DIT_CONFIG)
config["stage_args"][0]["runtime"]["devices"] = _quant_devices()
config["stage_args"][0]["engine_args"]["parallel_config"]["tensor_parallel_size"] = _quant_tensor_parallel_size()
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
def _run_offline(stage_configs_path: str, output_path: Path) -> tuple[Image.Image, str, float]:
from transformers import AutoTokenizer
@@ -220,6 +308,65 @@ def _run_offline(stage_configs_path: str, output_path: Path) -> tuple[Image.Imag
return image, cot_text, elapsed
def _extract_image(outputs) -> Image.Image:
assert outputs, "Pipeline produced no outputs"
for output in outputs:
images = getattr(output, "images", None)
request_output = getattr(output, "request_output", None)
if not images and request_output is not None:
images = getattr(request_output, "images", None)
if images:
image = images[0].convert("RGB")
image.load()
return image
raise AssertionError("Pipeline output had no images")
def _run_dit_model(
model: str,
stage_config_path: str,
output_path: Path,
*,
nvfp4_backend: str | None = None,
) -> tuple[Image.Image, float]:
from tests.helpers.runtime import OmniRunner
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform
old_backend = os.environ.get("VLLM_NVFP4_GEMM_BACKEND")
if nvfp4_backend is not None:
os.environ["VLLM_NVFP4_GEMM_BACKEND"] = nvfp4_backend
try:
with OmniRunner(model, stage_configs_path=stage_config_path, mode="text-to-image") as runner:
generator = torch.Generator(device=current_omni_platform.device_type or "cuda").manual_seed(SEED)
params = OmniDiffusionSamplingParams(
height=QUANT_HEIGHT,
width=QUANT_WIDTH,
seed=SEED,
generator=generator,
num_inference_steps=20,
guidance_scale=4.0,
guidance_scale_provided=True,
)
t0 = time.perf_counter()
outputs = runner.omni.generate({"prompt": QUANT_PROMPT}, params)
elapsed = time.perf_counter() - t0
image = _extract_image(outputs)
image.save(output_path)
return image, elapsed
finally:
if nvfp4_backend is not None:
if old_backend is None:
os.environ.pop("VLLM_NVFP4_GEMM_BACKEND", None)
else:
os.environ["VLLM_NVFP4_GEMM_BACKEND"] = old_backend
gc.collect()
if torch.accelerator.is_available():
torch.accelerator.empty_cache()
@hardware_test(res={"cuda": "H100"}, num_cards=8)
@pytest.mark.skipif(
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
@@ -265,3 +412,88 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_
)
assert ssim_value >= THRESHOLDS["ssim"], f"SSIM {ssim_value:.4f} is below threshold {THRESHOLDS['ssim']}"
assert psnr_value >= THRESHOLDS["psnr"], f"PSNR {psnr_value:.2f} dB is below threshold {THRESHOLDS['psnr']} dB"
@pytest.mark.parametrize("case", _quant_accuracy_cases())
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Needs 2+ GPUs for HunyuanImage3 DiT")
def test_quantized_dit_matches_bf16_accuracy(
case: _QuantAccuracyCase,
accuracy_artifact_root: Path,
) -> None:
"""Quantized DiT checkpoints should preserve prompt-aligned image quality."""
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME + "-quant")
bf16_model = os.environ[QUANT_BF16_ENV]
quant_model = os.environ[case.model_env]
with tempfile.TemporaryDirectory() as tmpdir:
stage_config_path = Path(tmpdir) / "hunyuan_image3_quant_dit.yaml"
_make_quant_dit_config(stage_config_path)
bf16_image, bf16_time = _run_dit_model(
bf16_model,
str(stage_config_path),
output_dir / "bf16.png",
)
quant_image, quant_time = _run_dit_model(
quant_model,
str(stage_config_path),
output_dir / f"{case.name}.png",
nvfp4_backend=case.nvfp4_backend,
)
ssim_score, psnr_score = compute_image_ssim_psnr(
prediction=quant_image,
reference=bf16_image,
)
assert_similarity(
model_name=f"{MODEL_NAME} {case.name} vs bf16",
vllm_image=quant_image,
diffusers_image=bf16_image,
ssim_threshold=QUANT_SSIM_THRESHOLD,
psnr_threshold=QUANT_PSNR_THRESHOLD,
width=QUANT_WIDTH,
height=QUANT_HEIGHT,
)
clip_scorer = CLIPScorer()
bf16_clip_score = clip_scorer.score(bf16_image, QUANT_PROMPT)
quant_clip_score = clip_scorer.score(quant_image, QUANT_PROMPT)
clip_score_drop = bf16_clip_score - quant_clip_score
metrics = {
"case": case.name,
"bf16_model": bf16_model,
"quant_model": quant_model,
"prompt": QUANT_PROMPT,
"seed": SEED,
"height": QUANT_HEIGHT,
"width": QUANT_WIDTH,
"num_inference_steps": 20,
"guidance_scale": 4.0,
"bf16_elapsed_s": bf16_time,
"quant_elapsed_s": quant_time,
"ssim": ssim_score,
"psnr": psnr_score,
"bf16_clip_score": bf16_clip_score,
"quant_clip_score": quant_clip_score,
"clip_score_drop": clip_score_drop,
}
metrics_path = output_dir / f"{case.name}_metrics.json"
metrics_path.write_text(json.dumps(metrics, indent=2, sort_keys=True), encoding="utf-8")
print(f"\nHunyuanImage3 quant accuracy ({case.name})")
print(f" bf16 model: {bf16_model}")
print(f" quant model: {quant_model}")
print(f" BF16 CLIP score: {bf16_clip_score:.4f}")
print(f" quant CLIP score: {quant_clip_score:.4f} threshold>={QUANT_CLIP_SCORE_THRESHOLD:.4f}")
print(f" CLIP score drop: {clip_score_drop:.4f} threshold<={QUANT_CLIP_SCORE_DROP_THRESHOLD:.4f}")
print(f" metrics: {metrics_path}")
assert quant_clip_score >= QUANT_CLIP_SCORE_THRESHOLD, (
f"{case.name} CLIP score below threshold: got {quant_clip_score:.4f}, "
f"expected >= {QUANT_CLIP_SCORE_THRESHOLD:.4f}"
)
assert clip_score_drop <= QUANT_CLIP_SCORE_DROP_THRESHOLD, (
f"{case.name} CLIP score drop too large: got {clip_score_drop:.4f}, "
f"expected <= {QUANT_CLIP_SCORE_DROP_THRESHOLD:.4f}"
)
@@ -58,7 +58,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize("method", sorted(PRE_QUANTIZED_METHODS))
def test_pre_quantized_returns_none(method: str) -> None:
"""visual_quant_config and audio_quant_config must be None for
pre-quantized methods (modelopt, modelopt_fp4, modelopt_mxfp8)."""
pre-quantized methods (modelopt, modelopt_fp4, modelopt_mxfp8,
modelopt_mixed)."""
mock_config = MagicMock()
mock_config.get_name.return_value = method
@@ -100,5 +101,5 @@ def test_component_config_passed_through() -> None:
def test_pre_quantized_methods_contains_expected() -> None:
"""Guard against accidental removal of a known pre-quantized method."""
expected = {"modelopt", "modelopt_fp4", "modelopt_mxfp8"}
expected = {"modelopt", "modelopt_fp4", "modelopt_mxfp8", "modelopt_mixed"}
assert PRE_QUANTIZED_METHODS == expected