(Phase 2)Add ModelOpt mixed FP8/NVFP4 support for image generation (#3570)
Signed-off-by: roG0d <baonudesifeizhai@gmail.com> Signed-off-by: baonudesifeizhai <85092850+baonudesifeizhai@users.noreply.github.com>
This commit is contained in:
1 parent
7d8816b46b
commit
076c34abfb
16 files changed
+768
-68
No files matched your search
@@ -6,9 +6,11 @@ from __future__ import annotations
|
||||
import copy
|
||||
import gc
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -19,16 +21,17 @@ from PIL import Image
|
||||
from tests.e2e.accuracy.helpers import (
|
||||
CLIPScorer,
|
||||
SemanticSimilarityScorer,
|
||||
assert_similarity,
|
||||
compute_image_ssim_psnr,
|
||||
download_images,
|
||||
model_output_dir,
|
||||
)
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import build_prompt_tokens, resolve_stop_token_ids
|
||||
|
||||
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
|
||||
|
||||
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion, *hardware_marks(res={"cuda": "H100"}, num_cards=8)]
|
||||
pytestmark = [pytest.mark.local_model, pytest.mark.diffusion]
|
||||
|
||||
# ============================================================================
|
||||
# Configurable Parameters
|
||||
@@ -76,6 +79,18 @@ THRESHOLDS = {
|
||||
"ssim": 0.20, # Structural similarity
|
||||
"psnr": 11.0, # Peak signal-to-noise ratio (dB)
|
||||
}
|
||||
|
||||
QUANT_PROMPT = "A brown and white dog is running on the grass."
|
||||
QUANT_HEIGHT, QUANT_WIDTH = 1024, 1024
|
||||
QUANT_PSNR_THRESHOLD = 10.0
|
||||
QUANT_SSIM_THRESHOLD = 0.20
|
||||
QUANT_CLIP_SCORE_THRESHOLD = 20.0
|
||||
QUANT_CLIP_SCORE_DROP_THRESHOLD = float(os.environ.get("HUNYUAN_IMAGE3_QUANT_CLIP_SCORE_DROP_THRESHOLD", "5.0"))
|
||||
QUANT_RUN_ENV = "HUNYUAN_IMAGE3_RUN_QUANT_ACCURACY"
|
||||
QUANT_BF16_ENV = "HUNYUAN_IMAGE3_BF16_MODEL"
|
||||
QUANT_FP8_ENV = "HUNYUAN_IMAGE3_FP8_MODEL"
|
||||
QUANT_NVFP4_ENV = "HUNYUAN_IMAGE3_NVFP4_MODEL"
|
||||
_TRUE_ENV_VALUES = {"1", "true", "yes", "on"}
|
||||
# fmt: off
|
||||
_BASE_CONFIG = {
|
||||
"stage_args": [
|
||||
@@ -124,6 +139,64 @@ _BASE_CONFIG = {
|
||||
}
|
||||
# fmt: on
|
||||
|
||||
_QUANT_DIT_CONFIG = {
|
||||
"stage_args": [
|
||||
{
|
||||
"stage_id": 0,
|
||||
"stage_type": "diffusion",
|
||||
"runtime": {
|
||||
"devices": "0,1",
|
||||
"max_batch_size": 1,
|
||||
},
|
||||
"engine_args": {
|
||||
"model_stage": "dit",
|
||||
"enforce_eager": True,
|
||||
"trust_remote_code": True,
|
||||
"distributed_executor_backend": "mp",
|
||||
"force_cutlass_fp8": True,
|
||||
"moe_backend": "cutlass",
|
||||
"parallel_config": {
|
||||
"tensor_parallel_size": 2,
|
||||
"enable_expert_parallel": True,
|
||||
},
|
||||
"omni_kv_config": {"need_recv_cache": True},
|
||||
},
|
||||
"final_output": True,
|
||||
"final_output_type": "image",
|
||||
"is_comprehension": False,
|
||||
"default_sampling_params": {"seed": SEED},
|
||||
}
|
||||
],
|
||||
"runtime": {"enabled": True},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _QuantAccuracyCase:
|
||||
name: str
|
||||
model_env: str
|
||||
nvfp4_backend: str | None = None
|
||||
|
||||
|
||||
def _quant_accuracy_cases() -> list[pytest.ParameterSet]:
|
||||
cases = [
|
||||
_QuantAccuracyCase(name="fp8", model_env=QUANT_FP8_ENV),
|
||||
_QuantAccuracyCase(name="mixed_nvfp4", model_env=QUANT_NVFP4_ENV, nvfp4_backend="cutlass"),
|
||||
]
|
||||
params: list[pytest.ParameterSet] = []
|
||||
run_quant_accuracy = os.environ.get(QUANT_RUN_ENV, "").lower() in _TRUE_ENV_VALUES
|
||||
for case in cases:
|
||||
marks = []
|
||||
if not run_quant_accuracy:
|
||||
marks.append(pytest.mark.skip(reason=f"Set {QUANT_RUN_ENV}=1 to run HunyuanImage3 quant accuracy."))
|
||||
if not os.environ.get(QUANT_BF16_ENV):
|
||||
marks.append(pytest.mark.skip(reason=f"Set {QUANT_BF16_ENV} to run HunyuanImage3 quant accuracy."))
|
||||
if not os.environ.get(case.model_env):
|
||||
marks.append(pytest.mark.skip(reason=f"Set {case.model_env} to run this quant accuracy case."))
|
||||
params.append(pytest.param(case, id=case.name, marks=marks))
|
||||
return params
|
||||
|
||||
|
||||
# fmt: off
|
||||
COT_REF = ("首先,我分析所有输入图像:图像1是一个圆形的logo,设计现代且抽象。它由不同色调的蓝色(深蓝、中蓝、浅蓝)和白色构成,这些色块以流畅的曲线相互交织,形成一个动态的、类似旋涡或波浪的图案。整个logo是扁平化的矢量图形,背景为纯黑色。图像2展示了四个并排摆放的卡通动物造型冰箱贴,"
|
||||
"它们被放置在灰色的织物背景上。这些冰箱贴的关键特征是其材质:它们具有光滑、高光的珐琅或烤漆质感,边缘有明显的金属包边,整体呈现出一种立体的、有厚度的实体感。用户的指令是“基于图一的logo,参考图二中冰箱贴的材质,制作一个新的冰箱贴”。这个指令要求将一个二维的平面设计(logo)"
|
||||
@@ -144,6 +217,21 @@ def _make_config(enable_kv_reuse: bool, path: Path) -> None:
|
||||
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
||||
|
||||
|
||||
def _quant_devices() -> str:
|
||||
return os.environ.get("HUNYUAN_IMAGE3_QUANT_DEVICES", "0,1")
|
||||
|
||||
|
||||
def _quant_tensor_parallel_size() -> int:
|
||||
return int(os.environ.get("HUNYUAN_IMAGE3_QUANT_TP", str(len(_quant_devices().split(",")))))
|
||||
|
||||
|
||||
def _make_quant_dit_config(path: Path) -> None:
|
||||
config = copy.deepcopy(_QUANT_DIT_CONFIG)
|
||||
config["stage_args"][0]["runtime"]["devices"] = _quant_devices()
|
||||
config["stage_args"][0]["engine_args"]["parallel_config"]["tensor_parallel_size"] = _quant_tensor_parallel_size()
|
||||
path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))
|
||||
|
||||
|
||||
def _run_offline(stage_configs_path: str, output_path: Path) -> tuple[Image.Image, str, float]:
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -220,6 +308,65 @@ def _run_offline(stage_configs_path: str, output_path: Path) -> tuple[Image.Imag
|
||||
return image, cot_text, elapsed
|
||||
|
||||
|
||||
def _extract_image(outputs) -> Image.Image:
|
||||
assert outputs, "Pipeline produced no outputs"
|
||||
for output in outputs:
|
||||
images = getattr(output, "images", None)
|
||||
request_output = getattr(output, "request_output", None)
|
||||
if not images and request_output is not None:
|
||||
images = getattr(request_output, "images", None)
|
||||
if images:
|
||||
image = images[0].convert("RGB")
|
||||
image.load()
|
||||
return image
|
||||
raise AssertionError("Pipeline output had no images")
|
||||
|
||||
|
||||
def _run_dit_model(
|
||||
model: str,
|
||||
stage_config_path: str,
|
||||
output_path: Path,
|
||||
*,
|
||||
nvfp4_backend: str | None = None,
|
||||
) -> tuple[Image.Image, float]:
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
old_backend = os.environ.get("VLLM_NVFP4_GEMM_BACKEND")
|
||||
if nvfp4_backend is not None:
|
||||
os.environ["VLLM_NVFP4_GEMM_BACKEND"] = nvfp4_backend
|
||||
|
||||
try:
|
||||
with OmniRunner(model, stage_configs_path=stage_config_path, mode="text-to-image") as runner:
|
||||
generator = torch.Generator(device=current_omni_platform.device_type or "cuda").manual_seed(SEED)
|
||||
params = OmniDiffusionSamplingParams(
|
||||
height=QUANT_HEIGHT,
|
||||
width=QUANT_WIDTH,
|
||||
seed=SEED,
|
||||
generator=generator,
|
||||
num_inference_steps=20,
|
||||
guidance_scale=4.0,
|
||||
guidance_scale_provided=True,
|
||||
)
|
||||
t0 = time.perf_counter()
|
||||
outputs = runner.omni.generate({"prompt": QUANT_PROMPT}, params)
|
||||
elapsed = time.perf_counter() - t0
|
||||
image = _extract_image(outputs)
|
||||
image.save(output_path)
|
||||
return image, elapsed
|
||||
finally:
|
||||
if nvfp4_backend is not None:
|
||||
if old_backend is None:
|
||||
os.environ.pop("VLLM_NVFP4_GEMM_BACKEND", None)
|
||||
else:
|
||||
os.environ["VLLM_NVFP4_GEMM_BACKEND"] = old_backend
|
||||
gc.collect()
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=8)
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < AR_TP_SIZE + DIT_TP_SIZE,
|
||||
reason=f"Needs {AR_TP_SIZE + DIT_TP_SIZE}+ GPUs ({AR_TP_SIZE} AR + {DIT_TP_SIZE} DiT)",
|
||||
@@ -265,3 +412,88 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path, accuracy_assets_
|
||||
)
|
||||
assert ssim_value >= THRESHOLDS["ssim"], f"SSIM {ssim_value:.4f} is below threshold {THRESHOLDS['ssim']}"
|
||||
assert psnr_value >= THRESHOLDS["psnr"], f"PSNR {psnr_value:.2f} dB is below threshold {THRESHOLDS['psnr']} dB"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", _quant_accuracy_cases())
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Needs 2+ GPUs for HunyuanImage3 DiT")
|
||||
def test_quantized_dit_matches_bf16_accuracy(
|
||||
case: _QuantAccuracyCase,
|
||||
accuracy_artifact_root: Path,
|
||||
) -> None:
|
||||
"""Quantized DiT checkpoints should preserve prompt-aligned image quality."""
|
||||
output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME + "-quant")
|
||||
bf16_model = os.environ[QUANT_BF16_ENV]
|
||||
quant_model = os.environ[case.model_env]
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
stage_config_path = Path(tmpdir) / "hunyuan_image3_quant_dit.yaml"
|
||||
_make_quant_dit_config(stage_config_path)
|
||||
|
||||
bf16_image, bf16_time = _run_dit_model(
|
||||
bf16_model,
|
||||
str(stage_config_path),
|
||||
output_dir / "bf16.png",
|
||||
)
|
||||
quant_image, quant_time = _run_dit_model(
|
||||
quant_model,
|
||||
str(stage_config_path),
|
||||
output_dir / f"{case.name}.png",
|
||||
nvfp4_backend=case.nvfp4_backend,
|
||||
)
|
||||
|
||||
ssim_score, psnr_score = compute_image_ssim_psnr(
|
||||
prediction=quant_image,
|
||||
reference=bf16_image,
|
||||
)
|
||||
assert_similarity(
|
||||
model_name=f"{MODEL_NAME} {case.name} vs bf16",
|
||||
vllm_image=quant_image,
|
||||
diffusers_image=bf16_image,
|
||||
ssim_threshold=QUANT_SSIM_THRESHOLD,
|
||||
psnr_threshold=QUANT_PSNR_THRESHOLD,
|
||||
width=QUANT_WIDTH,
|
||||
height=QUANT_HEIGHT,
|
||||
)
|
||||
|
||||
clip_scorer = CLIPScorer()
|
||||
bf16_clip_score = clip_scorer.score(bf16_image, QUANT_PROMPT)
|
||||
quant_clip_score = clip_scorer.score(quant_image, QUANT_PROMPT)
|
||||
clip_score_drop = bf16_clip_score - quant_clip_score
|
||||
|
||||
metrics = {
|
||||
"case": case.name,
|
||||
"bf16_model": bf16_model,
|
||||
"quant_model": quant_model,
|
||||
"prompt": QUANT_PROMPT,
|
||||
"seed": SEED,
|
||||
"height": QUANT_HEIGHT,
|
||||
"width": QUANT_WIDTH,
|
||||
"num_inference_steps": 20,
|
||||
"guidance_scale": 4.0,
|
||||
"bf16_elapsed_s": bf16_time,
|
||||
"quant_elapsed_s": quant_time,
|
||||
"ssim": ssim_score,
|
||||
"psnr": psnr_score,
|
||||
"bf16_clip_score": bf16_clip_score,
|
||||
"quant_clip_score": quant_clip_score,
|
||||
"clip_score_drop": clip_score_drop,
|
||||
}
|
||||
metrics_path = output_dir / f"{case.name}_metrics.json"
|
||||
metrics_path.write_text(json.dumps(metrics, indent=2, sort_keys=True), encoding="utf-8")
|
||||
|
||||
print(f"\nHunyuanImage3 quant accuracy ({case.name})")
|
||||
print(f" bf16 model: {bf16_model}")
|
||||
print(f" quant model: {quant_model}")
|
||||
print(f" BF16 CLIP score: {bf16_clip_score:.4f}")
|
||||
print(f" quant CLIP score: {quant_clip_score:.4f} threshold>={QUANT_CLIP_SCORE_THRESHOLD:.4f}")
|
||||
print(f" CLIP score drop: {clip_score_drop:.4f} threshold<={QUANT_CLIP_SCORE_DROP_THRESHOLD:.4f}")
|
||||
print(f" metrics: {metrics_path}")
|
||||
|
||||
assert quant_clip_score >= QUANT_CLIP_SCORE_THRESHOLD, (
|
||||
f"{case.name} CLIP score below threshold: got {quant_clip_score:.4f}, "
|
||||
f"expected >= {QUANT_CLIP_SCORE_THRESHOLD:.4f}"
|
||||
)
|
||||
assert clip_score_drop <= QUANT_CLIP_SCORE_DROP_THRESHOLD, (
|
||||
f"{case.name} CLIP score drop too large: got {clip_score_drop:.4f}, "
|
||||
f"expected <= {QUANT_CLIP_SCORE_DROP_THRESHOLD:.4f}"
|
||||
)
|
||||
@@ -58,7 +58,8 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@pytest.mark.parametrize("method", sorted(PRE_QUANTIZED_METHODS))
|
||||
def test_pre_quantized_returns_none(method: str) -> None:
|
||||
"""visual_quant_config and audio_quant_config must be None for
|
||||
pre-quantized methods (modelopt, modelopt_fp4, modelopt_mxfp8)."""
|
||||
pre-quantized methods (modelopt, modelopt_fp4, modelopt_mxfp8,
|
||||
modelopt_mixed)."""
|
||||
mock_config = MagicMock()
|
||||
mock_config.get_name.return_value = method
|
||||
|
||||
@@ -100,5 +101,5 @@ def test_component_config_passed_through() -> None:
|
||||
|
||||
def test_pre_quantized_methods_contains_expected() -> None:
|
||||
"""Guard against accidental removal of a known pre-quantized method."""
|
||||
expected = {"modelopt", "modelopt_fp4", "modelopt_mxfp8"}
|
||||
expected = {"modelopt", "modelopt_fp4", "modelopt_mxfp8", "modelopt_mixed"}
|
||||
assert PRE_QUANTIZED_METHODS == expected
|
||||
Reference in new issue
Block a user