[Feat] Add support and testcase for HunyuanImage-3.0-Instruct-Distil (#4048)

Signed-off-by: zengchuang <zengchuang3@huawei.com>
Signed-off-by: zengchuang-hw <zengchuang1005@gmail.com>
This commit is contained in:
Zeng Chuang authored and GitHub committed 2026-09-03 02:43:13 +00:00
1 parent 4927fd5412
commit b5e898f295
12 files changed
+751 -102

No files matched your search

+102 -69
View File
@@ -94,15 +94,28 @@ def _default_ar_dit_devices() -> tuple[str, str]:
AR_DEVICES, DIT_DEVICES = _default_ar_dit_devices()
MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
HUNYUAN_IMAGE_REF_PATH = get_asset_path("hunyuan/hunyuan_image_ref.png")
NUM_INFERENCE_STEPS = 50
# Model configuration based on HUNYUAN_MODEL_PATH
DISTIL_MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct-Distil"
INSTRUCT_MODEL_NAME = "tencent/HunyuanImage-3.0-Instruct"
MODEL_PATH = os.environ.get("HUNYUAN_MODEL_PATH", DISTIL_MODEL_NAME)
# Determine model type and set parameters accordingly
IS_DISTIL = "distil" in MODEL_PATH.lower()
MODEL_NAME = DISTIL_MODEL_NAME if IS_DISTIL else INSTRUCT_MODEL_NAME
NUM_INFERENCE_STEPS = 8 if IS_DISTIL else 50
GUIDANCE_SCALE = 2.5
# Asset paths based on model type
ASSET_SUBDIR = "hunyuan_image3"
REF_IMAGE_NAME = "hunyuan_image_distill_ref.png" if IS_DISTIL else "hunyuan_image_instruct_ref.png"
COT_REF_NAME = "hunyuan_image_distill_cot_ref.txt" if IS_DISTIL else "hunyuan_image3_instruct_cot_ref.txt"
HUNYUAN_IMAGE_REF_PATH = get_asset_path(f"{ASSET_SUBDIR}/{REF_IMAGE_NAME}")
HUNYUAN_COT_REF_PATH = get_asset_path(f"{ASSET_SUBDIR}/{COT_REF_NAME}")
# ============================================================================
# Constants
# ============================================================================
MODEL_PATH = os.environ.get("HUNYUAN_MODEL_PATH", MODEL_NAME)
# Test input
PROMPT = "基于图一的logo,参考图二中冰箱贴的材质,制作一个新的冰箱贴"
TEST_IMAGE_URLS = [
@@ -113,23 +126,52 @@ SEED = 42
AR_TP_SIZE = len(AR_DEVICES.split(","))
DIT_TP_SIZE = len(DIT_DEVICES.split(","))
# Precision thresholds
# Precision thresholds organized by model variant and test type
THRESHOLDS = {
# AR text comparison
"text_prefix_match": 10, # First 10 characters must match exactly
"cot_semantic_sim": 0.9, # Full CoT semantic similarity
# Image comparison
"clip_score": 90, # CLIP image semantic similarity
"ssim": 0.26, # Structural similarity
"psnr": 12.5, # Peak signal-to-noise ratio (dB)
"instruct": {
# AR text comparison
"text_prefix_match": 10, # First 10 characters must match exactly
"cot_semantic_sim": 0.9, # Full CoT semantic similarity
# Image comparison
"clip_score": 90, # CLIP image semantic similarity
"ssim": 0.26, # Structural similarity
"psnr": 12.5, # Peak signal-to-noise ratio (dB)
},
# The 8-step distilled model has more structural variance across GPU
# runtimes; CLIP and PSNR retain the same semantic and pixel safeguards.
"distil": {
"text_prefix_match": 10,
"cot_semantic_sim": 0.9,
"clip_score": 90,
"ssim": 0.23,
"psnr": 12.5,
},
# Thresholds for quantized DiT accuracy tests (bf16 vs quant comparison)
"quant": {
"psnr": 10.0,
"ssim": 0.20,
"clip_score": 20.0, # Minimum CLIP score for quantized output
"clip_score_drop": float(
os.environ.get("HUNYUAN_IMAGE3_QUANT_CLIP_SCORE_DROP_THRESHOLD", "5.0")
), # Max allowed drop from bf16
},
}
# Helper to get thresholds for current model variant
def _get_thresholds() -> dict:
"""Return appropriate thresholds based on test context."""
return THRESHOLDS["distil" if IS_DISTIL else "instruct"]
def _get_quant_thresholds() -> dict:
"""Return thresholds for quantization accuracy tests."""
return THRESHOLDS["quant"]
# Quant test configuration
QUANT_PROMPT = "A brown and white dog is running on the grass."
QUANT_HEIGHT, QUANT_WIDTH = 1024, 1024
QUANT_PSNR_THRESHOLD = 10.0
QUANT_SSIM_THRESHOLD = 0.20
QUANT_CLIP_SCORE_THRESHOLD = 20.0
QUANT_CLIP_SCORE_DROP_THRESHOLD = float(os.environ.get("HUNYUAN_IMAGE3_QUANT_CLIP_SCORE_DROP_THRESHOLD", "5.0"))
QUANT_RUN_ENV = "HUNYUAN_IMAGE3_RUN_QUANT_ACCURACY"
QUANT_BF16_ENV = "HUNYUAN_IMAGE3_BF16_MODEL"
QUANT_FP8_ENV = "HUNYUAN_IMAGE3_FP8_MODEL"
@@ -167,7 +209,6 @@ _DEPLOY_CONFIG = {
"top_p": 1,
"top_k": -1,
"max_tokens": 8192,
"stop_token_ids": [128025],
"detokenize": True,
"skip_special_tokens": False,
"include_stop_str_in_output": True,
@@ -246,19 +287,6 @@ def _quant_accuracy_cases() -> list[pytest.ParameterSet]:
return params
# fmt: off
COT_REF = ("首先,我分析所有输入图像:图像1是一个圆形的logo,设计现代且抽象。它由不同色调的蓝色(深蓝、中蓝、浅蓝)和白色构成,这些色块以流畅的曲线相互交织,形成一个动态的、类似旋涡或波浪的图案。整个logo是扁平化的矢量图形,背景为纯黑色。图像2展示了四个并排摆放的卡通动物造型冰箱贴,"
"它们被放置在灰色的织物背景上。这些冰箱贴的关键特征是其材质:它们具有光滑、高光的珐琅或烤漆质感,边缘有明显的金属包边,整体呈现出一种立体的、有厚度的实体感。用户的指令是“基于图一的logo,参考图二中冰箱贴的材质,制作一个新的冰箱贴”。这个指令要求将一个二维的平面设计(logo)"
"转化为一个具有特定物理属性(材质和立体感)的三维物体。核心任务是保留logo的视觉识别性,同时赋予其冰箱贴的实体质感。为了构建答案图像,我会将图一的圆形logo作为基础形状。然后,我会将图二中冰箱贴的材质特性应用到这个logo上。具体来说,logo中的每一个色块(深蓝、中蓝、浅蓝、白色)"
"都会被渲染成具有高光泽度的珐琅质感,表面会反射出柔和的环境光,形成自然的高光。logo中不同颜色区域之间的分界线,将被处理成纤细的、带有金属光泽的凸起边缘,这既能清晰地勾勒出图案,也符合珐琅工艺品的典型特征。整个冰箱贴会呈现出轻微的厚度和圆润的边缘,使其看起来像一个真实的、可触摸的物体。"
"最后,将这个制作完成的冰箱贴放置在图二所示的灰色织物背景上,并为其添加一个微妙的、柔和的阴影,以增强其立体感和与背景的融合度,最终呈现出一个精致、逼真的产品展示图。</think><recaption>这幅图像以产品摄影的精致风格,呈现了一枚根据`image_1`标志定制的圆形珐琅冰箱贴。最终图像使用`image_2`的分辨率。"
"冰箱贴居中放置在`image_2`的灰色织物背景上,其设计完美复刻了`image_1`中由深蓝、中蓝、浅蓝和白色构成的动态旋涡图案。整个冰箱贴被赋予了`image_2`中冰箱贴特有的高级质感:表面覆盖着一层光滑如镜的珐琅釉面,反射出柔和而清晰的高光;图案的每一个色块边缘都由纤细的抛光金属边框精确勾勒,增强了立体感。"
"柔和的顶光在冰箱贴的弧形边缘上形成平滑的过渡,并在其下方投下淡淡的、轮廓模糊的阴影,使其与织物背景无缝融合,营造出一种真实、静谧的视觉效果。<relation_1>最终图像完整保留了`image_1`中标志的全部设计元素。这包括其完美的圆形轮廓,以及内部由深蓝、中蓝、浅蓝和白色组成的精确旋涡状图案布局、形状和色彩关系。"
"</relation_1><relation_2>最终图像的分辨率、背景和材质均来自`image_2`。背景中灰色织物的纹理和质感被完整保留。冰箱贴的材质被完美重构,精确复刻了`image_2`中冰箱贴所展示的光滑珐琅质感、抛光金属边框的视觉效果,以及整体柔和、均匀的布光环境和由此产生的自然阴影。</relation_2></recaption><answer><boi>"
"<img_size_1024><img_ratio_36><timestep>[<img>]{3600}<eoi></answer>")
# fmt: on
def _make_config(enable_kv_reuse: bool, path: Path) -> None:
config = copy.deepcopy(_DEPLOY_CONFIG)
config["stages"][0]["omni_kv_config"] = {"need_send_cache": enable_kv_reuse}
@@ -432,34 +460,36 @@ def test_image_to_image_alignment_online(accuracy_artifact_root: Path) -> None:
online_cot = online_cot.lstrip("\n")
scorer = SemanticSimilarityScorer()
clip_scorer = CLIPScorer()
cot_results = scorer.text_similarity(online_cot, COT_REF)
cot_ref = HUNYUAN_COT_REF_PATH.read_text(encoding="utf-8")
cot_results = scorer.text_similarity(online_cot, cot_ref)
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
image_clip_score = clip_scorer.image_image_score(online_image, image_ref)
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=online_image, reference=image_ref, compare_mode="RGB")
thresholds = _get_thresholds()
table = [
["COT similarity to reference", f"{cot_results['cot_semantic_sim']:.4f}", 0.9644],
["COT prefix match", f"{cot_results['text_prefix_match_count']:.4f}", 29],
["Image-Image similarity", f"{image_clip_score:.4f}", 94.5538],
["SSIM", f"{ssim_value:.4f}", 0.242],
["PSNR (dB)", f"{psnr_value:.2f}", 14.1],
["COT similarity to reference", f"{cot_results['cot_semantic_sim']:.4f}", thresholds["cot_semantic_sim"]],
["COT prefix match", f"{cot_results['text_prefix_match_count']:.4f}", thresholds["text_prefix_match"]],
["Image-Image similarity", f"{image_clip_score:.4f}", thresholds["clip_score"]],
["SSIM", f"{ssim_value:.4f}", thresholds["ssim"]],
["PSNR (dB)", f"{psnr_value:.2f}", thresholds["psnr"]],
]
print("[ONLINE] " + tabulate(table, headers=["Metric", "Value", "L20x Reference"], tablefmt="grid"))
print("[ONLINE] " + tabulate(table, headers=["Metric", "Value", "Threshold"], tablefmt="grid"))
_assert_cot_structural_markers(online_cot, "ONLINE")
assert cot_results["cot_semantic_sim"] >= THRESHOLDS["cot_semantic_sim"], (
f"[ONLINE] COT semantic similarity {cot_results['cot_semantic_sim']:.4f} below threshold {THRESHOLDS['cot_semantic_sim']}"
assert cot_results["cot_semantic_sim"] >= thresholds["cot_semantic_sim"], (
f"[ONLINE] COT semantic similarity {cot_results['cot_semantic_sim']:.4f} below threshold {thresholds['cot_semantic_sim']}"
)
assert cot_results["text_prefix_match_count"] >= THRESHOLDS["text_prefix_match"], (
f"[ONLINE] COT prefix match {cot_results['text_prefix_match_count']} below threshold {THRESHOLDS['text_prefix_match']}"
assert cot_results["text_prefix_match_count"] >= thresholds["text_prefix_match"], (
f"[ONLINE] COT prefix match {cot_results['text_prefix_match_count']} below threshold {thresholds['text_prefix_match']}"
)
assert image_clip_score >= THRESHOLDS["clip_score"], (
f"[ONLINE] Image-Image similarity {image_clip_score:.4f} below threshold {THRESHOLDS['clip_score']}"
assert image_clip_score >= thresholds["clip_score"], (
f"[ONLINE] Image-Image similarity {image_clip_score:.4f} below threshold {thresholds['clip_score']}"
)
assert ssim_value >= THRESHOLDS["ssim"], f"[ONLINE] SSIM {ssim_value:.4f} below threshold {THRESHOLDS['ssim']}"
assert psnr_value >= THRESHOLDS["psnr"], (
f"[ONLINE] PSNR {psnr_value:.2f} dB below threshold {THRESHOLDS['psnr']} dB"
assert ssim_value >= thresholds["ssim"], f"[ONLINE] SSIM {ssim_value:.4f} below threshold {thresholds['ssim']}"
assert psnr_value >= thresholds["psnr"], (
f"[ONLINE] PSNR {psnr_value:.2f} dB below threshold {thresholds['psnr']} dB"
)
@@ -546,34 +576,36 @@ def test_image_to_image_alignment(accuracy_artifact_root: Path) -> None:
scorer = SemanticSimilarityScorer()
clip_scorer = CLIPScorer()
cot_results = scorer.text_similarity(omni_cot, COT_REF)
cot_ref = HUNYUAN_COT_REF_PATH.read_text(encoding="utf-8")
cot_results = scorer.text_similarity(omni_cot, cot_ref)
image_ref = Image.open(HUNYUAN_IMAGE_REF_PATH).convert("RGB")
image_clip_score = clip_scorer.image_image_score(omni_image, image_ref)
ssim_value, psnr_value = compute_image_ssim_psnr(prediction=omni_image, reference=image_ref, compare_mode="RGB")
thresholds = _get_thresholds()
table = [
["COT similarity to reference", f"{cot_results['cot_semantic_sim']:.4f}", 0.9644],
["COT prefix match", f"{cot_results['text_prefix_match_count']:.4f}", 29],
["Image-Image similarity", f"{image_clip_score:.4f}", 94.5538],
["SSIM", f"{ssim_value:.4f}", 0.242],
["PSNR (dB)", f"{psnr_value:.2f}", 14.1],
["COT similarity to reference", f"{cot_results['cot_semantic_sim']:.4f}", thresholds["cot_semantic_sim"]],
["COT prefix match", f"{cot_results['text_prefix_match_count']:.4f}", thresholds["text_prefix_match"]],
["Image-Image similarity", f"{image_clip_score:.4f}", thresholds["clip_score"]],
["SSIM", f"{ssim_value:.4f}", thresholds["ssim"]],
["PSNR (dB)", f"{psnr_value:.2f}", thresholds["psnr"]],
]
print(tabulate(table, headers=["Metric", "Value", "L20x Reference"], tablefmt="grid"))
print(tabulate(table, headers=["Metric", "Value", "Threshold"], tablefmt="grid"))
_assert_cot_structural_markers(omni_cot, "OFFLINE")
assert cot_results["cot_semantic_sim"] >= THRESHOLDS["cot_semantic_sim"], (
f"COT semantic similarity {cot_results['cot_semantic_sim']:.4f} is below threshold {THRESHOLDS['cot_semantic_sim']}"
assert cot_results["cot_semantic_sim"] >= thresholds["cot_semantic_sim"], (
f"COT semantic similarity {cot_results['cot_semantic_sim']:.4f} is below threshold {thresholds['cot_semantic_sim']}"
)
assert cot_results["text_prefix_match_count"] >= THRESHOLDS["text_prefix_match"], (
f"COT prefix match count {cot_results['text_prefix_match_count']} is below threshold {THRESHOLDS['text_prefix_match']}"
assert cot_results["text_prefix_match_count"] >= thresholds["text_prefix_match"], (
f"COT prefix match count {cot_results['text_prefix_match_count']} is below threshold {thresholds['text_prefix_match']}"
)
assert image_clip_score >= THRESHOLDS["clip_score"], (
f"Image-Image similarity{image_clip_score:.4f} is below threshold {THRESHOLDS['clip_score']}"
assert image_clip_score >= thresholds["clip_score"], (
f"Image-Image similarity{image_clip_score:.4f} is below threshold {thresholds['clip_score']}"
)
assert ssim_value >= THRESHOLDS["ssim"], f"SSIM {ssim_value:.4f} is below threshold {THRESHOLDS['ssim']}"
assert psnr_value >= THRESHOLDS["psnr"], f"PSNR {psnr_value:.2f} dB is below threshold {THRESHOLDS['psnr']} dB"
assert ssim_value >= thresholds["ssim"], f"SSIM {ssim_value:.4f} is below threshold {thresholds['ssim']}"
assert psnr_value >= thresholds["psnr"], f"PSNR {psnr_value:.2f} dB is below threshold {thresholds['psnr']} dB"
@pytest.mark.parametrize("case", _quant_accuracy_cases())
@@ -607,12 +639,13 @@ def test_quantized_dit_matches_bf16_accuracy(
prediction=quant_image,
reference=bf16_image,
)
quant_thresholds = _get_quant_thresholds()
assert_similarity(
model_name=f"{MODEL_NAME} {case.name} vs bf16",
vllm_image=quant_image,
diffusers_image=bf16_image,
ssim_threshold=QUANT_SSIM_THRESHOLD,
psnr_threshold=QUANT_PSNR_THRESHOLD,
ssim_threshold=quant_thresholds["ssim"],
psnr_threshold=quant_thresholds["psnr"],
width=QUANT_WIDTH,
height=QUANT_HEIGHT,
)
@@ -647,15 +680,15 @@ def test_quantized_dit_matches_bf16_accuracy(
print(f" bf16 model: {bf16_model}")
print(f" quant model: {quant_model}")
print(f" BF16 CLIP score: {bf16_clip_score:.4f}")
print(f" quant CLIP score: {quant_clip_score:.4f} threshold>={QUANT_CLIP_SCORE_THRESHOLD:.4f}")
print(f" CLIP score drop: {clip_score_drop:.4f} threshold<={QUANT_CLIP_SCORE_DROP_THRESHOLD:.4f}")
print(f" quant CLIP score: {quant_clip_score:.4f} threshold>={quant_thresholds['clip_score']:.4f}")
print(f" CLIP score drop: {clip_score_drop:.4f} threshold<={quant_thresholds['clip_score_drop']:.4f}")
print(f" metrics: {metrics_path}")
assert quant_clip_score >= QUANT_CLIP_SCORE_THRESHOLD, (
assert quant_clip_score >= quant_thresholds["clip_score"], (
f"{case.name} CLIP score below threshold: got {quant_clip_score:.4f}, "
f"expected >= {QUANT_CLIP_SCORE_THRESHOLD:.4f}"
f"expected >= {quant_thresholds['clip_score']:.4f}"
)
assert clip_score_drop <= QUANT_CLIP_SCORE_DROP_THRESHOLD, (
assert clip_score_drop <= quant_thresholds["clip_score_drop"], (
f"{case.name} CLIP score drop too large: got {clip_score_drop:.4f}, "
f"expected <= {QUANT_CLIP_SCORE_DROP_THRESHOLD:.4f}"
f"expected <= {quant_thresholds['clip_score_drop']:.4f}"
)