[Feat] Add support and testcase for HunyuanImage-3.0-Instruct-Distil (#4048)

Signed-off-by: zengchuang <zengchuang3@huawei.com>
Signed-off-by: zengchuang-hw <zengchuang1005@gmail.com>
This commit is contained in:
Zeng Chuang authored and GitHub committed 2026-09-03 02:43:13 +00:00
1 parent 4927fd5412
commit b5e898f295
12 files changed
+751 -102

No files matched your search

@@ -22,6 +22,7 @@ class TokenizerEncodeOutput(BaseOutput):
tokens: torch.Tensor | None = None
timestep_scatter_index: torch.Tensor | None = None
guidance_scatter_index: torch.Tensor | None = None
timesteps_r_scatter_index: torch.Tensor | None = None # For MeanFlow distilled models
text_slices: list[slice] | None = None
gen_image_slices: list[slice] | None = None
joint_image_slices: list[slice] | None = None
@@ -196,6 +197,7 @@ class TokenizerWrapper:
extra_token_pos,
add_timestep_token=False,
add_image_shape_token=False,
add_timestep_r_token=False,
base_size=None,
ratio_idx=None,
image_type=None,
@@ -221,6 +223,10 @@ class TokenizerWrapper:
token_seq.extend([self.special_token_map["<guidance>"]])
extra_token_pos["guidance"].append(token_count)
token_count += 1
if add_timestep_r_token:
token_seq.extend([self.special_token_map["<timestep_r>"]])
extra_token_pos["gen_timestep_r"].append(token_count)
token_count += 1
return token_count
def encode_sequence(
@@ -230,6 +236,7 @@ class TokenizerWrapper:
total_length=None,
add_timestep_token=False,
add_guidance_token=False,
add_timestep_r_token=False,
last_key_only_prefix=False,
add_eos=True,
use_front_boi_token=True,
@@ -329,6 +336,7 @@ class TokenizerWrapper:
2
+ (1 if source.get("timestep", add_timestep_token) else 0)
+ (1 if source.get("guidance", add_guidance_token) else 0)
+ (1 if source.get("timestep_r", add_timestep_r_token) else 0)
+ (2 if source.get("image_shape", add_image_shape_token) else 0)
)
if drop_last is True and token_count + extra_count + source["length"] > total_length:
@@ -344,6 +352,7 @@ class TokenizerWrapper:
extra_token_pos=extra_token_pos,
add_timestep_token=source.get("timestep", add_timestep_token),
add_guidance_token=source.get("guidance", add_guidance_token),
add_timestep_r_token=source.get("timestep_r", add_timestep_r_token),
add_image_shape_token=source.get("image_shape", add_image_shape_token),
base_size=source.get("base_size"),
ratio_idx=source.get("ratio_idx"),
@@ -739,6 +748,7 @@ class TokenizerWrapper:
length=section["token_length"],
timestep=section.get("add_timestep_token", False),
guidance=section.get("add_guidance_token", False),
timestep_r=section.get("add_timestep_r_token", False),
front_boi=section.get("use_front_boi_token", False),
image_shape=section.get("add_image_shape_token", False),
base_size=section.get("base_size"),
@@ -777,6 +787,11 @@ class TokenizerWrapper:
guidance_scatter_index = (
torch.tensor(extra_token_pos["guidance"], dtype=torch.long) if "guidance" in extra_token_pos else None
)
timesteps_r_scatter_index = (
torch.tensor(extra_token_pos["gen_timestep_r"], dtype=torch.long)
if "gen_timestep_r" in extra_token_pos
else None
)
cond_timestep_scatter_index = (
torch.tensor(extra_token_pos["cond_timestep"], dtype=torch.long)
if "cond_timestep" in extra_token_pos
@@ -847,6 +862,7 @@ class TokenizerWrapper:
tokens=full_seq_token_tensor,
timestep_scatter_index=timestep_scatter_index,
guidance_scatter_index=guidance_scatter_index,
timesteps_r_scatter_index=timesteps_r_scatter_index,
text_slices=text_slices,
gen_image_slices=gen_image_slices,
joint_image_slices=joint_image_slices,
@@ -630,6 +630,7 @@ class ImageInfo:
self.add_timestep_token = kwargs.get("add_timestep_token", True)
self.add_guidance_token = kwargs.get("add_guidance_token", False)
self.add_timestep_r_token = kwargs.get("add_timestep_r_token", False)
self.use_front_boi_token = kwargs.get("use_front_boi_token", True)
self.add_image_shape_token = kwargs.get("add_image_shape_token", True)
@@ -667,6 +668,7 @@ class ImageInfo:
token_length=self.image_token_length,
add_timestep_token=self.add_timestep_token,
add_guidance_token=self.add_guidance_token,
add_timestep_r_token=self.add_timestep_r_token,
use_front_boi_token=self.use_front_boi_token,
add_image_shape_token=self.add_image_shape_token,
base_size=self.base_size,
@@ -1442,6 +1444,9 @@ class HunyuanImage3Config(PretrainedConfig):
vit=None,
vit_processor=None,
vit_aligner=None,
# Distillation flags
cfg_distilled: bool = False,
use_meanflow: bool = False,
**kwargs,
):
self.vocab_size = vocab_size
@@ -1521,6 +1526,10 @@ class HunyuanImage3Config(PretrainedConfig):
self.patch_embed_hidden_dim = patch_embed_hidden_dim
self.image_base_size = image_base_size
# Distillation flags
self.cfg_distilled = cfg_distilled
self.use_meanflow = use_meanflow
# token id
self.eod_token_id = eod_token_id
self.im_start_id = im_start_id
@@ -1555,7 +1564,7 @@ class HunyuanImage3ImageProcessor:
)
self.vision_encoder_processor = Siglip2ImageProcessorFast.from_dict(config.vit_processor)
def build_image_info(self, image_size):
def build_image_info(self, image_size, add_guidance_token=False, add_timestep_r_token=False):
# parse image size (HxW, H:W, or <img_ratio_i>)
if isinstance(image_size, str):
if image_size.startswith("<img_ratio_"):
@@ -1592,6 +1601,8 @@ class HunyuanImage3ImageProcessor:
token_height=token_height,
base_size=base_size,
ratio_index=ratio_idx,
add_guidance_token=add_guidance_token,
add_timestep_r_token=add_timestep_r_token,
)
return image_info
@@ -2244,6 +2255,8 @@ class HunyuanImage3Model(nn.Module):
"timestep_emb",
"time_embed",
"time_embed_2",
# These embeddings belong to the outer pipeline model rather than
# this transformer core and are loaded by the outer weight loader.
"guidance_emb",
"timestep_r_emb",
]
@@ -2802,6 +2815,8 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
"position_ids",
"image_mask",
"gen_timestep_scatter_index",
"guidance_scatter_index",
"timesteps_r_scatter_index",
"cond_vae_image_mask",
"cond_vit_image_mask",
"cond_timestep_scatter_index",
@@ -2971,9 +2986,12 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
model_kwargs["attention_mask"] = model_kwargs["attention_mask"][:, :, positive_reuse_len:, :]
model_kwargs["position_ids"] = model_kwargs["position_ids"][:, positive_reuse_len:]
# Shift image_mask and gen_timestep_scatter_index to match truncated sequence
# Shift generated-image placeholder indices to match the truncated sequence.
model_kwargs["image_mask"] = model_kwargs["image_mask"][:, positive_reuse_len:]
model_kwargs["gen_timestep_scatter_index"] = model_kwargs["gen_timestep_scatter_index"] - positive_reuse_len
for key in ("guidance_scatter_index", "timesteps_r_scatter_index"):
if model_kwargs.get(key) is not None:
model_kwargs[key] = model_kwargs[key] - positive_reuse_len
model_kwargs["ar_kv_reuse_offset"] = positive_reuse_len
# cond-image have computed in ar, we may skip it by index in the future.
@@ -3016,7 +3034,8 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
self.model.inject_ar_kv_into_layers(ar_kv_data, positive_reuse_len)
# 3. negative cfg prefill
if self.do_classifier_free_guidance:
# For CFG distilled models, skip negative CFG prefill (cfg_factor=1, no negative prompt)
if self.do_classifier_free_guidance and not self.model.config.cfg_distilled:
self._maybe_run_negative_cfg_prefill(
input_ids=input_ids,
model_kwargs=model_kwargs,
@@ -3124,7 +3143,12 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
self._guidance_rescale = guidance_rescale
# Detect CFG parallel configuration (only 2-branch layout is supported)
cfg_parallel_ready = self.do_classifier_free_guidance and get_classifier_free_guidance_world_size() == 2
# For CFG distilled models, skip CFG parallel (cfg_factor=1, no negative branch)
cfg_parallel_ready = (
self.do_classifier_free_guidance
and not self.model.config.cfg_distilled
and get_classifier_free_guidance_world_size() == 2
)
# Define call parameters
device = self._execution_device
@@ -3177,7 +3201,11 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
attention_mask = attention_mask[s]
self._split_model_kwargs_for_cfg_parallel(model_kwargs, batch_size, cfg_rank)
else:
cfg_factor = 1 + self.do_classifier_free_guidance
# For CFG distilled models, cfg_factor is always 1 (CFG embedded in model)
if self.model.config.cfg_distilled:
cfg_factor = 1
else:
cfg_factor = 1 + self.do_classifier_free_guidance
cfg_rank = None
b, _, q_len1, seq_len = attention_mask.shape
@@ -3219,6 +3247,13 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
# Sequential CFG: double the batch
latent_model_input = torch.cat([latents] * cfg_factor)
if self.model.config.use_meanflow:
r = self.scheduler.get_timestep_r(t)
r_expand = r.repeat(latent_model_input.shape[0])
else:
r_expand = None
model_kwargs["timesteps_r"] = r_expand
t_expand = t.repeat(latent_model_input.shape[0])
# ---- TeaCache: decide whether to compute or reuse ----
@@ -3240,6 +3275,13 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
tc_prev_mod = cur_mod.detach()
if should_compute:
# Handle guidance for CFG distilled models
if self.model.config.cfg_distilled:
model_kwargs["guidance"] = torch.tensor(
[1000.0 * self._guidance_scale],
device=self.device,
dtype=torch.bfloat16,
).repeat(latent_model_input.shape[0])
model_inputs = self.model.prepare_inputs_for_generation(
input_ids,
images=latent_model_input,
@@ -3259,7 +3301,12 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
pred = tc_prev_pred
# Perform guidance
if cfg_parallel_ready:
# For CFG distilled models, guidance is already embedded in the model,
# so we skip the explicit CFG computation
if self.model.config.cfg_distilled:
# CFG distilled: guidance is handled internally via guidance_emb
pass
elif cfg_parallel_ready:
# CFG parallel: all_gather → all ranks combine locally (no broadcast needed)
gathered = cfg_group.all_gather(pred, separate_tensors=True)
pred = self.cfg_operator(gathered[0], gathered[1], self.guidance_scale, step=i)
@@ -75,6 +75,21 @@ _STEP_PROMPT_KV = "hunyuan_prompt_kv"
_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil"
def _config_flag(config: Any, name: str) -> bool:
"""Read an optional Hunyuan config flag with legacy-safe defaults."""
return bool(getattr(config, name, False))
class HunyuanImage3FlowMatchEulerDiscreteScheduler(FlowMatchEulerDiscreteScheduler):
def get_timestep_r(self, timestep: float | torch.FloatTensor):
if self.step_index is None:
self._init_step_index(timestep)
if self.step_index + 1 < len(self.timesteps):
return self.timesteps[self.step_index + 1]
return self.timesteps.new_zeros(())
def default(val, d):
return val if val is not None else d
@@ -264,6 +279,8 @@ def get_hunyuan_image_3_pre_process_func(od_config: OmniDiffusionConfig):
image_processor=image_processor,
generation_config=generation_config,
image_base_size=hf_config.image_base_size,
cfg_distilled=_config_flag(hf_config, "cfg_distilled"),
use_meanflow=_config_flag(hf_config, "use_meanflow"),
)
request.prepared_layout = prepared_layout
request.diffusion_kv_requests = request_layout_utils.build_hunyuan_diffusion_kv_requests(
@@ -398,6 +415,11 @@ class HunyuanImage3Pipeline(
out_norm=True,
)
self.time_embed_2 = TimestepEmbedder(hidden_size=self.hf_config.hidden_size)
# Distillation-specific embedding modules
if _config_flag(self.hf_config, "cfg_distilled"):
self.guidance_emb = TimestepEmbedder(hidden_size=self.hf_config.hidden_size)
if _config_flag(self.hf_config, "use_meanflow"):
self.timestep_r_emb = TimestepEmbedder(hidden_size=self.hf_config.hidden_size)
self.lm_head = nn.Linear(self.hf_config.hidden_size, self.hf_config.vocab_size, bias=False)
self.vllm_config = get_current_vllm_config()
self.post_init()
@@ -419,6 +441,8 @@ class HunyuanImage3Pipeline(
"model.ln_f",
"time_embed",
"time_embed_2",
"guidance_emb",
"timestep_r_emb",
"final_layer.model",
]
device = get_local_device()
@@ -428,11 +452,8 @@ class HunyuanImage3Pipeline(
if mod:
mod.to(device)
unexpected_keywords = [
"guidance_emb",
"timestep_r_emb",
]
skip_prefixes.extend(unexpected_keywords)
# Note: guidance_emb and timestep_r_emb are no longer skipped
# to support HunyuanImage-3.0-Distil and MeanFlow distilled models
loader = AutoWeightsLoader(
self,
skip_prefixes=skip_prefixes,
@@ -452,7 +473,7 @@ class HunyuanImage3Pipeline(
def pipeline(self):
if self._pipeline is None:
# shift hard code
self.scheduler = FlowMatchEulerDiscreteScheduler(
self.scheduler = HunyuanImage3FlowMatchEulerDiscreteScheduler(
num_train_timesteps=1000,
shift=self.generation_config.flow_shift,
use_dynamic_shifting=False,
@@ -902,6 +923,70 @@ class HunyuanImage3Pipeline(
return x
def instantiate_guidance_tokens(
self,
x: torch.Tensor,
guidance: BatchRaggedTensor | None = None,
guidance_scatter_index: torch.Tensor | None = None,
):
"""Instantiate guidance embedding tokens for CFG distilled models.
Args:
x: Input sequence tensor (batch_size, seq_len, n_embd)
guidance: Guidance scale values (as tensor, typically guidance_scale * 1000)
guidance_scatter_index: Index positions to scatter guidance embeddings
"""
if guidance is None or guidance_scatter_index is None:
return x
if not hasattr(self, "guidance_emb"):
return x
batch_size, seq_len, n_embd = x.shape
guidance_src = self.guidance_emb(guidance.reshape(-1)) # (bsz * n, n_embd)
x.scatter_(
dim=1,
index=guidance_scatter_index.unsqueeze(-1).repeat(1, 1, n_embd),
src=guidance_src.reshape(batch_size, -1, n_embd),
)
return x
def instantiate_timestep_r_tokens(
self,
x: torch.Tensor,
timesteps_r: BatchRaggedTensor | None = None,
timesteps_r_scatter_index: torch.Tensor | None = None,
):
"""Instantiate timestep_r embedding tokens for MeanFlow distilled models.
Args:
x: Input sequence tensor (batch_size, seq_len, n_embd)
timesteps_r: Timestep r values for MeanFlow
timesteps_r_scatter_index: Index positions to scatter embeddings
"""
if timesteps_r is None or timesteps_r_scatter_index is None:
return x
if not hasattr(self, "timestep_r_emb"):
return x
batch_size, seq_len, n_embd = x.shape
if isinstance(timesteps_r, list):
for i, timestep_r in enumerate(timesteps_r):
timestep_r_src = self.timestep_r_emb(timestep_r) # (n, n_embd)
x[i : i + 1].scatter_(
dim=1,
index=timesteps_r_scatter_index[i].unsqueeze(0).unsqueeze(-1).repeat(1, 1, n_embd),
src=timestep_r_src.reshape(1, -1, n_embd),
)
else:
timesteps_r_src = self.timestep_r_emb(timesteps_r.reshape(-1)) # (bsz * n, n_embd)
x.scatter_(
dim=1,
index=timesteps_r_scatter_index.unsqueeze(-1).repeat(1, 1, n_embd),
src=timesteps_r_src.reshape(batch_size, -1, n_embd),
)
return x
def instantiate_vit_image_tokens(
self,
x: torch.Tensor,
@@ -936,12 +1021,12 @@ class HunyuanImage3Pipeline(
return x
def ragged_final_layer(self, x, image_mask, timestep, token_h, token_w, first_step):
def ragged_final_layer(self, x, image_mask, timestep, token_h, token_w, first_step, num_special_tokens: int = None):
bsz, seq_len, n_embd = x.shape
if first_step:
image_output = x.masked_select(image_mask.unsqueeze(-1).bool()).reshape(bsz, -1, n_embd)
else:
image_output = x[:, 1:, :]
image_output = x[:, num_special_tokens:, :]
timestep_emb = self.time_embed_2(timestep)
pred = self.final_layer(image_output, timestep_emb, token_h, token_w)
return pred
@@ -1130,7 +1215,12 @@ class HunyuanImage3Pipeline(
batch_gen_image_info = [prepared_layout.generated_image_info]
else:
batch_gen_image_info = [
self.image_processor.build_image_info(image_size) for _ in range(batch_size)
self.image_processor.build_image_info(
image_size,
add_guidance_token=_config_flag(getattr(self, "hf_config", None), "cfg_distilled"),
add_timestep_r_token=_config_flag(getattr(self, "hf_config", None), "use_meanflow"),
)
for _ in range(batch_size)
]
if batch_cond_image_info is not None:
@@ -1146,7 +1236,11 @@ class HunyuanImage3Pipeline(
generator = [torch.Generator(self.device).manual_seed(seed) for seed in seeds]
# 3. apply chat template
# For CFG distilled models, cfg_factor is always 1 (CFG is embedded in the model)
# For non-distilled models, cfg_factor depends on guidance_scale
cfg_factor = {"gen_text": 1, "gen_image": 1 + int(guidance_scale > 1.0)}
if _config_flag(getattr(self, "hf_config", None), "cfg_distilled"):
cfg_factor["gen_image"] = 1
bot_task = kwargs.pop("bot_task", "auto")
if prepared_layout is None:
# Pull template options from generation_config only when the
@@ -1268,6 +1362,14 @@ class HunyuanImage3Pipeline(
if vit_kwargs is not None
else None,
cond_timestep_scatter_index=to_device(output.cond_timestep_scatter_index, device),
# for CFG distilled models
guidance_scatter_index=to_device(output.guidance_scatter_index, device)
if hasattr(output, "guidance_scatter_index") and output.guidance_scatter_index is not None
else None,
# for MeanFlow distilled models
timesteps_r_scatter_index=to_device(output.timesteps_r_scatter_index, device)
if hasattr(output, "timesteps_r_scatter_index") and output.timesteps_r_scatter_index is not None
else None,
# for inner usage
tokenizer_output=output,
batch_gen_image_info=batch_gen_image_info,
@@ -1350,9 +1452,16 @@ class HunyuanImage3Pipeline(
"cond_vit_image_mask": kwargs.get("cond_vit_image_mask"),
"vit_kwargs": kwargs.get("vit_kwargs"),
"cond_timestep_scatter_index": kwargs.get("cond_timestep_scatter_index"),
# For CFG distilled models
"guidance": kwargs.get("guidance"),
"guidance_scatter_index": kwargs.get("guidance_scatter_index"),
# For MeanFlow distilled models
"timesteps_r": kwargs.get("timesteps_r"),
"timesteps_r_scatter_index": kwargs.get("timesteps_r_scatter_index"),
"query_lens": kwargs.get("query_lens"),
"seq_lens": kwargs.get("seq_lens"),
"num_image_tokens": kwargs.get("num_image_tokens"),
"num_special_tokens": kwargs.get("num_special_tokens"),
"ar_kv_reuse_len": kwargs.get("ar_kv_reuse_len", 0),
"full_attn_spans": kwargs.get("full_attn_spans"),
}
@@ -1372,6 +1481,11 @@ class HunyuanImage3Pipeline(
"mode": mode,
"custom_pos_emb": model_kwargs["custom_pos_emb"],
"num_image_tokens": model_kwargs["num_image_tokens"],
"num_special_tokens": model_kwargs["num_special_tokens"],
"guidance": model_kwargs.get("guidance"),
"guidance_scatter_index": model_kwargs.get("guidance_scatter_index"),
"timesteps_r": model_kwargs.get("timesteps_r"),
"timesteps_r_scatter_index": model_kwargs.get("timesteps_r_scatter_index"),
}
if "full_attn_spans" in model_kwargs:
updated_model_kwargs["full_attn_spans"] = model_kwargs["full_attn_spans"]
@@ -1400,15 +1514,24 @@ class HunyuanImage3Pipeline(
bsz, seq_len = image_mask.shape
offset = model_kwargs.get("ar_kv_reuse_offset", 0) # should be an absolute position.
index = torch.arange(offset, offset + seq_len, device=image_mask.device).unsqueeze(0).repeat(bsz, 1)
position_ids = index.masked_select(image_mask.bool()).reshape(bsz, -1)
timestep_position_ids = index[
torch.arange(bsz), model_kwargs["gen_timestep_scatter_index"][:, -1]
].unsqueeze(-1)
updated_model_kwargs["position_ids"] = torch.cat([timestep_position_ids, position_ids], dim=1)
image_position_ids = index.masked_select(image_mask.bool()).reshape(bsz, -1)
special_position_ids = [
index[torch.arange(bsz), model_kwargs["gen_timestep_scatter_index"][:, -1]].unsqueeze(-1)
]
if model_kwargs.get("guidance_scatter_index") is not None:
special_position_ids.append(
index[torch.arange(bsz), model_kwargs["guidance_scatter_index"][:, -1]].unsqueeze(-1)
)
if model_kwargs.get("timesteps_r_scatter_index") is not None:
special_position_ids.append(
index[torch.arange(bsz), model_kwargs["timesteps_r_scatter_index"][:, -1]].unsqueeze(-1)
)
special_position_ids = torch.cat(special_position_ids, dim=1)
updated_model_kwargs["position_ids"] = torch.cat([special_position_ids, image_position_ids], dim=1)
# attention mask
mask_list = []
current_starts = timestep_position_ids.reshape(-1)
current_starts = special_position_ids[:, 0]
max_current_start = int(current_starts.max().item())
for attention_mask_i, position_ids_i, current_start_i in zip(
model_kwargs["attention_mask"], updated_model_kwargs["position_ids"], current_starts
@@ -1461,6 +1584,7 @@ class HunyuanImage3Pipeline(
image_info: ImageInfo = batch_gen_image_info[0]
kwargs["num_image_tokens"] = request_layout_utils.hunyuan_num_image_tokens(image_info)
kwargs["num_special_tokens"] = request_layout_utils.hunyuan_num_special_tokens(image_info)
# 50 and 5.0 hard code
results = self.pipeline(
batch_size=len(batch_gen_image_info),
@@ -1514,9 +1638,16 @@ class HunyuanImage3Pipeline(
query_lens: list[int] | None = None,
seq_lens: list[int] | None = None,
num_image_tokens: int | None = None,
num_special_tokens: int | None = None,
uncond_cfg_prefill: bool = False,
ar_kv_reuse_len: int = 0,
full_attn_spans: list[list[tuple[int, int]]] | None = None,
# for CFG distilled models
guidance: torch.Tensor | None = None,
guidance_scatter_index: torch.Tensor | None = None,
# for MeanFlow distilled models
timesteps_r: torch.Tensor | None = None,
timesteps_r_scatter_index: torch.Tensor | None = None,
) -> tuple | CausalMMOutputWithPast:
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
# Sanity Check of Inputs
@@ -1578,11 +1709,23 @@ class HunyuanImage3Pipeline(
inputs_embeds, images, timestep, image_mask
)
inputs_embeds = self.instantiate_timestep_tokens(inputs_embeds, timestep, gen_timestep_scatter_index)
# Instantiate guidance and timestep_r tokens for distilled models
inputs_embeds = self.instantiate_guidance_tokens(inputs_embeds, guidance, guidance_scatter_index)
inputs_embeds = self.instantiate_timestep_r_tokens(
inputs_embeds, timesteps_r, timesteps_r_scatter_index
)
else:
t_emb = self.time_embed(timestep)
image_emb, token_h, token_w = self.patch_embed(images, t_emb)
timestep_emb = self.timestep_emb(timestep).reshape(bsz, -1, n_embd)
inputs_embeds = torch.cat([timestep_emb, image_emb], dim=1)
extra_embeds = []
if hasattr(self, "guidance_emb") and guidance is not None:
guidance_emb = self.guidance_emb(guidance.reshape(-1)).reshape(bsz, -1, n_embd)
extra_embeds.append(guidance_emb)
if hasattr(self, "timestep_r_emb") and timesteps_r is not None:
timesteps_r_emb = self.timestep_r_emb(timesteps_r.reshape(-1)).reshape(bsz, -1, n_embd)
extra_embeds.append(timesteps_r_emb)
inputs_embeds = torch.cat([timestep_emb, *extra_embeds, image_emb], dim=1)
# Instantiate placeholder tokens: <timestep>, <img> for cond images
# Should only run once with kv-cache enabled.
@@ -1641,7 +1784,7 @@ class HunyuanImage3Pipeline(
)
hidden_states = hidden_states.reshape(bsz, seq_len, n_embd)
diffusion_prediction = self.ragged_final_layer(
hidden_states, image_mask, timestep, token_h, token_w, first_step
hidden_states, image_mask, timestep, token_h, token_w, first_step, num_special_tokens
)
if not return_dict:
@@ -1771,6 +1914,7 @@ class HunyuanImage3Pipeline(
target_height = int(image_info.image_height)
target_width = int(image_info.image_width)
model_kwargs["num_image_tokens"] = request_layout_utils.hunyuan_num_image_tokens(image_info)
model_kwargs["num_special_tokens"] = request_layout_utils.hunyuan_num_special_tokens(image_info)
timesteps, _ = retrieve_timesteps(
self.scheduler,
@@ -1819,13 +1963,14 @@ class HunyuanImage3Pipeline(
state.timesteps = timesteps
state.step_index = 0
state.scheduler = req_scheduler
state.do_true_cfg = guidance_scale > 1.0
cfg_factor = request_layout_utils.hunyuan_cfg_factor(image_info, guidance_scale)
state.do_true_cfg = cfg_factor > 1
state.extra = {
_STEP_MODEL_KWARGS: model_kwargs,
_STEP_INPUT_IDS: input_ids,
_STEP_GENERATOR: model_kwargs["generator"],
_STEP_GUIDANCE_SCALE: guidance_scale,
_STEP_CFG_FACTOR: 1 + int(guidance_scale > 1.0),
_STEP_CFG_FACTOR: cfg_factor,
_STEP_OUTPUT_SIZE: (target_height, target_width),
_STEP_COT_TEXT_LIST: cot_text_list,
_STEP_AR_KV: self._snapshot_injected_ar_kv(),
@@ -1990,6 +2135,21 @@ class HunyuanImage3Pipeline(
row_branches,
first_step,
)
if _config_flag(self.hf_config, "cfg_distilled"):
model_kwargs["guidance"] = torch.tensor(
[1000.0 * state.extra[_STEP_GUIDANCE_SCALE] for state in states] * cfg_factor,
device=latents.device,
dtype=torch.bfloat16,
)
if _config_flag(self.hf_config, "use_meanflow"):
model_kwargs["timesteps_r"] = torch.cat(
[
state.scheduler.get_timestep_r(state.current_timestep).reshape(1).to(device=latents.device)
for _ in range(cfg_factor)
for state in states
],
dim=0,
)
if first_step:
self._restore_injected_ar_kv(states, row_state_indexes, row_branches)
else:
@@ -50,6 +50,7 @@ def _image_info_to_payload(image_info: ImageInfo) -> dict[str, Any]:
"ratio_index": _to_python_scalar(image_info.ratio_index),
"add_timestep_token": image_info.add_timestep_token,
"add_guidance_token": image_info.add_guidance_token,
"add_timestep_r_token": image_info.add_timestep_r_token,
"use_front_boi_token": image_info.use_front_boi_token,
"add_image_shape_token": image_info.add_image_shape_token,
}
@@ -68,6 +69,7 @@ def _image_info_from_payload(payload: dict[str, Any]) -> ImageInfo:
ratio_index=payload.get("ratio_index"),
add_timestep_token=payload.get("add_timestep_token", True),
add_guidance_token=payload.get("add_guidance_token", False),
add_timestep_r_token=payload.get("add_timestep_r_token", False),
use_front_boi_token=payload.get("use_front_boi_token", True),
add_image_shape_token=payload.get("add_image_shape_token", True),
)
@@ -197,7 +199,21 @@ def resolve_hunyuan_guidance_scale(sampling: Any, default_scale: float = 5.0) ->
def hunyuan_num_image_tokens(image_info: ImageInfo) -> int:
"""Return the generated-image span overwritten on every denoise step."""
return int(image_info.image_token_length) + int(image_info.add_timestep_token) + int(image_info.add_guidance_token)
return int(image_info.image_token_length) + hunyuan_num_special_tokens(image_info)
def hunyuan_num_special_tokens(image_info: ImageInfo) -> int:
"""Return the generated-image prefix tokens emitted before latent tokens."""
return (
int(image_info.add_timestep_token) + int(image_info.add_guidance_token) + int(image_info.add_timestep_r_token)
)
def hunyuan_cfg_factor(image_info: ImageInfo, guidance_scale: float) -> int:
"""Return the execution branch count for standard or embedded CFG."""
return 1 if image_info.add_guidance_token else 1 + int(guidance_scale > 1.0)
def build_hunyuan_batch_rope_image_info(
@@ -242,6 +258,8 @@ def prepare_hunyuan_layout(
image_processor: HunyuanImage3ImageProcessor,
generation_config: GenerationConfig,
image_base_size: int,
cfg_distilled: bool = False,
use_meanflow: bool = False,
) -> HunyuanPreparedLayout:
"""Build the CPU token/image layout reused by Scheduler and Worker."""
@@ -261,7 +279,12 @@ def prepare_hunyuan_layout(
height = sampling.height or 1024
width = sampling.width or 1024
guidance_scale = resolve_hunyuan_guidance_scale(sampling)
generated_image_info = image_processor.build_image_info((height, width))
image_info_kwargs: dict[str, bool] = {}
if cfg_distilled:
image_info_kwargs["add_guidance_token"] = True
if use_meanflow:
image_info_kwargs["add_timestep_r_token"] = True
generated_image_info = image_processor.build_image_info((height, width), **image_info_kwargs)
result = tokenizer_wrapper.apply_chat_template(
batch_prompt=prompt,
mode="gen_image",
@@ -273,7 +296,7 @@ def prepare_hunyuan_layout(
bot_task=tokenizer_bot_task,
image_base_size=image_base_size,
sequence_template=getattr(generation_config, "sequence_template", "pretrain"),
cfg_factor=1 + int(guidance_scale > 1.0),
cfg_factor=hunyuan_cfg_factor(generated_image_info, guidance_scale),
drop_think=getattr(generation_config, "drop_think", False),
)
tokenizer_output = result["output"]
@@ -337,7 +360,10 @@ def build_hunyuan_diffusion_kv_requests(
"""Build one persistent Scheduler KV request per Hunyuan execution row."""
tokenizer_output = prepared_layout.tokenizer_output
cfg_factor = 1 + int(resolve_hunyuan_guidance_scale(request.sampling_params) > 1.0)
cfg_factor = hunyuan_cfg_factor(
prepared_layout.generated_image_info,
resolve_hunyuan_guidance_scale(request.sampling_params),
)
if prepared_layout.num_branches != cfg_factor:
raise ValueError(
"Hunyuan tokenizer sequence count does not match CFG execution: "