[Feat] Add support and testcase for HunyuanImage-3.0-Instruct-Distil (#4048)
Signed-off-by: zengchuang <zengchuang3@huawei.com> Signed-off-by: zengchuang-hw <zengchuang1005@gmail.com>
This commit is contained in:
1 parent
4927fd5412
commit
b5e898f295
12 files changed
+751
-102
No files matched your search
@@ -22,6 +22,7 @@ class TokenizerEncodeOutput(BaseOutput):
|
||||
tokens: torch.Tensor | None = None
|
||||
timestep_scatter_index: torch.Tensor | None = None
|
||||
guidance_scatter_index: torch.Tensor | None = None
|
||||
timesteps_r_scatter_index: torch.Tensor | None = None # For MeanFlow distilled models
|
||||
text_slices: list[slice] | None = None
|
||||
gen_image_slices: list[slice] | None = None
|
||||
joint_image_slices: list[slice] | None = None
|
||||
@@ -196,6 +197,7 @@ class TokenizerWrapper:
|
||||
extra_token_pos,
|
||||
add_timestep_token=False,
|
||||
add_image_shape_token=False,
|
||||
add_timestep_r_token=False,
|
||||
base_size=None,
|
||||
ratio_idx=None,
|
||||
image_type=None,
|
||||
@@ -221,6 +223,10 @@ class TokenizerWrapper:
|
||||
token_seq.extend([self.special_token_map["<guidance>"]])
|
||||
extra_token_pos["guidance"].append(token_count)
|
||||
token_count += 1
|
||||
if add_timestep_r_token:
|
||||
token_seq.extend([self.special_token_map["<timestep_r>"]])
|
||||
extra_token_pos["gen_timestep_r"].append(token_count)
|
||||
token_count += 1
|
||||
return token_count
|
||||
|
||||
def encode_sequence(
|
||||
@@ -230,6 +236,7 @@ class TokenizerWrapper:
|
||||
total_length=None,
|
||||
add_timestep_token=False,
|
||||
add_guidance_token=False,
|
||||
add_timestep_r_token=False,
|
||||
last_key_only_prefix=False,
|
||||
add_eos=True,
|
||||
use_front_boi_token=True,
|
||||
@@ -329,6 +336,7 @@ class TokenizerWrapper:
|
||||
2
|
||||
+ (1 if source.get("timestep", add_timestep_token) else 0)
|
||||
+ (1 if source.get("guidance", add_guidance_token) else 0)
|
||||
+ (1 if source.get("timestep_r", add_timestep_r_token) else 0)
|
||||
+ (2 if source.get("image_shape", add_image_shape_token) else 0)
|
||||
)
|
||||
if drop_last is True and token_count + extra_count + source["length"] > total_length:
|
||||
@@ -344,6 +352,7 @@ class TokenizerWrapper:
|
||||
extra_token_pos=extra_token_pos,
|
||||
add_timestep_token=source.get("timestep", add_timestep_token),
|
||||
add_guidance_token=source.get("guidance", add_guidance_token),
|
||||
add_timestep_r_token=source.get("timestep_r", add_timestep_r_token),
|
||||
add_image_shape_token=source.get("image_shape", add_image_shape_token),
|
||||
base_size=source.get("base_size"),
|
||||
ratio_idx=source.get("ratio_idx"),
|
||||
@@ -739,6 +748,7 @@ class TokenizerWrapper:
|
||||
length=section["token_length"],
|
||||
timestep=section.get("add_timestep_token", False),
|
||||
guidance=section.get("add_guidance_token", False),
|
||||
timestep_r=section.get("add_timestep_r_token", False),
|
||||
front_boi=section.get("use_front_boi_token", False),
|
||||
image_shape=section.get("add_image_shape_token", False),
|
||||
base_size=section.get("base_size"),
|
||||
@@ -777,6 +787,11 @@ class TokenizerWrapper:
|
||||
guidance_scatter_index = (
|
||||
torch.tensor(extra_token_pos["guidance"], dtype=torch.long) if "guidance" in extra_token_pos else None
|
||||
)
|
||||
timesteps_r_scatter_index = (
|
||||
torch.tensor(extra_token_pos["gen_timestep_r"], dtype=torch.long)
|
||||
if "gen_timestep_r" in extra_token_pos
|
||||
else None
|
||||
)
|
||||
cond_timestep_scatter_index = (
|
||||
torch.tensor(extra_token_pos["cond_timestep"], dtype=torch.long)
|
||||
if "cond_timestep" in extra_token_pos
|
||||
@@ -847,6 +862,7 @@ class TokenizerWrapper:
|
||||
tokens=full_seq_token_tensor,
|
||||
timestep_scatter_index=timestep_scatter_index,
|
||||
guidance_scatter_index=guidance_scatter_index,
|
||||
timesteps_r_scatter_index=timesteps_r_scatter_index,
|
||||
text_slices=text_slices,
|
||||
gen_image_slices=gen_image_slices,
|
||||
joint_image_slices=joint_image_slices,
|
||||
|
||||
@@ -630,6 +630,7 @@ class ImageInfo:
|
||||
|
||||
self.add_timestep_token = kwargs.get("add_timestep_token", True)
|
||||
self.add_guidance_token = kwargs.get("add_guidance_token", False)
|
||||
self.add_timestep_r_token = kwargs.get("add_timestep_r_token", False)
|
||||
self.use_front_boi_token = kwargs.get("use_front_boi_token", True)
|
||||
self.add_image_shape_token = kwargs.get("add_image_shape_token", True)
|
||||
|
||||
@@ -667,6 +668,7 @@ class ImageInfo:
|
||||
token_length=self.image_token_length,
|
||||
add_timestep_token=self.add_timestep_token,
|
||||
add_guidance_token=self.add_guidance_token,
|
||||
add_timestep_r_token=self.add_timestep_r_token,
|
||||
use_front_boi_token=self.use_front_boi_token,
|
||||
add_image_shape_token=self.add_image_shape_token,
|
||||
base_size=self.base_size,
|
||||
@@ -1442,6 +1444,9 @@ class HunyuanImage3Config(PretrainedConfig):
|
||||
vit=None,
|
||||
vit_processor=None,
|
||||
vit_aligner=None,
|
||||
# Distillation flags
|
||||
cfg_distilled: bool = False,
|
||||
use_meanflow: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
@@ -1521,6 +1526,10 @@ class HunyuanImage3Config(PretrainedConfig):
|
||||
self.patch_embed_hidden_dim = patch_embed_hidden_dim
|
||||
self.image_base_size = image_base_size
|
||||
|
||||
# Distillation flags
|
||||
self.cfg_distilled = cfg_distilled
|
||||
self.use_meanflow = use_meanflow
|
||||
|
||||
# token id
|
||||
self.eod_token_id = eod_token_id
|
||||
self.im_start_id = im_start_id
|
||||
@@ -1555,7 +1564,7 @@ class HunyuanImage3ImageProcessor:
|
||||
)
|
||||
self.vision_encoder_processor = Siglip2ImageProcessorFast.from_dict(config.vit_processor)
|
||||
|
||||
def build_image_info(self, image_size):
|
||||
def build_image_info(self, image_size, add_guidance_token=False, add_timestep_r_token=False):
|
||||
# parse image size (HxW, H:W, or <img_ratio_i>)
|
||||
if isinstance(image_size, str):
|
||||
if image_size.startswith("<img_ratio_"):
|
||||
@@ -1592,6 +1601,8 @@ class HunyuanImage3ImageProcessor:
|
||||
token_height=token_height,
|
||||
base_size=base_size,
|
||||
ratio_index=ratio_idx,
|
||||
add_guidance_token=add_guidance_token,
|
||||
add_timestep_r_token=add_timestep_r_token,
|
||||
)
|
||||
return image_info
|
||||
|
||||
@@ -2244,6 +2255,8 @@ class HunyuanImage3Model(nn.Module):
|
||||
"timestep_emb",
|
||||
"time_embed",
|
||||
"time_embed_2",
|
||||
# These embeddings belong to the outer pipeline model rather than
|
||||
# this transformer core and are loaded by the outer weight loader.
|
||||
"guidance_emb",
|
||||
"timestep_r_emb",
|
||||
]
|
||||
@@ -2802,6 +2815,8 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
"position_ids",
|
||||
"image_mask",
|
||||
"gen_timestep_scatter_index",
|
||||
"guidance_scatter_index",
|
||||
"timesteps_r_scatter_index",
|
||||
"cond_vae_image_mask",
|
||||
"cond_vit_image_mask",
|
||||
"cond_timestep_scatter_index",
|
||||
@@ -2971,9 +2986,12 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
model_kwargs["attention_mask"] = model_kwargs["attention_mask"][:, :, positive_reuse_len:, :]
|
||||
model_kwargs["position_ids"] = model_kwargs["position_ids"][:, positive_reuse_len:]
|
||||
|
||||
# Shift image_mask and gen_timestep_scatter_index to match truncated sequence
|
||||
# Shift generated-image placeholder indices to match the truncated sequence.
|
||||
model_kwargs["image_mask"] = model_kwargs["image_mask"][:, positive_reuse_len:]
|
||||
model_kwargs["gen_timestep_scatter_index"] = model_kwargs["gen_timestep_scatter_index"] - positive_reuse_len
|
||||
for key in ("guidance_scatter_index", "timesteps_r_scatter_index"):
|
||||
if model_kwargs.get(key) is not None:
|
||||
model_kwargs[key] = model_kwargs[key] - positive_reuse_len
|
||||
model_kwargs["ar_kv_reuse_offset"] = positive_reuse_len
|
||||
|
||||
# cond-image have computed in ar, we may skip it by index in the future.
|
||||
@@ -3016,7 +3034,8 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
self.model.inject_ar_kv_into_layers(ar_kv_data, positive_reuse_len)
|
||||
|
||||
# 3. negative cfg prefill
|
||||
if self.do_classifier_free_guidance:
|
||||
# For CFG distilled models, skip negative CFG prefill (cfg_factor=1, no negative prompt)
|
||||
if self.do_classifier_free_guidance and not self.model.config.cfg_distilled:
|
||||
self._maybe_run_negative_cfg_prefill(
|
||||
input_ids=input_ids,
|
||||
model_kwargs=model_kwargs,
|
||||
@@ -3124,7 +3143,12 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
self._guidance_rescale = guidance_rescale
|
||||
|
||||
# Detect CFG parallel configuration (only 2-branch layout is supported)
|
||||
cfg_parallel_ready = self.do_classifier_free_guidance and get_classifier_free_guidance_world_size() == 2
|
||||
# For CFG distilled models, skip CFG parallel (cfg_factor=1, no negative branch)
|
||||
cfg_parallel_ready = (
|
||||
self.do_classifier_free_guidance
|
||||
and not self.model.config.cfg_distilled
|
||||
and get_classifier_free_guidance_world_size() == 2
|
||||
)
|
||||
|
||||
# Define call parameters
|
||||
device = self._execution_device
|
||||
@@ -3177,7 +3201,11 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
attention_mask = attention_mask[s]
|
||||
self._split_model_kwargs_for_cfg_parallel(model_kwargs, batch_size, cfg_rank)
|
||||
else:
|
||||
cfg_factor = 1 + self.do_classifier_free_guidance
|
||||
# For CFG distilled models, cfg_factor is always 1 (CFG embedded in model)
|
||||
if self.model.config.cfg_distilled:
|
||||
cfg_factor = 1
|
||||
else:
|
||||
cfg_factor = 1 + self.do_classifier_free_guidance
|
||||
cfg_rank = None
|
||||
|
||||
b, _, q_len1, seq_len = attention_mask.shape
|
||||
@@ -3219,6 +3247,13 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
# Sequential CFG: double the batch
|
||||
latent_model_input = torch.cat([latents] * cfg_factor)
|
||||
|
||||
if self.model.config.use_meanflow:
|
||||
r = self.scheduler.get_timestep_r(t)
|
||||
r_expand = r.repeat(latent_model_input.shape[0])
|
||||
else:
|
||||
r_expand = None
|
||||
model_kwargs["timesteps_r"] = r_expand
|
||||
|
||||
t_expand = t.repeat(latent_model_input.shape[0])
|
||||
|
||||
# ---- TeaCache: decide whether to compute or reuse ----
|
||||
@@ -3240,6 +3275,13 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
tc_prev_mod = cur_mod.detach()
|
||||
|
||||
if should_compute:
|
||||
# Handle guidance for CFG distilled models
|
||||
if self.model.config.cfg_distilled:
|
||||
model_kwargs["guidance"] = torch.tensor(
|
||||
[1000.0 * self._guidance_scale],
|
||||
device=self.device,
|
||||
dtype=torch.bfloat16,
|
||||
).repeat(latent_model_input.shape[0])
|
||||
model_inputs = self.model.prepare_inputs_for_generation(
|
||||
input_ids,
|
||||
images=latent_model_input,
|
||||
@@ -3259,7 +3301,12 @@ class HunyuanImage3Text2ImagePipeline(DiffusionPipeline):
|
||||
pred = tc_prev_pred
|
||||
|
||||
# Perform guidance
|
||||
if cfg_parallel_ready:
|
||||
# For CFG distilled models, guidance is already embedded in the model,
|
||||
# so we skip the explicit CFG computation
|
||||
if self.model.config.cfg_distilled:
|
||||
# CFG distilled: guidance is handled internally via guidance_emb
|
||||
pass
|
||||
elif cfg_parallel_ready:
|
||||
# CFG parallel: all_gather → all ranks combine locally (no broadcast needed)
|
||||
gathered = cfg_group.all_gather(pred, separate_tensors=True)
|
||||
pred = self.cfg_operator(gathered[0], gathered[1], self.guidance_scale, step=i)
|
||||
|
||||
@@ -75,6 +75,21 @@ _STEP_PROMPT_KV = "hunyuan_prompt_kv"
|
||||
_HUNYUAN_DEFAULT_OUTPUT_TYPE = "pil"
|
||||
|
||||
|
||||
def _config_flag(config: Any, name: str) -> bool:
|
||||
"""Read an optional Hunyuan config flag with legacy-safe defaults."""
|
||||
|
||||
return bool(getattr(config, name, False))
|
||||
|
||||
|
||||
class HunyuanImage3FlowMatchEulerDiscreteScheduler(FlowMatchEulerDiscreteScheduler):
|
||||
def get_timestep_r(self, timestep: float | torch.FloatTensor):
|
||||
if self.step_index is None:
|
||||
self._init_step_index(timestep)
|
||||
if self.step_index + 1 < len(self.timesteps):
|
||||
return self.timesteps[self.step_index + 1]
|
||||
return self.timesteps.new_zeros(())
|
||||
|
||||
|
||||
def default(val, d):
|
||||
return val if val is not None else d
|
||||
|
||||
@@ -264,6 +279,8 @@ def get_hunyuan_image_3_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
image_processor=image_processor,
|
||||
generation_config=generation_config,
|
||||
image_base_size=hf_config.image_base_size,
|
||||
cfg_distilled=_config_flag(hf_config, "cfg_distilled"),
|
||||
use_meanflow=_config_flag(hf_config, "use_meanflow"),
|
||||
)
|
||||
request.prepared_layout = prepared_layout
|
||||
request.diffusion_kv_requests = request_layout_utils.build_hunyuan_diffusion_kv_requests(
|
||||
@@ -398,6 +415,11 @@ class HunyuanImage3Pipeline(
|
||||
out_norm=True,
|
||||
)
|
||||
self.time_embed_2 = TimestepEmbedder(hidden_size=self.hf_config.hidden_size)
|
||||
# Distillation-specific embedding modules
|
||||
if _config_flag(self.hf_config, "cfg_distilled"):
|
||||
self.guidance_emb = TimestepEmbedder(hidden_size=self.hf_config.hidden_size)
|
||||
if _config_flag(self.hf_config, "use_meanflow"):
|
||||
self.timestep_r_emb = TimestepEmbedder(hidden_size=self.hf_config.hidden_size)
|
||||
self.lm_head = nn.Linear(self.hf_config.hidden_size, self.hf_config.vocab_size, bias=False)
|
||||
self.vllm_config = get_current_vllm_config()
|
||||
self.post_init()
|
||||
@@ -419,6 +441,8 @@ class HunyuanImage3Pipeline(
|
||||
"model.ln_f",
|
||||
"time_embed",
|
||||
"time_embed_2",
|
||||
"guidance_emb",
|
||||
"timestep_r_emb",
|
||||
"final_layer.model",
|
||||
]
|
||||
device = get_local_device()
|
||||
@@ -428,11 +452,8 @@ class HunyuanImage3Pipeline(
|
||||
if mod:
|
||||
mod.to(device)
|
||||
|
||||
unexpected_keywords = [
|
||||
"guidance_emb",
|
||||
"timestep_r_emb",
|
||||
]
|
||||
skip_prefixes.extend(unexpected_keywords)
|
||||
# Note: guidance_emb and timestep_r_emb are no longer skipped
|
||||
# to support HunyuanImage-3.0-Distil and MeanFlow distilled models
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=skip_prefixes,
|
||||
@@ -452,7 +473,7 @@ class HunyuanImage3Pipeline(
|
||||
def pipeline(self):
|
||||
if self._pipeline is None:
|
||||
# shift hard code
|
||||
self.scheduler = FlowMatchEulerDiscreteScheduler(
|
||||
self.scheduler = HunyuanImage3FlowMatchEulerDiscreteScheduler(
|
||||
num_train_timesteps=1000,
|
||||
shift=self.generation_config.flow_shift,
|
||||
use_dynamic_shifting=False,
|
||||
@@ -902,6 +923,70 @@ class HunyuanImage3Pipeline(
|
||||
|
||||
return x
|
||||
|
||||
def instantiate_guidance_tokens(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
guidance: BatchRaggedTensor | None = None,
|
||||
guidance_scatter_index: torch.Tensor | None = None,
|
||||
):
|
||||
"""Instantiate guidance embedding tokens for CFG distilled models.
|
||||
|
||||
Args:
|
||||
x: Input sequence tensor (batch_size, seq_len, n_embd)
|
||||
guidance: Guidance scale values (as tensor, typically guidance_scale * 1000)
|
||||
guidance_scatter_index: Index positions to scatter guidance embeddings
|
||||
"""
|
||||
if guidance is None or guidance_scatter_index is None:
|
||||
return x
|
||||
if not hasattr(self, "guidance_emb"):
|
||||
return x
|
||||
|
||||
batch_size, seq_len, n_embd = x.shape
|
||||
guidance_src = self.guidance_emb(guidance.reshape(-1)) # (bsz * n, n_embd)
|
||||
x.scatter_(
|
||||
dim=1,
|
||||
index=guidance_scatter_index.unsqueeze(-1).repeat(1, 1, n_embd),
|
||||
src=guidance_src.reshape(batch_size, -1, n_embd),
|
||||
)
|
||||
return x
|
||||
|
||||
def instantiate_timestep_r_tokens(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
timesteps_r: BatchRaggedTensor | None = None,
|
||||
timesteps_r_scatter_index: torch.Tensor | None = None,
|
||||
):
|
||||
"""Instantiate timestep_r embedding tokens for MeanFlow distilled models.
|
||||
|
||||
Args:
|
||||
x: Input sequence tensor (batch_size, seq_len, n_embd)
|
||||
timesteps_r: Timestep r values for MeanFlow
|
||||
timesteps_r_scatter_index: Index positions to scatter embeddings
|
||||
"""
|
||||
if timesteps_r is None or timesteps_r_scatter_index is None:
|
||||
return x
|
||||
if not hasattr(self, "timestep_r_emb"):
|
||||
return x
|
||||
|
||||
batch_size, seq_len, n_embd = x.shape
|
||||
|
||||
if isinstance(timesteps_r, list):
|
||||
for i, timestep_r in enumerate(timesteps_r):
|
||||
timestep_r_src = self.timestep_r_emb(timestep_r) # (n, n_embd)
|
||||
x[i : i + 1].scatter_(
|
||||
dim=1,
|
||||
index=timesteps_r_scatter_index[i].unsqueeze(0).unsqueeze(-1).repeat(1, 1, n_embd),
|
||||
src=timestep_r_src.reshape(1, -1, n_embd),
|
||||
)
|
||||
else:
|
||||
timesteps_r_src = self.timestep_r_emb(timesteps_r.reshape(-1)) # (bsz * n, n_embd)
|
||||
x.scatter_(
|
||||
dim=1,
|
||||
index=timesteps_r_scatter_index.unsqueeze(-1).repeat(1, 1, n_embd),
|
||||
src=timesteps_r_src.reshape(batch_size, -1, n_embd),
|
||||
)
|
||||
return x
|
||||
|
||||
def instantiate_vit_image_tokens(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
@@ -936,12 +1021,12 @@ class HunyuanImage3Pipeline(
|
||||
|
||||
return x
|
||||
|
||||
def ragged_final_layer(self, x, image_mask, timestep, token_h, token_w, first_step):
|
||||
def ragged_final_layer(self, x, image_mask, timestep, token_h, token_w, first_step, num_special_tokens: int = None):
|
||||
bsz, seq_len, n_embd = x.shape
|
||||
if first_step:
|
||||
image_output = x.masked_select(image_mask.unsqueeze(-1).bool()).reshape(bsz, -1, n_embd)
|
||||
else:
|
||||
image_output = x[:, 1:, :]
|
||||
image_output = x[:, num_special_tokens:, :]
|
||||
timestep_emb = self.time_embed_2(timestep)
|
||||
pred = self.final_layer(image_output, timestep_emb, token_h, token_w)
|
||||
return pred
|
||||
@@ -1130,7 +1215,12 @@ class HunyuanImage3Pipeline(
|
||||
batch_gen_image_info = [prepared_layout.generated_image_info]
|
||||
else:
|
||||
batch_gen_image_info = [
|
||||
self.image_processor.build_image_info(image_size) for _ in range(batch_size)
|
||||
self.image_processor.build_image_info(
|
||||
image_size,
|
||||
add_guidance_token=_config_flag(getattr(self, "hf_config", None), "cfg_distilled"),
|
||||
add_timestep_r_token=_config_flag(getattr(self, "hf_config", None), "use_meanflow"),
|
||||
)
|
||||
for _ in range(batch_size)
|
||||
]
|
||||
|
||||
if batch_cond_image_info is not None:
|
||||
@@ -1146,7 +1236,11 @@ class HunyuanImage3Pipeline(
|
||||
generator = [torch.Generator(self.device).manual_seed(seed) for seed in seeds]
|
||||
|
||||
# 3. apply chat template
|
||||
# For CFG distilled models, cfg_factor is always 1 (CFG is embedded in the model)
|
||||
# For non-distilled models, cfg_factor depends on guidance_scale
|
||||
cfg_factor = {"gen_text": 1, "gen_image": 1 + int(guidance_scale > 1.0)}
|
||||
if _config_flag(getattr(self, "hf_config", None), "cfg_distilled"):
|
||||
cfg_factor["gen_image"] = 1
|
||||
bot_task = kwargs.pop("bot_task", "auto")
|
||||
if prepared_layout is None:
|
||||
# Pull template options from generation_config only when the
|
||||
@@ -1268,6 +1362,14 @@ class HunyuanImage3Pipeline(
|
||||
if vit_kwargs is not None
|
||||
else None,
|
||||
cond_timestep_scatter_index=to_device(output.cond_timestep_scatter_index, device),
|
||||
# for CFG distilled models
|
||||
guidance_scatter_index=to_device(output.guidance_scatter_index, device)
|
||||
if hasattr(output, "guidance_scatter_index") and output.guidance_scatter_index is not None
|
||||
else None,
|
||||
# for MeanFlow distilled models
|
||||
timesteps_r_scatter_index=to_device(output.timesteps_r_scatter_index, device)
|
||||
if hasattr(output, "timesteps_r_scatter_index") and output.timesteps_r_scatter_index is not None
|
||||
else None,
|
||||
# for inner usage
|
||||
tokenizer_output=output,
|
||||
batch_gen_image_info=batch_gen_image_info,
|
||||
@@ -1350,9 +1452,16 @@ class HunyuanImage3Pipeline(
|
||||
"cond_vit_image_mask": kwargs.get("cond_vit_image_mask"),
|
||||
"vit_kwargs": kwargs.get("vit_kwargs"),
|
||||
"cond_timestep_scatter_index": kwargs.get("cond_timestep_scatter_index"),
|
||||
# For CFG distilled models
|
||||
"guidance": kwargs.get("guidance"),
|
||||
"guidance_scatter_index": kwargs.get("guidance_scatter_index"),
|
||||
# For MeanFlow distilled models
|
||||
"timesteps_r": kwargs.get("timesteps_r"),
|
||||
"timesteps_r_scatter_index": kwargs.get("timesteps_r_scatter_index"),
|
||||
"query_lens": kwargs.get("query_lens"),
|
||||
"seq_lens": kwargs.get("seq_lens"),
|
||||
"num_image_tokens": kwargs.get("num_image_tokens"),
|
||||
"num_special_tokens": kwargs.get("num_special_tokens"),
|
||||
"ar_kv_reuse_len": kwargs.get("ar_kv_reuse_len", 0),
|
||||
"full_attn_spans": kwargs.get("full_attn_spans"),
|
||||
}
|
||||
@@ -1372,6 +1481,11 @@ class HunyuanImage3Pipeline(
|
||||
"mode": mode,
|
||||
"custom_pos_emb": model_kwargs["custom_pos_emb"],
|
||||
"num_image_tokens": model_kwargs["num_image_tokens"],
|
||||
"num_special_tokens": model_kwargs["num_special_tokens"],
|
||||
"guidance": model_kwargs.get("guidance"),
|
||||
"guidance_scatter_index": model_kwargs.get("guidance_scatter_index"),
|
||||
"timesteps_r": model_kwargs.get("timesteps_r"),
|
||||
"timesteps_r_scatter_index": model_kwargs.get("timesteps_r_scatter_index"),
|
||||
}
|
||||
if "full_attn_spans" in model_kwargs:
|
||||
updated_model_kwargs["full_attn_spans"] = model_kwargs["full_attn_spans"]
|
||||
@@ -1400,15 +1514,24 @@ class HunyuanImage3Pipeline(
|
||||
bsz, seq_len = image_mask.shape
|
||||
offset = model_kwargs.get("ar_kv_reuse_offset", 0) # should be an absolute position.
|
||||
index = torch.arange(offset, offset + seq_len, device=image_mask.device).unsqueeze(0).repeat(bsz, 1)
|
||||
position_ids = index.masked_select(image_mask.bool()).reshape(bsz, -1)
|
||||
timestep_position_ids = index[
|
||||
torch.arange(bsz), model_kwargs["gen_timestep_scatter_index"][:, -1]
|
||||
].unsqueeze(-1)
|
||||
updated_model_kwargs["position_ids"] = torch.cat([timestep_position_ids, position_ids], dim=1)
|
||||
image_position_ids = index.masked_select(image_mask.bool()).reshape(bsz, -1)
|
||||
special_position_ids = [
|
||||
index[torch.arange(bsz), model_kwargs["gen_timestep_scatter_index"][:, -1]].unsqueeze(-1)
|
||||
]
|
||||
if model_kwargs.get("guidance_scatter_index") is not None:
|
||||
special_position_ids.append(
|
||||
index[torch.arange(bsz), model_kwargs["guidance_scatter_index"][:, -1]].unsqueeze(-1)
|
||||
)
|
||||
if model_kwargs.get("timesteps_r_scatter_index") is not None:
|
||||
special_position_ids.append(
|
||||
index[torch.arange(bsz), model_kwargs["timesteps_r_scatter_index"][:, -1]].unsqueeze(-1)
|
||||
)
|
||||
special_position_ids = torch.cat(special_position_ids, dim=1)
|
||||
updated_model_kwargs["position_ids"] = torch.cat([special_position_ids, image_position_ids], dim=1)
|
||||
|
||||
# attention mask
|
||||
mask_list = []
|
||||
current_starts = timestep_position_ids.reshape(-1)
|
||||
current_starts = special_position_ids[:, 0]
|
||||
max_current_start = int(current_starts.max().item())
|
||||
for attention_mask_i, position_ids_i, current_start_i in zip(
|
||||
model_kwargs["attention_mask"], updated_model_kwargs["position_ids"], current_starts
|
||||
@@ -1461,6 +1584,7 @@ class HunyuanImage3Pipeline(
|
||||
|
||||
image_info: ImageInfo = batch_gen_image_info[0]
|
||||
kwargs["num_image_tokens"] = request_layout_utils.hunyuan_num_image_tokens(image_info)
|
||||
kwargs["num_special_tokens"] = request_layout_utils.hunyuan_num_special_tokens(image_info)
|
||||
# 50 and 5.0 hard code
|
||||
results = self.pipeline(
|
||||
batch_size=len(batch_gen_image_info),
|
||||
@@ -1514,9 +1638,16 @@ class HunyuanImage3Pipeline(
|
||||
query_lens: list[int] | None = None,
|
||||
seq_lens: list[int] | None = None,
|
||||
num_image_tokens: int | None = None,
|
||||
num_special_tokens: int | None = None,
|
||||
uncond_cfg_prefill: bool = False,
|
||||
ar_kv_reuse_len: int = 0,
|
||||
full_attn_spans: list[list[tuple[int, int]]] | None = None,
|
||||
# for CFG distilled models
|
||||
guidance: torch.Tensor | None = None,
|
||||
guidance_scatter_index: torch.Tensor | None = None,
|
||||
# for MeanFlow distilled models
|
||||
timesteps_r: torch.Tensor | None = None,
|
||||
timesteps_r_scatter_index: torch.Tensor | None = None,
|
||||
) -> tuple | CausalMMOutputWithPast:
|
||||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||
# Sanity Check of Inputs
|
||||
@@ -1578,11 +1709,23 @@ class HunyuanImage3Pipeline(
|
||||
inputs_embeds, images, timestep, image_mask
|
||||
)
|
||||
inputs_embeds = self.instantiate_timestep_tokens(inputs_embeds, timestep, gen_timestep_scatter_index)
|
||||
# Instantiate guidance and timestep_r tokens for distilled models
|
||||
inputs_embeds = self.instantiate_guidance_tokens(inputs_embeds, guidance, guidance_scatter_index)
|
||||
inputs_embeds = self.instantiate_timestep_r_tokens(
|
||||
inputs_embeds, timesteps_r, timesteps_r_scatter_index
|
||||
)
|
||||
else:
|
||||
t_emb = self.time_embed(timestep)
|
||||
image_emb, token_h, token_w = self.patch_embed(images, t_emb)
|
||||
timestep_emb = self.timestep_emb(timestep).reshape(bsz, -1, n_embd)
|
||||
inputs_embeds = torch.cat([timestep_emb, image_emb], dim=1)
|
||||
extra_embeds = []
|
||||
if hasattr(self, "guidance_emb") and guidance is not None:
|
||||
guidance_emb = self.guidance_emb(guidance.reshape(-1)).reshape(bsz, -1, n_embd)
|
||||
extra_embeds.append(guidance_emb)
|
||||
if hasattr(self, "timestep_r_emb") and timesteps_r is not None:
|
||||
timesteps_r_emb = self.timestep_r_emb(timesteps_r.reshape(-1)).reshape(bsz, -1, n_embd)
|
||||
extra_embeds.append(timesteps_r_emb)
|
||||
inputs_embeds = torch.cat([timestep_emb, *extra_embeds, image_emb], dim=1)
|
||||
|
||||
# Instantiate placeholder tokens: <timestep>, <img> for cond images
|
||||
# Should only run once with kv-cache enabled.
|
||||
@@ -1641,7 +1784,7 @@ class HunyuanImage3Pipeline(
|
||||
)
|
||||
hidden_states = hidden_states.reshape(bsz, seq_len, n_embd)
|
||||
diffusion_prediction = self.ragged_final_layer(
|
||||
hidden_states, image_mask, timestep, token_h, token_w, first_step
|
||||
hidden_states, image_mask, timestep, token_h, token_w, first_step, num_special_tokens
|
||||
)
|
||||
|
||||
if not return_dict:
|
||||
@@ -1771,6 +1914,7 @@ class HunyuanImage3Pipeline(
|
||||
target_height = int(image_info.image_height)
|
||||
target_width = int(image_info.image_width)
|
||||
model_kwargs["num_image_tokens"] = request_layout_utils.hunyuan_num_image_tokens(image_info)
|
||||
model_kwargs["num_special_tokens"] = request_layout_utils.hunyuan_num_special_tokens(image_info)
|
||||
|
||||
timesteps, _ = retrieve_timesteps(
|
||||
self.scheduler,
|
||||
@@ -1819,13 +1963,14 @@ class HunyuanImage3Pipeline(
|
||||
state.timesteps = timesteps
|
||||
state.step_index = 0
|
||||
state.scheduler = req_scheduler
|
||||
state.do_true_cfg = guidance_scale > 1.0
|
||||
cfg_factor = request_layout_utils.hunyuan_cfg_factor(image_info, guidance_scale)
|
||||
state.do_true_cfg = cfg_factor > 1
|
||||
state.extra = {
|
||||
_STEP_MODEL_KWARGS: model_kwargs,
|
||||
_STEP_INPUT_IDS: input_ids,
|
||||
_STEP_GENERATOR: model_kwargs["generator"],
|
||||
_STEP_GUIDANCE_SCALE: guidance_scale,
|
||||
_STEP_CFG_FACTOR: 1 + int(guidance_scale > 1.0),
|
||||
_STEP_CFG_FACTOR: cfg_factor,
|
||||
_STEP_OUTPUT_SIZE: (target_height, target_width),
|
||||
_STEP_COT_TEXT_LIST: cot_text_list,
|
||||
_STEP_AR_KV: self._snapshot_injected_ar_kv(),
|
||||
@@ -1990,6 +2135,21 @@ class HunyuanImage3Pipeline(
|
||||
row_branches,
|
||||
first_step,
|
||||
)
|
||||
if _config_flag(self.hf_config, "cfg_distilled"):
|
||||
model_kwargs["guidance"] = torch.tensor(
|
||||
[1000.0 * state.extra[_STEP_GUIDANCE_SCALE] for state in states] * cfg_factor,
|
||||
device=latents.device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if _config_flag(self.hf_config, "use_meanflow"):
|
||||
model_kwargs["timesteps_r"] = torch.cat(
|
||||
[
|
||||
state.scheduler.get_timestep_r(state.current_timestep).reshape(1).to(device=latents.device)
|
||||
for _ in range(cfg_factor)
|
||||
for state in states
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
if first_step:
|
||||
self._restore_injected_ar_kv(states, row_state_indexes, row_branches)
|
||||
else:
|
||||
|
||||
@@ -50,6 +50,7 @@ def _image_info_to_payload(image_info: ImageInfo) -> dict[str, Any]:
|
||||
"ratio_index": _to_python_scalar(image_info.ratio_index),
|
||||
"add_timestep_token": image_info.add_timestep_token,
|
||||
"add_guidance_token": image_info.add_guidance_token,
|
||||
"add_timestep_r_token": image_info.add_timestep_r_token,
|
||||
"use_front_boi_token": image_info.use_front_boi_token,
|
||||
"add_image_shape_token": image_info.add_image_shape_token,
|
||||
}
|
||||
@@ -68,6 +69,7 @@ def _image_info_from_payload(payload: dict[str, Any]) -> ImageInfo:
|
||||
ratio_index=payload.get("ratio_index"),
|
||||
add_timestep_token=payload.get("add_timestep_token", True),
|
||||
add_guidance_token=payload.get("add_guidance_token", False),
|
||||
add_timestep_r_token=payload.get("add_timestep_r_token", False),
|
||||
use_front_boi_token=payload.get("use_front_boi_token", True),
|
||||
add_image_shape_token=payload.get("add_image_shape_token", True),
|
||||
)
|
||||
@@ -197,7 +199,21 @@ def resolve_hunyuan_guidance_scale(sampling: Any, default_scale: float = 5.0) ->
|
||||
def hunyuan_num_image_tokens(image_info: ImageInfo) -> int:
|
||||
"""Return the generated-image span overwritten on every denoise step."""
|
||||
|
||||
return int(image_info.image_token_length) + int(image_info.add_timestep_token) + int(image_info.add_guidance_token)
|
||||
return int(image_info.image_token_length) + hunyuan_num_special_tokens(image_info)
|
||||
|
||||
|
||||
def hunyuan_num_special_tokens(image_info: ImageInfo) -> int:
|
||||
"""Return the generated-image prefix tokens emitted before latent tokens."""
|
||||
|
||||
return (
|
||||
int(image_info.add_timestep_token) + int(image_info.add_guidance_token) + int(image_info.add_timestep_r_token)
|
||||
)
|
||||
|
||||
|
||||
def hunyuan_cfg_factor(image_info: ImageInfo, guidance_scale: float) -> int:
|
||||
"""Return the execution branch count for standard or embedded CFG."""
|
||||
|
||||
return 1 if image_info.add_guidance_token else 1 + int(guidance_scale > 1.0)
|
||||
|
||||
|
||||
def build_hunyuan_batch_rope_image_info(
|
||||
@@ -242,6 +258,8 @@ def prepare_hunyuan_layout(
|
||||
image_processor: HunyuanImage3ImageProcessor,
|
||||
generation_config: GenerationConfig,
|
||||
image_base_size: int,
|
||||
cfg_distilled: bool = False,
|
||||
use_meanflow: bool = False,
|
||||
) -> HunyuanPreparedLayout:
|
||||
"""Build the CPU token/image layout reused by Scheduler and Worker."""
|
||||
|
||||
@@ -261,7 +279,12 @@ def prepare_hunyuan_layout(
|
||||
height = sampling.height or 1024
|
||||
width = sampling.width or 1024
|
||||
guidance_scale = resolve_hunyuan_guidance_scale(sampling)
|
||||
generated_image_info = image_processor.build_image_info((height, width))
|
||||
image_info_kwargs: dict[str, bool] = {}
|
||||
if cfg_distilled:
|
||||
image_info_kwargs["add_guidance_token"] = True
|
||||
if use_meanflow:
|
||||
image_info_kwargs["add_timestep_r_token"] = True
|
||||
generated_image_info = image_processor.build_image_info((height, width), **image_info_kwargs)
|
||||
result = tokenizer_wrapper.apply_chat_template(
|
||||
batch_prompt=prompt,
|
||||
mode="gen_image",
|
||||
@@ -273,7 +296,7 @@ def prepare_hunyuan_layout(
|
||||
bot_task=tokenizer_bot_task,
|
||||
image_base_size=image_base_size,
|
||||
sequence_template=getattr(generation_config, "sequence_template", "pretrain"),
|
||||
cfg_factor=1 + int(guidance_scale > 1.0),
|
||||
cfg_factor=hunyuan_cfg_factor(generated_image_info, guidance_scale),
|
||||
drop_think=getattr(generation_config, "drop_think", False),
|
||||
)
|
||||
tokenizer_output = result["output"]
|
||||
@@ -337,7 +360,10 @@ def build_hunyuan_diffusion_kv_requests(
|
||||
"""Build one persistent Scheduler KV request per Hunyuan execution row."""
|
||||
|
||||
tokenizer_output = prepared_layout.tokenizer_output
|
||||
cfg_factor = 1 + int(resolve_hunyuan_guidance_scale(request.sampling_params) > 1.0)
|
||||
cfg_factor = hunyuan_cfg_factor(
|
||||
prepared_layout.generated_image_info,
|
||||
resolve_hunyuan_guidance_scale(request.sampling_params),
|
||||
)
|
||||
if prepared_layout.num_branches != cfg_factor:
|
||||
raise ValueError(
|
||||
"Hunyuan tokenizer sequence count does not match CFG execution: "
|
||||
|
||||
Reference in new issue
Block a user