Feat/Add HunyuanImage-3.0-Instruct ar part support: (#2713)

Signed-off-by: TaffyOfficial <2324465096@qq.com>
Co-authored-by: TaffyOfficial <2324465096@qq.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
authored and GitHub committed 2026-04-16 23:15:25 +08:00
1 parent 7d64a7c996
commit c3ca5daafb
10 files changed
+816 -82

No files matched your search

@@ -77,7 +77,9 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.utils.tensor_schema import TensorSchema
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from vllm_omni.model_executor.models.hunyuan_image3.autoencoder_kl_3d import AutoencoderKLConv3D
from vllm_omni.model_executor.models.hunyuan_image3.siglip2 import LightProjector, Siglip2VisionTransformer
@@ -175,8 +177,11 @@ class HunyuanModel(HunYuanModel):
return True
return False
skipped_unexpected: set[str] = set()
for name, loaded_weight in weights:
if contains_unexpected_keyword(name, unexpected_keywords):
skipped_unexpected.add(name)
continue
if "rotary_emb.inv_freq" in name:
@@ -362,6 +367,17 @@ class HunyuanModel(HunYuanModel):
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if skipped_unexpected:
logger.warning_once(
"Skipped %d weights matching unexpected_keywords "
"(e.g. vae, vision_model, patch_embed, timestep_emb). "
"If upstream renamed components, these may be silently "
"lost. Skipped names: %s",
len(skipped_unexpected),
sorted(skipped_unexpected)[:10],
)
return loaded_params
@@ -1149,6 +1165,8 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
HunyuanImage3Inputs: TypeAlias = HunyuanImage3PixelInputs
prefer_model_sampler = True
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -1199,6 +1217,10 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
else:
self.lm_head = PPMissingLayer()
# --- AR-stage components ---
# These are needed for image encoding in the AR stage.
# If a future text-only stage is added, gate on vllm_config.model_config.model_stage.
# vae
self.vae = AutoencoderKLConv3D.from_config(config.vae)
self.patch_embed = UNetDown(
@@ -1226,6 +1248,63 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
self._mrope_joint_img_sep_token_id = tokenizer.convert_tokens_to_ids("<joint_img_sep>")
self._mrope_max_num_patches = config.vit_processor.get("max_num_patches", 729)
# Special token IDs for logits processors (stage transitions).
# These mirror the official tokenization_hunyuan_image_3.py setup.
self._end_of_think_id = tokenizer.convert_tokens_to_ids("</think>")
self._recaption_id = tokenizer.convert_tokens_to_ids("<recaption>")
self._end_of_recaption_id = tokenizer.convert_tokens_to_ids("</recaption>")
self._answer_id = tokenizer.convert_tokens_to_ids("<answer>")
self._end_of_answer_id = tokenizer.convert_tokens_to_ids("</answer>")
image_base_size = getattr(config, "image_base_size", 1024)
self._size_token_id = tokenizer.convert_tokens_to_ids(f"<img_size_{image_base_size}>")
self._start_ratio_id = tokenizer.convert_tokens_to_ids("<img_ratio_0>")
self._end_ratio_id = tokenizer.convert_tokens_to_ids("<img_ratio_32>")
ratio_33 = tokenizer.convert_tokens_to_ids("<img_ratio_33>")
ratio_36 = tokenizer.convert_tokens_to_ids("<img_ratio_36>")
self._ratio_other_slices = [(ratio_33, ratio_36 + 1)]
# Build the full set of ratio token IDs for use as stop tokens.
self._all_ratio_ids = set(range(self._start_ratio_id, self._end_ratio_id + 1))
for s, e in self._ratio_other_slices:
self._all_ratio_ids.update(range(s, e))
# Determine mode: comprehension (I2T/T2T) vs generation (IT2I/T2I).
engine_output_type = getattr(vllm_config.model_config, "engine_output_type", None)
self._is_comprehension = engine_output_type in (None, "text")
# For comprehension mode, block image generation tokens but allow
# text structure tokens (<think>, <answer>, etc.) so the model can
# follow its natural generation pattern. Stop tokens in YAML will
# terminate at </answer> or EOS.
self._blocked_token_ids: set[int] = set()
if self._is_comprehension:
self._blocked_token_ids.update(
[
self._mrope_boi_token_id, # <boi>
self._mrope_eoi_token_id, # <eoi>
self._size_token_id, # <img_size_*>
]
)
self._blocked_token_ids.update(self._all_ratio_ids)
# For generation mode, build stage transition map.
# Official logic: </think> → [<recaption>],
# </recaption> → [<answer>, <boi>, <img_size_*>]
# After <img_size_*>, restrict vocab to ratio tokens only.
# Stage-transition forced sequences, keyed by trigger token.
self._stage_transitions: dict[int, list[int]] = {}
if not self._is_comprehension:
self._stage_transitions[self._end_of_think_id] = [
self._recaption_id,
]
self._stage_transitions[self._end_of_recaption_id] = [
self._answer_id,
self._mrope_boi_token_id,
self._size_token_id,
]
self._sampler: Sampler | None = None
self._eos_token_id: int = tokenizer.eos_token_id
self._replace_rotary_embeddings()
def _replace_rotary_embeddings(self):
@@ -1257,6 +1336,12 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
head_dim,
rope_theta,
)
if replaced == 0:
raise RuntimeError(
"HunyuanImage3: _replace_rotary_embeddings replaced 0 layers. "
"The custom interleaved 2D mRoPE is not active — model outputs "
"will be incorrect. Check that model.layers[*].self_attn.rotary_emb exists."
)
def _parse_and_validate_image_input(
self,
@@ -1274,6 +1359,10 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
if vit_pixel_values is None or vae_pixel_values is None:
return None
# Handle empty batch (e.g., during profiling with 0 images / T2T mode)
if vit_pixel_values.numel() == 0 or vae_pixel_values.numel() == 0:
return None
return HunyuanImage3PixelInputs(
type="pixel_values",
pixel_values={
@@ -1472,6 +1561,112 @@ class HunyuanImage3ForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
# ------------------------------------------------------------------
# Custom sampler — applies HunyuanImage3-specific logits processors
# before the standard sampling step.
#
# Comprehension (I2T / T2T):
# Block generation-specific special tokens so sampling can't
# accidentally produce <answer>, <boi>, ratio tokens, etc.
#
# Generation (IT2I / T2I think):
# 1. _StageTransitionLogitsProcessor — force token sequences at
# transition boundaries (</think> → <recaption>, etc.)
# 2. _ConditionalSliceVocabLogitsProcessor — after <img_size_*>,
# restrict vocab to ratio tokens only (greedy).
# ------------------------------------------------------------------
def sample(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> SamplerOutput | None:
if logits is None or logits.numel() == 0:
return None
if self._sampler is None:
self._sampler = Sampler()
min_score = torch.finfo(logits.dtype).min
assert logits.shape[0] == 1, f"HunyuanImage3 sampler requires max_num_seqs=1, got batch size {logits.shape[0]}"
for req_idx in range(logits.shape[0]):
decoded_tokens: list[int] = (
sampling_metadata.output_token_ids[req_idx] if req_idx < len(sampling_metadata.output_token_ids) else []
)
last_token = decoded_tokens[-1] if decoded_tokens else -1
if self._is_comprehension:
for tid in self._blocked_token_ids:
logits[req_idx, tid] = min_score
else:
forced = self._get_forced_token(decoded_tokens)
if forced is not None:
logits[req_idx].fill_(min_score)
logits[req_idx, forced] = 0
elif last_token == self._size_token_id:
self._apply_ratio_restriction(logits, req_idx, min_score)
elif last_token in self._all_ratio_ids:
logits[req_idx].fill_(min_score)
logits[req_idx, self._eos_token_id] = 0
return self._sampler(logits=logits, sampling_metadata=sampling_metadata)
def _get_forced_token(self, decoded_tokens: list[int]) -> int | None:
"""Derive the next forced token from output history (stateless).
Scans decoded_tokens backwards for the most recent trigger token,
then prefix-matches the forced sequence against what followed.
Returns the next token to force, or None if the sequence is complete
or history has diverged from the expected forced sequence.
"""
for i in range(len(decoded_tokens) - 1, -1, -1):
trigger = decoded_tokens[i]
if trigger not in self._stage_transitions:
continue
forced_seq = self._stage_transitions[trigger]
emitted = decoded_tokens[i + 1 :]
matched = 0
for expected, actual in zip(forced_seq, emitted):
if actual != expected:
# History diverged from the expected forced sequence.
# Stop applying transition forcing for safety.
return None
matched += 1
if matched < len(forced_seq):
return forced_seq[matched]
return None
return None
def _apply_ratio_restriction(
self,
logits: torch.Tensor,
req_idx: int,
min_score: float,
) -> None:
"""Port of official _ConditionalSliceVocabLogitsProcessor.__call__.
After the size token, only allow ratio tokens and pick greedily.
"""
original = logits[req_idx].clone()
logits[req_idx].fill_(min_score)
# Allow primary ratio range.
logits[req_idx, self._start_ratio_id : self._end_ratio_id + 1] = original[
self._start_ratio_id : self._end_ratio_id + 1
]
# Allow extra ratio slices.
for s, e in self._ratio_other_slices:
logits[req_idx, s:e] = original[s:e]
# Force greedy: keep only the argmax.
max_id = logits[req_idx].argmax().item()
logits[req_idx].fill_(min_score)
logits[req_idx, max_id] = 0
def make_empty_intermediate_tensors(
self, batch_size: int, dtype: torch.dtype, device: torch.device
) -> IntermediateTensors:
@@ -0,0 +1,44 @@
# Stage config for HunyuanImage-3.0 Image-to-Text (I2T / image understanding).
# Single LLM stage: AR model reads image + text prompt, generates text output.
stage_args:
- stage_id: 0
stage_type: llm
runtime:
process: true
devices: "0,1,2,3"
max_batch_size: 1
requires_multimodal_data: true
engine_args:
model_stage: AR
max_num_seqs: 1
model_arch: HunyuanImage3ForCausalMM
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.95
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
max_num_batched_tokens: 32768
tensor_parallel_size: 4
pipeline_parallel_size: 1
hf_overrides:
rope_parameters:
mrope_section: [0, 32, 32]
rope_type: default
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 0.95
top_k: 1024
max_tokens: 2048
stop_token_ids: [127957, 128026] # <|endoftext|>, </answer>
detokenize: True
runtime:
enabled: true
defaults:
window_size: -1
max_inflight: 1
@@ -0,0 +1,78 @@
# Stage config for HunyuanImage-3.0 Image+Text-to-Image (image editing).
# Stage 0: AR (HunyuanImage3ForConditionalGeneration) — reads (image, text), emits latent tokens
# Stage 1: Diffusion (HunyuanImage3Pipeline / DiT + VAE) — denoise + decode latents → image
stage_args:
# Stage 0: AR Model
- stage_id: 0
stage_type: llm
runtime:
process: true
devices: "0,1,2,3"
max_batch_size: 1
requires_multimodal_data: true # AR needs the original image
engine_args:
model_stage: AR
model_arch: HunyuanImage3ForCausalMM
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.95
enforce_eager: true
trust_remote_code: true
engine_output_type: latent # AR outputs latent for DiT
enable_prefix_caching: false
max_num_batched_tokens: 32768
tensor_parallel_size: 4
pipeline_parallel_size: 1
hf_overrides:
rope_parameters:
mrope_section: [0, 32, 32]
rope_type: default
is_comprehension: false # Generation task, not comprehension
final_output: false # AR is not the final output
default_sampling_params:
temperature: 0.6
top_p: 0.95
top_k: 1024
max_tokens: 4096
stop_token_ids: [127957] # <|endoftext|>
detokenize: false
# Stage 1: Diffusion (DiT + VAE)
# Receives latents from AR stage, performs denoising + VAE decode
- stage_id: 1
stage_type: diffusion
runtime:
process: true
devices: "4,5,6,7"
max_batch_size: 1
requires_multimodal_data: true # May need condition images
engine_args:
model_stage: dit
model_arch: HunyuanImage3ForCausalMM
enforce_eager: true
trust_remote_code: true
distributed_executor_backend: "mp"
parallel_config:
tensor_parallel_size: 4
enable_expert_parallel: true
omni_kv_config:
need_recv_cache: true
engine_input_source: [0] # Input from AR stage
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.hunyuan_image3.ar2diffusion
final_output: true
final_output_type: image
default_sampling_params:
num_inference_steps: 50
guidance_scale: 2.5
# Top-level runtime config
runtime:
enabled: true
defaults:
window_size: -1 # Trigger downstream only after full upstream completion
max_inflight: 1 # Process serially within each stage
edges:
- from: 0 # AR → Diffusion
to: 1
window_size: -1
@@ -1,81 +0,0 @@
# Stage config for running Hunyuan-Image3.0 for multi-stage omni runtime.
# Stage 0: AR Model (vLLM implementation)
# The following config has been verified on 8x L40S-48G GPU.
modes:
- mode: text-to-image
stages: [1]
- mode: image-to-text
stages: [0]
stage_args:
- stage_id: 0
stage_type: llm # Use llm stage type for AR stages
runtime:
process: true # Run this stage in a separate process
devices: "0,1,2,3,4,5,6,7" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
engine_args:
model_stage: AR
max_num_seqs: 1
model_arch: HunyuanImage3ForCausalMM
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.3
enforce_eager: true # Now we only support eager mode
trust_remote_code: true
engine_output_type: latent
enable_prefix_caching: false
max_num_batched_tokens: 32768
tensor_parallel_size: 8
pipeline_parallel_size: 1
hf_overrides:
rope_parameters:
mrope_section: [0, 32, 32]
rope_type: default
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 1.0
top_k: -1
max_tokens: 2048
seed: 42
detokenize: True
repetition_penalty: 1.1
- stage_id: 1
stage_type: diffusion
runtime:
process: true
devices: "0,1,2,3,4,5,6,7"
max_batch_size: 1
engine_args:
model_stage: diffusion
enforce_eager: true
distributed_executor_backend: "mp"
vae_use_slicing: false
vae_use_tiling: false
cache_backend: null
cache_config: null
enable_cache_dit_summary: false
parallel_config:
pipeline_parallel_size: 1
data_parallel_size: 1
tensor_parallel_size: 8
enable_expert_parallel: false
sequence_parallel_size: 1
ulysses_degree: 1
ring_degree: 1
cfg_parallel_size: 1
vae_patch_parallel_size: 1
use_hsdp: false
hsdp_shard_size: -1
hsdp_replicate_size: 1
final_output: true
final_output_type: image
# Top-level runtime config (concise): default windows and stage edges
runtime:
enabled: true
defaults:
window_size: -1 # Simplified: trigger downstream only after full upstream completion
max_inflight: 1 # Simplified: process serially within each stage
@@ -0,0 +1,45 @@
# Stage config for HunyuanImage-3.0 Text-to-Text (T2T / pure text generation).
# Single LLM stage: AR model reads text prompt only, generates text output.
# Sampling params aligned with official generation_config.json.
stage_args:
- stage_id: 0
stage_type: llm
runtime:
process: true
devices: "0,1,2,3"
max_batch_size: 1
requires_multimodal_data: false
engine_args:
model_stage: AR
max_num_seqs: 1
model_arch: HunyuanImage3ForCausalMM
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.95
enforce_eager: true
trust_remote_code: true
enable_prefix_caching: false
max_num_batched_tokens: 32768
tensor_parallel_size: 4
pipeline_parallel_size: 1
hf_overrides:
rope_parameters:
mrope_section: [0, 32, 32]
rope_type: default
is_comprehension: true
final_output: true
final_output_type: text
default_sampling_params:
temperature: 0.0
top_p: 0.95
top_k: 1024
max_tokens: 2048
stop_token_ids: [127957, 128026] # <|endoftext|>, </answer>
detokenize: True
runtime:
enabled: true
defaults:
window_size: -1
max_inflight: 1
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stage input processor for HunyuanImage3: AR → Diffusion transition.
In IT2I (image editing) mode:
- Stage 0 (AR) receives (image + edit instruction), generates CoT/latent tokens
- Stage 1 (DiT) receives the AR output + original image, denoises → edited image
The ar2diffusion function bridges these two stages, following the same
signature pattern as glm_image.ar2diffusion.
"""
from typing import Any
import torch
from vllm.inputs import TextPrompt
from vllm.logger import init_logger
from vllm_omni.inputs.data import OmniTokensPrompt
logger = init_logger(__name__)
def ar2diffusion(
stage_list: list[Any],
engine_input_source: list[int],
prompt: OmniTokensPrompt | TextPrompt | list | None = None,
requires_multimodal_data: bool = False,
) -> list[dict[str, Any]]:
"""Process AR stage outputs to create Diffusion stage inputs.
Args:
stage_list: List of stage clients (set by orchestrator).
engine_input_source: List of source stage IDs (from YAML).
prompt: Original user prompt (may contain multimodal data).
requires_multimodal_data: Whether to forward multimodal data.
Returns:
List of dicts, each consumable by the HunyuanImage3 diffusion pipeline.
"""
if not engine_input_source:
raise ValueError("engine_input_source cannot be empty")
source_stage_id = engine_input_source[0]
if source_stage_id >= len(stage_list):
raise IndexError(f"Invalid source stage_id: {source_stage_id}")
if stage_list[source_stage_id].engine_outputs is None:
raise RuntimeError(f"Stage {source_stage_id} has no outputs yet")
ar_outputs = stage_list[source_stage_id].engine_outputs
diffusion_inputs = []
# Normalize prompt to list
if not isinstance(prompt, list):
prompt = [prompt] if prompt is not None else [{}]
for i, ar_output in enumerate(ar_outputs):
output = ar_output.outputs[0]
generated_token_ids = output.token_ids
generated_text = getattr(output, "text", "") or ""
# Get original prompt info
original_prompt = prompt[i] if i < len(prompt) else {}
if isinstance(original_prompt, dict):
pass
elif hasattr(original_prompt, "_asdict"):
original_prompt = original_prompt._asdict()
elif hasattr(original_prompt, "__dict__"):
original_prompt = vars(original_prompt)
else:
original_prompt = {}
height = original_prompt.get("height", 1024)
width = original_prompt.get("width", 1024)
text_prompt = original_prompt.get("prompt", "")
logger.info(
"[ar2diffusion] Request %d: AR generated %d tokens, text length=%d, target size=%dx%d",
i,
len(generated_token_ids),
len(generated_text),
height,
width,
)
token_tensor = torch.tensor(generated_token_ids, dtype=torch.long)
diffusion_input: dict[str, Any] = {
"prompt": text_prompt,
"height": height,
"width": width,
"extra": {
"ar_token_ids": token_tensor,
"ar_generated_text": generated_text,
},
}
# Forward multimodal data (original image for IT2I conditioning)
mm_data = original_prompt.get("multi_modal_data")
if mm_data:
pil_image = mm_data.get("image")
if pil_image is None:
images = mm_data.get("images")
if images:
pil_image = images[0] if isinstance(images, list) else images
if pil_image is not None:
diffusion_input["pil_image"] = pil_image
# Forward multimodal output from AR (if any)
if hasattr(ar_output, "multimodal_output") and ar_output.multimodal_output:
mm_output = ar_output.multimodal_output
if isinstance(mm_output, dict):
diffusion_input["extra"]["ar_multimodal_output"] = mm_output
# Forward sampling params
for key in ["seed", "num_inference_steps", "guidance_scale", "negative_prompt"]:
if key in original_prompt:
diffusion_input[key] = original_prompt[key]
diffusion_inputs.append(diffusion_input)
return diffusion_inputs