[Feature] HunyuanImage-3.0 IT2I (image editing) support (#3107)
Signed-off-by: TaffyOfficial <2324465096@qq.com> Signed-off-by: zuiho <2324465096@qq.com> Signed-off-by: skf1999 <13234016272@163.com> Co-authored-by: TaffyOfficial <2324465096@qq.com> Co-authored-by: dengyunyang <584797741@qq.com> Co-authored-by: John Liu BUAA <liukecheng97@gmail.com>
This commit is contained in:
8 files changed
+573
-28
No files matched your search
@@ -31,6 +31,14 @@ python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
|
||||
--prompts "A cute cat sitting on a windowsill watching the sunset"
|
||||
```
|
||||
|
||||
**With VAE tiling (required on A100 GPUs):**
|
||||
```bash
|
||||
python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
|
||||
--modality text2img \
|
||||
--prompts "A cute cat sitting on a windowsill watching the sunset" \
|
||||
--vae-use-tiling
|
||||
```
|
||||
|
||||
#### Image to Image (img2img)
|
||||
|
||||
- **Pipeline**: Image + Text → AR (CoT + recaption + latent) → DiT → Edited Image
|
||||
@@ -103,6 +111,7 @@ python end2end.py --modality text2img \
|
||||
| `--stage-configs-path` | string | auto | Custom stage config YAML path |
|
||||
| `--enforce-eager` | flag | `False` | Disable torch.compile |
|
||||
| `--init-timeout` | int | `300` | Initialization timeout (seconds) |
|
||||
| `--vae-use-tiling` | flag | `False` | Enable VAE tiling for memory optimization (required to avoid OOM on A100) |
|
||||
|
||||
------
|
||||
|
||||
@@ -153,7 +162,7 @@ helper handles segment-by-segment tokenization (matches HF `apply_chat_template`
|
||||
|
||||
## FAQ
|
||||
|
||||
- **OOM errors**: Decrease `gpu_memory_utilization` in the YAML stage config, or use a smaller `max_num_batched_tokens`.
|
||||
- **OOM errors**: Decrease `gpu_memory_utilization` in the YAML stage config, use a smaller `max_num_batched_tokens`, or enable VAE tiling with `--vae-use-tiling` (required on A100 GPUs).
|
||||
- **Custom image sizes**: Use `--height` and `--width` flags (multiples of 16 recommended).
|
||||
|
||||
| Stage | VRAM (approx) |
|
||||
|
||||
@@ -22,6 +22,17 @@ from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniPromptType
|
||||
|
||||
# task -> (sys_type, bot_task, trigger_tag)
|
||||
_TASK_PRESETS: dict[str, tuple[str, str | None, str | None]] = {
|
||||
"t2t": ("en_unified", None, None),
|
||||
"i2t": ("en_unified", None, None),
|
||||
"it2i_think": ("en_unified", "think", "<think>"),
|
||||
"it2i_recaption": ("en_unified", "recaption", "<recaption>"),
|
||||
"t2i_think": ("en_unified", "think", "<think>"),
|
||||
"t2i_recaption": ("en_unified", "recaption", "<recaption>"),
|
||||
"t2i_vanilla": ("en_vanilla", "image", None),
|
||||
}
|
||||
|
||||
# Modality → prompt_utils task mapping
|
||||
_MODALITY_TASK_MAP = {
|
||||
"text2img": "t2i_think",
|
||||
@@ -73,6 +84,11 @@ def parse_args():
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--height", type=int, default=1024, help="Output image height.")
|
||||
parser.add_argument("--width", type=int, default=1024, help="Output image width.")
|
||||
parser.add_argument(
|
||||
"--vae-use-tiling",
|
||||
action="store_true",
|
||||
help="Enable VAE tiling for memory optimization.",
|
||||
)
|
||||
|
||||
# Prompt configuration
|
||||
parser.add_argument(
|
||||
@@ -113,6 +129,7 @@ def main():
|
||||
# Build Omni
|
||||
omni_kwargs = {
|
||||
"model": args.model,
|
||||
"vae_use_tiling": args.vae_use_tiling,
|
||||
"stage_configs_path": stage_configs_path,
|
||||
"log_stats": args.log_stats,
|
||||
"init_timeout": args.init_timeout,
|
||||
@@ -148,8 +165,18 @@ def main():
|
||||
formatted_prompts: list[OmniPromptType] = []
|
||||
for p in prompts:
|
||||
token_ids = build_prompt_tokens(p, tokenizer, task=task, sys_type=args.sys_type)
|
||||
preset_sys_type, _, _ = _TASK_PRESETS[task]
|
||||
effective_sys_type = args.sys_type or preset_sys_type
|
||||
|
||||
prompt_dict: dict = {"prompt_token_ids": token_ids, "prompt": p}
|
||||
# `prompt_token_ids` drives the AR stage (matches HF byte-for-byte).
|
||||
# `prompt` and `use_system_prompt` are forwarded by ar2diffusion to
|
||||
# the DiT stage so the diffusion pipeline can rebuild the same
|
||||
# system prefix when constructing its model inputs.
|
||||
prompt_dict: dict = {
|
||||
"prompt_token_ids": token_ids,
|
||||
"prompt": p,
|
||||
"use_system_prompt": effective_sys_type,
|
||||
}
|
||||
|
||||
if args.modality == "text2img":
|
||||
prompt_dict["modalities"] = ["image"]
|
||||
|
||||
Reference in new issue
Block a user