[BugFix] Fix load_weights error when loading HunyuanImage3.0 (#1598)
Signed-off-by: Semmer2 <semmer@live.cn>
This commit is contained in:
1 parent
e37a89ffcb
commit
1ca198eaff
2 files changed
+13
-43
No files matched your search
@@ -1,11 +1,9 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import glob
|
|
||||||
import inspect
|
import inspect
|
||||||
import logging
|
import logging
|
||||||
import math
|
import math
|
||||||
import os
|
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
@@ -22,7 +20,6 @@ from diffusers.utils.outputs import BaseOutput
|
|||||||
from diffusers.utils.torch_utils import randn_tensor
|
from diffusers.utils.torch_utils import randn_tensor
|
||||||
from einops import rearrange
|
from einops import rearrange
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from safetensors.torch import load_file
|
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from torchvision import transforms
|
from torchvision import transforms
|
||||||
from transformers import PretrainedConfig, Siglip2ImageProcessorFast
|
from transformers import PretrainedConfig, Siglip2ImageProcessorFast
|
||||||
@@ -357,14 +354,6 @@ def build_batch_2d_rope(
|
|||||||
return stacked_cos, stacked_sin
|
return stacked_cos, stacked_sin
|
||||||
|
|
||||||
|
|
||||||
def get_full_state_dict(model_path):
|
|
||||||
files = glob.glob(os.path.join(model_path, "*.safetensors"))
|
|
||||||
full_sd = {}
|
|
||||||
for f in files:
|
|
||||||
full_sd.update(load_file(f))
|
|
||||||
return full_sd
|
|
||||||
|
|
||||||
|
|
||||||
def rotate_half(x):
|
def rotate_half(x):
|
||||||
"""Rotates half the hidden dims of the input."""
|
"""Rotates half the hidden dims of the input."""
|
||||||
x1 = x[..., : x.shape[-1] // 2]
|
x1 = x[..., : x.shape[-1] // 2]
|
||||||
@@ -1744,7 +1733,6 @@ class HunyuanImage3Model(nn.Module):
|
|||||||
lora_vocab = (lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1)) if lora_config else 0
|
lora_vocab = (lora_config.lora_extra_vocab_size * (lora_config.max_loras or 1)) if lora_config else 0
|
||||||
self.vocab_size = config.vocab_size + lora_vocab
|
self.vocab_size = config.vocab_size + lora_vocab
|
||||||
self.org_vocab_size = config.vocab_size
|
self.org_vocab_size = config.vocab_size
|
||||||
self.wte = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
|
||||||
if get_pp_group().is_first_rank or (config.tie_word_embeddings and get_pp_group().is_last_rank):
|
if get_pp_group().is_first_rank or (config.tie_word_embeddings and get_pp_group().is_last_rank):
|
||||||
self.embed_tokens = VocabParallelEmbedding(
|
self.embed_tokens = VocabParallelEmbedding(
|
||||||
self.vocab_size,
|
self.vocab_size,
|
||||||
@@ -2047,7 +2035,7 @@ class HunyuanImage3Model(nn.Module):
|
|||||||
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
||||||
|
|
||||||
if inputs_embeds is None:
|
if inputs_embeds is None:
|
||||||
inputs_embeds = self.wte(input_ids)
|
inputs_embeds = self.embed_tokens(input_ids)
|
||||||
|
|
||||||
# embed positions
|
# embed positions
|
||||||
hidden_states = inputs_embeds
|
hidden_states = inputs_embeds
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ from .hunyuan_image_3_transformer import (
|
|||||||
UNetDown,
|
UNetDown,
|
||||||
UNetUp,
|
UNetUp,
|
||||||
build_batch_2d_rope,
|
build_batch_2d_rope,
|
||||||
get_full_state_dict,
|
|
||||||
real_batched_index_select,
|
real_batched_index_select,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -113,10 +112,10 @@ class HunyuanImage3Pipeline(HunyuanImage3PreTrainedModel, GenerationMixin):
|
|||||||
self.vllm_config = get_current_vllm_config()
|
self.vllm_config = get_current_vllm_config()
|
||||||
self.post_init()
|
self.post_init()
|
||||||
|
|
||||||
def pre_load(self):
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||||
tp_rank = get_tensor_model_parallel_rank()
|
skip_prefixes = ["lm_head."] if self.hf_config.tie_word_embeddings else []
|
||||||
state_dict = get_full_state_dict(self.od_config.model)
|
# List of unexpected keywords in weight names
|
||||||
non_layer_prefixes = [
|
non_model_layer_prefixes = [
|
||||||
"vae",
|
"vae",
|
||||||
"vision_model",
|
"vision_model",
|
||||||
"vision_aligner",
|
"vision_aligner",
|
||||||
@@ -129,32 +128,15 @@ class HunyuanImage3Pipeline(HunyuanImage3PreTrainedModel, GenerationMixin):
|
|||||||
"time_embed_2",
|
"time_embed_2",
|
||||||
"final_layer.model",
|
"final_layer.model",
|
||||||
]
|
]
|
||||||
filtered_sd = {}
|
tp_rank = get_tensor_model_parallel_rank()
|
||||||
for k, v in state_dict.items():
|
device_str = f"{self.model.device.type}:{tp_rank}"
|
||||||
if any(k.startswith(prefix) for prefix in non_layer_prefixes):
|
named_modules = dict(self.named_modules())
|
||||||
filtered_sd[k] = v
|
for prefix in non_model_layer_prefixes:
|
||||||
|
mod = named_modules.get(prefix)
|
||||||
|
if mod:
|
||||||
|
mod.to(device_str)
|
||||||
|
|
||||||
missing, unexpected = self.load_state_dict(filtered_sd, strict=False)
|
|
||||||
|
|
||||||
for prefix in non_layer_prefixes:
|
|
||||||
if hasattr(self, prefix.split(".")[0]):
|
|
||||||
module = dict(self.named_modules()).get(prefix)
|
|
||||||
if module:
|
|
||||||
module.to(f"{self.model.device.type}:{tp_rank}")
|
|
||||||
|
|
||||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
||||||
self.pre_load()
|
|
||||||
skip_prefixes = ["lm_head."] if self.hf_config.tie_word_embeddings else []
|
|
||||||
# List of unexpected keywords in weight names
|
|
||||||
unexpected_keywords = [
|
unexpected_keywords = [
|
||||||
"vae",
|
|
||||||
"vision_aligner",
|
|
||||||
"vision_model",
|
|
||||||
"final_layer",
|
|
||||||
"patch_embed",
|
|
||||||
"timestep_emb",
|
|
||||||
"time_embed",
|
|
||||||
"time_embed_2",
|
|
||||||
"guidance_emb",
|
"guidance_emb",
|
||||||
"timestep_r_emb",
|
"timestep_r_emb",
|
||||||
]
|
]
|
||||||
@@ -892,7 +874,7 @@ class HunyuanImage3Pipeline(HunyuanImage3PreTrainedModel, GenerationMixin):
|
|||||||
|
|
||||||
custom_pos_emb = self.get_pos_emb(custom_pos_emb, position_ids)
|
custom_pos_emb = self.get_pos_emb(custom_pos_emb, position_ids)
|
||||||
|
|
||||||
inputs_embeds = self.model.wte(input_ids)
|
inputs_embeds = self.model.embed_tokens(input_ids)
|
||||||
|
|
||||||
bsz, seq_len, n_embd = inputs_embeds.shape
|
bsz, seq_len, n_embd = inputs_embeds.shape
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user