[Diffusion] Refactor diffusion models weights loading (#157)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py authored and GitHub committed 2025-12-06 11:12:13 +00:00
1 parent 08fb978c46
commit 059f45e48d
9 files changed
+346 -217

No files matched your search

@@ -6,6 +6,7 @@ import json
import logging
import math
import os
from collections.abc import Iterable
from typing import Any, Optional, Union
import numpy as np
@@ -20,9 +21,11 @@ from diffusers.schedulers.scheduling_flow_match_euler_discrete import (
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (
QwenImageTransformer2DModel,
)
@@ -240,9 +243,18 @@ class QwenImagePipeline(
):
super().__init__()
self.od_config = od_config
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer",
revision=None,
prefix="transformer.",
fall_back_to_pt=True,
)
]
self.device = get_local_device()
model = od_config.model
# Check if model is a local path
local_files_only = os.path.exists(model)
@@ -731,52 +743,6 @@ class QwenImagePipeline(
return DiffusionOutput(output=image)
def load_weights(self):
self.load_transformer()
# TODO: find a better way to load
def load_transformer(self):
import glob
import os
# Define the weight iterator
def weight_iterator(transformer_path):
if not os.path.exists(transformer_path):
logger.warning(f"Path {transformer_path} does not exist.")
return
# Look for safetensors first
safetensors_files = glob.glob(os.path.join(transformer_path, "*.safetensors"))
if safetensors_files:
try:
from safetensors.torch import load_file
except ImportError:
logger.warning("safetensors not installed, cannot load .safetensors files.")
return
for file_path in safetensors_files:
state_dict = load_file(file_path)
for name, tensor in state_dict.items():
yield name, tensor
else:
# Fallback to bin
bin_files = glob.glob(os.path.join(transformer_path, "*.bin"))
for file_path in bin_files:
state_dict = torch.load(file_path)
for name, tensor in state_dict.items():
yield name, tensor
try:
# Get model path from config or download from HF
model_name = self.od_config.model if hasattr(self, "od_config") else "Qwen/Qwen-Image"
if os.path.exists(model_name):
model_path = model_name
else:
model_path = download_weights_from_hf_specific(model_name, None, ["*"])
transformer_path = os.path.join(model_path, "transformer")
self.transformer.load_weights(weight_iterator(transformer_path))
except Exception as e:
print(f"An error occurred: {e}")
raise e
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -16,6 +16,7 @@ from diffusers.models.normalization import AdaLayerNormContinuous
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
@@ -603,79 +604,39 @@ class QwenImageTransformer2DModel(nn.Module):
return Transformer2DModelOutput(sample=output)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
# Collect incoming weights into a dict for flexible mapping and grouping
weights_map: dict[str, torch.Tensor] = {name: tensor for name, tensor in weights}
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
(
"to_qkv",
[
{"aliases": ["q_proj", "to_q", "q", "query"], "order": 0},
{"aliases": ["k_proj", "to_k", "k", "key"], "order": 1},
{"aliases": ["v_proj", "to_v", "v", "value"], "order": 2},
],
),
(
"add_kv_proj",
[
{"aliases": ["add_q_proj", "add_q"], "order": 0},
{"aliases": ["add_k_proj", "add_k"], "order": 1},
{"aliases": ["add_v_proj", "add_v"], "order": 2},
],
),
# (param_name, shard_name, shard_id)
# self-attn
(".to_qkv", ".to_q", "q"),
(".to_qkv", ".to_k", "k"),
(".to_qkv", ".to_v", "v"),
# cross-attn
(".add_kv_proj", ".add_q_proj", "q"),
(".add_kv_proj", ".add_k_proj", "k"),
(".add_kv_proj", ".add_v_proj", "v"),
]
# Helper that attempts to gather shards per base key using aliases and merges them
for target, shards in stacked_params_mapping:
base_candidates: dict[str, dict[str, dict[int, tuple[str, torch.Tensor]]]] = {}
for key in list(weights_map.keys()):
for shard in shards:
for alias in shard["aliases"]:
weight_suffix = f".{alias}.weight"
bias_suffix = f".{alias}.bias"
if key.endswith(weight_suffix):
base = key[: -len(weight_suffix)]
base_entry = base_candidates.setdefault(base, {"weight": {}, "bias": {}})
base_entry["weight"][shard["order"]] = (
key,
weights_map[key],
)
break
if key.endswith(bias_suffix):
base = key[: -len(bias_suffix)]
base_entry = base_candidates.setdefault(base, {"weight": {}, "bias": {}})
base_entry["bias"][shard["order"]] = (key, weights_map[key])
break
for base, entry in base_candidates.items():
if len(entry["weight"]) == len(shards):
ordered = [entry["weight"][order] for order in sorted(entry["weight"])]
combined = torch.cat([tensor for _, tensor in ordered], dim=0)
for key, _ in ordered:
weights_map.pop(key, None)
weights_map[base + f".{target}.weight"] = combined
if len(entry["bias"]) == len(shards):
ordered_bias = [entry["bias"][order] for order in sorted(entry["bias"])]
bcombined = torch.cat([tensor for _, tensor in ordered_bias], dim=0)
for key, _ in ordered_bias:
weights_map.pop(key, None)
weights_map[base + f".{target}.bias"] = bcombined
# Now perform the parameter copy using the possibly-updated weights_map
params_dict = dict(self.named_parameters())
for name, loaded_weight in weights_map.items():
# Handle potential name mismatches for FeedForward layers
if name.endswith(".proj.weight") and name not in params_dict:
mapped_name = name.replace(".proj.weight", ".weight")
elif name.endswith(".proj.bias") and name not in params_dict:
mapped_name = name.replace(".proj.bias", ".bias")
else:
mapped_name = name
if mapped_name in params_dict:
param = params_dict[mapped_name]
with torch.no_grad():
param.copy_(loaded_weight)
# we need to load the buffers for beta and eps (XIELU)
for name, buffer in self.named_buffers():
if name.endswith(".beta") or name.endswith(".eps"):
params_dict[name] = buffer
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
logger.warning(f"Weight {name} (mapped to {mapped_name}) not found in model params.")
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -18,6 +18,7 @@
import inspect
import json
import os
from collections.abc import Iterable
from typing import Any, Callable
import torch
@@ -28,9 +29,11 @@ from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
from diffusers.utils import logging
from diffusers.utils.torch_utils import randn_tensor
from transformers import AutoModel, AutoTokenizer
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.z_image.z_image_transformer import (
ZImageTransformer2DModel,
)
@@ -148,6 +151,15 @@ class ZImagePipeline(nn.Module):
):
super().__init__()
self.od_config = od_config
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer",
revision=None,
prefix="transformer.",
fall_back_to_pt=True,
)
]
self._execution_device = get_local_device()
model = od_config.model
local_files_only = os.path.exists(model)
@@ -598,51 +610,6 @@ class ZImagePipeline(nn.Module):
return DiffusionOutput(output=image)
def load_weights(self):
self.load_transformer()
def load_transformer(self):
import glob
import os
# Define the weight iterator
def weight_iterator(transformer_path):
if not os.path.exists(transformer_path):
logger.warning(f"Path {transformer_path} does not exist.")
return
# Look for safetensors first
safetensors_files = glob.glob(os.path.join(transformer_path, "*.safetensors"))
if safetensors_files:
try:
from safetensors.torch import load_file
except ImportError:
logger.warning("safetensors not installed, cannot load .safetensors files.")
return
for file_path in safetensors_files:
state_dict = load_file(file_path)
for name, tensor in state_dict.items():
yield name, tensor
else:
# Fallback to bin
bin_files = glob.glob(os.path.join(transformer_path, "*.bin"))
for file_path in bin_files:
state_dict = torch.load(file_path)
for name, tensor in state_dict.items():
yield name, tensor
try:
# Get model path from config or download from HF
model_name = self.od_config.model
if os.path.exists(model_name):
model_path = model_name
else:
model_path = download_weights_from_hf_specific(model_name, None, ["*"])
transformer_path = os.path.join(model_path, "transformer")
self.transformer.load_weights(weight_iterator(transformer_path))
except Exception as e:
print(f"An error occurred: {e}")
raise e
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -24,6 +24,7 @@ import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import pad_sequence
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.layer import Attention
@@ -642,33 +643,13 @@ class ZImageTransformer2DModel(nn.Module):
return x, {}
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
weights_map: dict[str, torch.Tensor] = {name: tensor for name, tensor in weights}
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
for name, loaded_weight in weights_map.items():
# In ZImageAttention, to_q, to_k, to_v are separate.
# The weight names are like '...attention.to_q.weight'.
# The logic from qwen_image_transformer for stacking is not needed here.
# We can do a direct mapping.
# The names in the state dict seem to match the model structure.
# Let's try a direct load.
if name in params_dict:
param = params_dict[name]
with torch.no_grad():
param.copy_(loaded_weight)
else:
# The provided weight list might use a different naming convention.
# For example, `layers.0.attention.to_out.0.weight` vs `...`
# Let's try to handle simple cases.
# Replace dots with hyphens for ModuleDict keys
# e.g. all_x_embedder.2-1.weight -> all_x_embedder.2-1.weight
# The names seem to match already.
# If there are any specific mismatches, they can be handled here.
# For now, we assume a direct mapping and warn if a weight is not found.
import logging
logging.warning(f"Weight {name} not found in model parameters.")
loaded_params = set[str]()
for name, loaded_weight in weights:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params