[feat] Extend disaggregated MiniMax-H3 stage with VAE encoders (#6939)
Signed-off-by: asukaqaq-s <1311722138@qq.com> Signed-off-by: Jader <yjader@foxmail.com> Co-authored-by: Jader <yjader@foxmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
27 files changed
+3663
-2399
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -15,8 +15,10 @@ import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
from PIL import Image
|
||||
from safetensors import safe_open
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedVaeMixin,
|
||||
@@ -106,6 +108,121 @@ def _load_remote_component(
|
||||
return component_cls.from_pretrained(component_path)
|
||||
|
||||
|
||||
def _check_trust_remote_code(
|
||||
component_path: str,
|
||||
config: dict[str, Any],
|
||||
trust_remote_code: bool,
|
||||
) -> None:
|
||||
"""Apply the remote-code gate before either full or encoder-only loading."""
|
||||
|
||||
class_reference = (config.get("auto_map") or {}).get("AutoModel")
|
||||
if isinstance(class_reference, str) and not trust_remote_code:
|
||||
raise ValueError(
|
||||
f"Loading {component_path} executes the modeling code shipped with "
|
||||
f"the checkpoint (auto_map.AutoModel = {class_reference}). Pass "
|
||||
"--trust-remote-code (or trust_remote_code=True) to allow it."
|
||||
)
|
||||
|
||||
|
||||
def _load_selected_state_dict(
|
||||
model: nn.Module,
|
||||
weights_path: Path,
|
||||
prefixes: tuple[str, ...],
|
||||
) -> None:
|
||||
with safe_open(str(weights_path), framework="pt", device="cpu") as checkpoint:
|
||||
selected = sorted(name for name in checkpoint.keys() if name.startswith(prefixes))
|
||||
if not selected:
|
||||
raise ValueError(f"{weights_path}: no VAE weights match prefixes {prefixes!r}")
|
||||
state_dict = {name: checkpoint.get_tensor(name) for name in selected}
|
||||
model.load_state_dict(state_dict, strict=True)
|
||||
|
||||
|
||||
def _remove_modules(model: nn.Module, names: tuple[str, ...]) -> None:
|
||||
for name in names:
|
||||
if not isinstance(getattr(model, name, None), nn.Module):
|
||||
raise ValueError(f"MiniMax H3 VAE model has no module {name!r}")
|
||||
delattr(model, name)
|
||||
|
||||
|
||||
def _load_video_vae_encoder(
|
||||
component_path: str,
|
||||
config: dict[str, Any],
|
||||
) -> nn.Module:
|
||||
class_reference = (config.get("auto_map") or {}).get("AutoModel")
|
||||
if not isinstance(class_reference, str):
|
||||
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
|
||||
component_cls = get_class_from_dynamic_module(class_reference, component_path)
|
||||
component_module = importlib.import_module(component_cls.__module__)
|
||||
source_classes = getattr(component_module, "_SOURCE_CLASSES", None)
|
||||
source_class_name = config.get("source_class_name")
|
||||
if not isinstance(source_classes, dict) or source_class_name not in source_classes:
|
||||
raise ValueError(f"unsupported MiniMax H3 video VAE source class {source_class_name!r}")
|
||||
if bool(config["vae_parallel_tiling"]):
|
||||
ensure_parallel_state = getattr(component_module, "_ensure_vae_parallel_state", None)
|
||||
if not callable(ensure_parallel_state):
|
||||
raise ValueError("MiniMax H3 video VAE remote module does not expose its parallel-state initializer")
|
||||
ensure_parallel_state()
|
||||
|
||||
source_path = Path(component_path) / str(config["source_path"])
|
||||
weights_path = source_path / str(config["source_safetensors_path"])
|
||||
load_kwargs = {
|
||||
"clip_length": int(config["vae_clip_length"]),
|
||||
"token_drop": int(config["vae_token_drop"]),
|
||||
"encoder_tiling": int(config["vae_encoder_tiling"]),
|
||||
"decoder_tiling": int(config["vae_decoder_tiling"]),
|
||||
"parallel_tiling": int(config["vae_parallel_tiling"]),
|
||||
"tile_size": int(config["vae_tile_size"]),
|
||||
"tile_overlap_min": int(config["vae_tile_overlap_min"]),
|
||||
"encoder_parallel": int(config["vae_encoder_parallel"]),
|
||||
"decoder_parallel": int(config["vae_decoder_parallel"]),
|
||||
"chunk_dim": int(config["vae_chunk_dim"]),
|
||||
}
|
||||
source_cls = source_classes[source_class_name]
|
||||
source_config = source_cls.load_config(str(source_path))
|
||||
with set_default_torch_dtype(torch.float32), torch.device("cpu"):
|
||||
model, _unused = source_cls.from_config(source_config, return_unused_kwargs=True, **load_kwargs)
|
||||
_remove_modules(model, ("decoder", "post_quant_conv"))
|
||||
_load_selected_state_dict(model, weights_path, ("encoder.", "quant_conv."))
|
||||
model.eval()
|
||||
return component_cls(model)
|
||||
|
||||
|
||||
def _load_audio_vae_encoder(
|
||||
component_path: str,
|
||||
config: dict[str, Any],
|
||||
) -> nn.Module:
|
||||
class_reference = (config.get("auto_map") or {}).get("AutoModel")
|
||||
if not isinstance(class_reference, str):
|
||||
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
|
||||
component_cls = get_class_from_dynamic_module(class_reference, component_path)
|
||||
component_module = importlib.import_module(component_cls.__module__)
|
||||
load_yaml = getattr(component_module, "_load_yaml", None)
|
||||
model_cls = getattr(component_module, "DacAudioVAE", None)
|
||||
if not callable(load_yaml) or not isinstance(model_cls, type):
|
||||
raise ValueError("MiniMax H3 audio VAE remote module does not expose its source loader")
|
||||
|
||||
component_dir = Path(component_path)
|
||||
audio_config = load_yaml(component_dir / str(config["source_config_path"]))
|
||||
metadata_path = component_dir / str(config["source_metadata_path"])
|
||||
metadata_doc = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||
metadata = metadata_doc["metadata"]["kwargs"]
|
||||
with set_default_torch_dtype(torch.float32), torch.device("cpu"):
|
||||
model = model_cls(
|
||||
encoder_rates=metadata["encoder_rates"],
|
||||
decoder_rates=metadata["decoder_rates"],
|
||||
attn_proj=metadata["attn_proj"],
|
||||
decoder_type=metadata["decoder_type"],
|
||||
decoder_dim=audio_config["model_config"]["decoder_dim"],
|
||||
vae_latent_channels=audio_config["model_config"]["vae_latent_channels"],
|
||||
sample_rate=metadata["sample_rate"],
|
||||
)
|
||||
_remove_modules(model, ("decoder", "dec_in_proj", "logs_proj"))
|
||||
weights_path = component_dir / str(config["source_safetensors_path"])
|
||||
_load_selected_state_dict(model, weights_path, ("encoder.", "pre_block.", "mean_proj."))
|
||||
model.eval()
|
||||
return component_cls(model)
|
||||
|
||||
|
||||
class _AudioVAEDeterminismContext(AbstractContextManager):
|
||||
def __enter__(self):
|
||||
backends = torch.backends
|
||||
@@ -156,16 +273,24 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
*,
|
||||
device: torch.device,
|
||||
load_device: torch.device | None = None,
|
||||
encode_only: bool = False,
|
||||
decode_only: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._device_target = device
|
||||
self.encode_only = bool(encode_only)
|
||||
self.decode_only = bool(decode_only)
|
||||
if self.encode_only and self.decode_only:
|
||||
raise ValueError("MiniMax H3 video VAE cannot be both encode-only and decode-only")
|
||||
self.config_dict = _load_component_config(component_path)
|
||||
self.remote = _load_remote_component(
|
||||
component_path,
|
||||
self.config_dict,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
_check_trust_remote_code(component_path, self.config_dict, trust_remote_code)
|
||||
if self.encode_only:
|
||||
self.remote = _load_video_vae_encoder(component_path, self.config_dict)
|
||||
else:
|
||||
self.remote = _load_remote_component(component_path, self.config_dict, trust_remote_code=trust_remote_code)
|
||||
if self.decode_only:
|
||||
_remove_modules(self.remote.model, ("encoder", "quant_conv"))
|
||||
# Match the reference loader contract before installing inference-only
|
||||
# decoder fast paths. Keyframe encoding remains FP32; decoder Linear
|
||||
# weights may be materialized in FP16 because reference decode casts
|
||||
@@ -222,10 +347,11 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
self,
|
||||
parallel_size: int,
|
||||
mode: str = "tile",
|
||||
process_group: dist.ProcessGroup | None = None,
|
||||
) -> None:
|
||||
if mode != "tile":
|
||||
raise ValueError(f"MiniMax H3 VAE supports its native tile parallel mode only, got {mode!r}")
|
||||
group = get_world_group().device_group
|
||||
group = process_group if process_group is not None else get_world_group().device_group
|
||||
world_size = dist.get_world_size(group)
|
||||
rank = dist.get_rank(group)
|
||||
parallel_size = int(parallel_size)
|
||||
@@ -412,15 +538,40 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
cols, _, _ = self.model.split_tiles(int(latent.shape[-1]) * ratio, True)
|
||||
return len(rows) * len(cols)
|
||||
|
||||
def _encoder_tile_count(self, height: int, width: int) -> int:
|
||||
processor = getattr(self.model, "processor", None)
|
||||
align = getattr(processor, "_align_to_total_patch_size", None)
|
||||
if callable(align):
|
||||
height, width = align(int(height), int(width))
|
||||
rows, _, _ = self.model.split_tiles(int(height), False)
|
||||
cols, _, _ = self.model.split_tiles(int(width), False)
|
||||
return len(rows) * len(cols)
|
||||
|
||||
def _encoder_tiling_context(
|
||||
self,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> AbstractContextManager:
|
||||
parallel_size = int(getattr(self, "parallel_size", 1))
|
||||
if parallel_size <= 1:
|
||||
return nullcontext()
|
||||
num_tiles = self._encoder_tile_count(height, width)
|
||||
if num_tiles >= parallel_size:
|
||||
return nullcontext()
|
||||
logger.warning_once(
|
||||
"MiniMax-H3 VAE encode splits a %dx%d input into %d tile(s) but "
|
||||
"the tile group has %d ranks; encoding rank-locally for this "
|
||||
"shape instead, which avoids ranks without tiles hanging the "
|
||||
"collective.",
|
||||
width,
|
||||
height,
|
||||
num_tiles,
|
||||
parallel_size,
|
||||
)
|
||||
return self._rank_local_tiling()
|
||||
|
||||
@contextmanager
|
||||
def _rank_local_tiling(self) -> Iterator[None]:
|
||||
"""Run one decode with tiling kept on this rank, then restore the group.
|
||||
|
||||
Used only when there are fewer tiles than ranks. Every rank then decodes
|
||||
every tile, which is slower than sharing the work but is correct and
|
||||
involves no collective.
|
||||
"""
|
||||
|
||||
state = self._native_parallel_state()
|
||||
saved_state = dict(state)
|
||||
saved_tiling = self.model.parallel_tiling
|
||||
@@ -448,15 +599,24 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
|
||||
@torch.inference_mode()
|
||||
def encode_image(self, image: Image.Image) -> torch.Tensor:
|
||||
if getattr(self, "decode_only", False):
|
||||
raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode images")
|
||||
previous_parallel = self.model.parallel_tiling
|
||||
self.model.parallel_tiling = False
|
||||
if int(getattr(self, "parallel_size", 1)) <= 1:
|
||||
self.model.parallel_tiling = False
|
||||
parameter = next(self.parameters())
|
||||
previous_dtype = parameter.dtype
|
||||
if previous_dtype != torch.float32:
|
||||
self.to(torch.float32)
|
||||
devices = [parameter.device] if parameter.device.type != "cpu" else []
|
||||
try:
|
||||
with torch.random.fork_rng(devices=devices, device_type=parameter.device.type):
|
||||
with (
|
||||
self._encoder_tiling_context(image.height, image.width),
|
||||
torch.random.fork_rng(
|
||||
devices=devices,
|
||||
device_type=parameter.device.type,
|
||||
),
|
||||
):
|
||||
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
for device in devices:
|
||||
with self.device_module.device(device):
|
||||
@@ -495,13 +655,26 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
self,
|
||||
frames: Any,
|
||||
) -> tuple[torch.Tensor, tuple[int, int, int]]:
|
||||
if getattr(self, "decode_only", False):
|
||||
raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode videos")
|
||||
parameter = next(self.parameters())
|
||||
previous_dtype = parameter.dtype
|
||||
if previous_dtype != torch.float32:
|
||||
self.to(torch.float32)
|
||||
devices = [parameter.device] if parameter.device.type != "cpu" else []
|
||||
shape = getattr(frames, "shape", None)
|
||||
if int(getattr(self, "parallel_size", 1)) > 1:
|
||||
if shape is None or len(shape) != 4:
|
||||
raise ValueError("parallel MiniMax H3 video encode requires a rank-4 frame array")
|
||||
if int(shape[-1]) in {1, 3, 4}:
|
||||
frame_height, frame_width = int(shape[-3]), int(shape[-2])
|
||||
else:
|
||||
frame_height, frame_width = int(shape[-2]), int(shape[-1])
|
||||
tiling_context = self._encoder_tiling_context(frame_height, frame_width)
|
||||
else:
|
||||
tiling_context = nullcontext()
|
||||
try:
|
||||
with torch.random.fork_rng(devices=devices, device_type=parameter.device.type):
|
||||
with tiling_context, torch.random.fork_rng(devices=devices, device_type=parameter.device.type):
|
||||
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
for device in devices:
|
||||
with self.device_module.device(device):
|
||||
@@ -561,6 +734,8 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
|
||||
@torch.inference_mode()
|
||||
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
if getattr(self, "encode_only", False):
|
||||
raise RuntimeError("MiniMax H3 encode-only video VAE cannot decode latents")
|
||||
with self._decode_tiling_context(latent):
|
||||
decoded = self.model.decode_base(self._denormalize_latent(latent))
|
||||
return self._normalize_decoded_frames(self.model.processor.revert_tensor(decoded))
|
||||
@@ -598,6 +773,8 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
the complete decode path. After a callback failure, the remaining
|
||||
chunks are decoded and discarded before the exception is re-raised.
|
||||
"""
|
||||
if getattr(self, "encode_only", False):
|
||||
raise RuntimeError("MiniMax H3 encode-only video VAE cannot decode latents")
|
||||
if not callable(on_chunk):
|
||||
raise TypeError("on_chunk must be callable")
|
||||
if not callable(getattr(self.model, "_adaptive_decode", None)):
|
||||
@@ -622,16 +799,24 @@ class MiniMaxH3AudioVAE(nn.Module):
|
||||
*,
|
||||
device: torch.device,
|
||||
load_device: torch.device | None = None,
|
||||
encode_only: bool = False,
|
||||
decode_only: bool = False,
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._device_target = device
|
||||
self.encode_only = bool(encode_only)
|
||||
self.decode_only = bool(decode_only)
|
||||
if self.encode_only and self.decode_only:
|
||||
raise ValueError("MiniMax H3 audio VAE cannot be both encode-only and decode-only")
|
||||
self.config_dict = _load_component_config(component_path)
|
||||
self.remote = _load_remote_component(
|
||||
component_path,
|
||||
self.config_dict,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
_check_trust_remote_code(component_path, self.config_dict, trust_remote_code)
|
||||
if self.encode_only:
|
||||
self.remote = _load_audio_vae_encoder(component_path, self.config_dict)
|
||||
else:
|
||||
self.remote = _load_remote_component(component_path, self.config_dict, trust_remote_code=trust_remote_code)
|
||||
if self.decode_only:
|
||||
_remove_modules(self.remote.model, ("encoder", "pre_block", "mean_proj"))
|
||||
# The checkpoint's audio VAE contract is FP32 for both reference
|
||||
# encoding and waveform decoding.
|
||||
initial_device = load_device or device
|
||||
@@ -674,6 +859,8 @@ class MiniMaxH3AudioVAE(nn.Module):
|
||||
waveform: torch.Tensor,
|
||||
sample_rate: int,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
if getattr(self, "decode_only", False):
|
||||
raise RuntimeError("MiniMax H3 decode-only audio VAE cannot encode waveforms")
|
||||
import torchaudio
|
||||
|
||||
waveform = waveform.float()
|
||||
@@ -719,6 +906,8 @@ class MiniMaxH3AudioVAE(nn.Module):
|
||||
|
||||
@torch.inference_mode()
|
||||
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
if getattr(self, "encode_only", False):
|
||||
raise RuntimeError("MiniMax H3 encode-only audio VAE cannot decode latents")
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
mean = torch.tensor(
|
||||
self.config_dict["latents_mean"],
|
||||
|
||||
Reference in new issue
Block a user