[feat] Extend disaggregated MiniMax-H3 stage with VAE encoders (#6939)

Signed-off-by: asukaqaq-s <1311722138@qq.com>
Signed-off-by: Jader <yjader@foxmail.com>
Co-authored-by: Jader <yjader@foxmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-16 16:52:35 +08:00
1 parent e78a5d01f7
commit d554e6b67e
27 files changed
+3663 -2399

No files matched your search

File diff suppressed because it is too large. Load diff
+210 -21
View File
@@ -15,8 +15,10 @@ import torch
import torch.distributed as dist
import torch.nn as nn
from PIL import Image
from safetensors import safe_open
from transformers.dynamic_module_utils import get_class_from_dynamic_module
from vllm.logger import init_logger
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
DistributedVaeMixin,
@@ -106,6 +108,121 @@ def _load_remote_component(
return component_cls.from_pretrained(component_path)
def _check_trust_remote_code(
component_path: str,
config: dict[str, Any],
trust_remote_code: bool,
) -> None:
"""Apply the remote-code gate before either full or encoder-only loading."""
class_reference = (config.get("auto_map") or {}).get("AutoModel")
if isinstance(class_reference, str) and not trust_remote_code:
raise ValueError(
f"Loading {component_path} executes the modeling code shipped with "
f"the checkpoint (auto_map.AutoModel = {class_reference}). Pass "
"--trust-remote-code (or trust_remote_code=True) to allow it."
)
def _load_selected_state_dict(
model: nn.Module,
weights_path: Path,
prefixes: tuple[str, ...],
) -> None:
with safe_open(str(weights_path), framework="pt", device="cpu") as checkpoint:
selected = sorted(name for name in checkpoint.keys() if name.startswith(prefixes))
if not selected:
raise ValueError(f"{weights_path}: no VAE weights match prefixes {prefixes!r}")
state_dict = {name: checkpoint.get_tensor(name) for name in selected}
model.load_state_dict(state_dict, strict=True)
def _remove_modules(model: nn.Module, names: tuple[str, ...]) -> None:
for name in names:
if not isinstance(getattr(model, name, None), nn.Module):
raise ValueError(f"MiniMax H3 VAE model has no module {name!r}")
delattr(model, name)
def _load_video_vae_encoder(
component_path: str,
config: dict[str, Any],
) -> nn.Module:
class_reference = (config.get("auto_map") or {}).get("AutoModel")
if not isinstance(class_reference, str):
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
component_cls = get_class_from_dynamic_module(class_reference, component_path)
component_module = importlib.import_module(component_cls.__module__)
source_classes = getattr(component_module, "_SOURCE_CLASSES", None)
source_class_name = config.get("source_class_name")
if not isinstance(source_classes, dict) or source_class_name not in source_classes:
raise ValueError(f"unsupported MiniMax H3 video VAE source class {source_class_name!r}")
if bool(config["vae_parallel_tiling"]):
ensure_parallel_state = getattr(component_module, "_ensure_vae_parallel_state", None)
if not callable(ensure_parallel_state):
raise ValueError("MiniMax H3 video VAE remote module does not expose its parallel-state initializer")
ensure_parallel_state()
source_path = Path(component_path) / str(config["source_path"])
weights_path = source_path / str(config["source_safetensors_path"])
load_kwargs = {
"clip_length": int(config["vae_clip_length"]),
"token_drop": int(config["vae_token_drop"]),
"encoder_tiling": int(config["vae_encoder_tiling"]),
"decoder_tiling": int(config["vae_decoder_tiling"]),
"parallel_tiling": int(config["vae_parallel_tiling"]),
"tile_size": int(config["vae_tile_size"]),
"tile_overlap_min": int(config["vae_tile_overlap_min"]),
"encoder_parallel": int(config["vae_encoder_parallel"]),
"decoder_parallel": int(config["vae_decoder_parallel"]),
"chunk_dim": int(config["vae_chunk_dim"]),
}
source_cls = source_classes[source_class_name]
source_config = source_cls.load_config(str(source_path))
with set_default_torch_dtype(torch.float32), torch.device("cpu"):
model, _unused = source_cls.from_config(source_config, return_unused_kwargs=True, **load_kwargs)
_remove_modules(model, ("decoder", "post_quant_conv"))
_load_selected_state_dict(model, weights_path, ("encoder.", "quant_conv."))
model.eval()
return component_cls(model)
def _load_audio_vae_encoder(
component_path: str,
config: dict[str, Any],
) -> nn.Module:
class_reference = (config.get("auto_map") or {}).get("AutoModel")
if not isinstance(class_reference, str):
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
component_cls = get_class_from_dynamic_module(class_reference, component_path)
component_module = importlib.import_module(component_cls.__module__)
load_yaml = getattr(component_module, "_load_yaml", None)
model_cls = getattr(component_module, "DacAudioVAE", None)
if not callable(load_yaml) or not isinstance(model_cls, type):
raise ValueError("MiniMax H3 audio VAE remote module does not expose its source loader")
component_dir = Path(component_path)
audio_config = load_yaml(component_dir / str(config["source_config_path"]))
metadata_path = component_dir / str(config["source_metadata_path"])
metadata_doc = json.loads(metadata_path.read_text(encoding="utf-8"))
metadata = metadata_doc["metadata"]["kwargs"]
with set_default_torch_dtype(torch.float32), torch.device("cpu"):
model = model_cls(
encoder_rates=metadata["encoder_rates"],
decoder_rates=metadata["decoder_rates"],
attn_proj=metadata["attn_proj"],
decoder_type=metadata["decoder_type"],
decoder_dim=audio_config["model_config"]["decoder_dim"],
vae_latent_channels=audio_config["model_config"]["vae_latent_channels"],
sample_rate=metadata["sample_rate"],
)
_remove_modules(model, ("decoder", "dec_in_proj", "logs_proj"))
weights_path = component_dir / str(config["source_safetensors_path"])
_load_selected_state_dict(model, weights_path, ("encoder.", "pre_block.", "mean_proj."))
model.eval()
return component_cls(model)
class _AudioVAEDeterminismContext(AbstractContextManager):
def __enter__(self):
backends = torch.backends
@@ -156,16 +273,24 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
*,
device: torch.device,
load_device: torch.device | None = None,
encode_only: bool = False,
decode_only: bool = False,
trust_remote_code: bool = False,
) -> None:
super().__init__()
self._device_target = device
self.encode_only = bool(encode_only)
self.decode_only = bool(decode_only)
if self.encode_only and self.decode_only:
raise ValueError("MiniMax H3 video VAE cannot be both encode-only and decode-only")
self.config_dict = _load_component_config(component_path)
self.remote = _load_remote_component(
component_path,
self.config_dict,
trust_remote_code=trust_remote_code,
)
_check_trust_remote_code(component_path, self.config_dict, trust_remote_code)
if self.encode_only:
self.remote = _load_video_vae_encoder(component_path, self.config_dict)
else:
self.remote = _load_remote_component(component_path, self.config_dict, trust_remote_code=trust_remote_code)
if self.decode_only:
_remove_modules(self.remote.model, ("encoder", "quant_conv"))
# Match the reference loader contract before installing inference-only
# decoder fast paths. Keyframe encoding remains FP32; decoder Linear
# weights may be materialized in FP16 because reference decode casts
@@ -222,10 +347,11 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
self,
parallel_size: int,
mode: str = "tile",
process_group: dist.ProcessGroup | None = None,
) -> None:
if mode != "tile":
raise ValueError(f"MiniMax H3 VAE supports its native tile parallel mode only, got {mode!r}")
group = get_world_group().device_group
group = process_group if process_group is not None else get_world_group().device_group
world_size = dist.get_world_size(group)
rank = dist.get_rank(group)
parallel_size = int(parallel_size)
@@ -412,15 +538,40 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
cols, _, _ = self.model.split_tiles(int(latent.shape[-1]) * ratio, True)
return len(rows) * len(cols)
def _encoder_tile_count(self, height: int, width: int) -> int:
processor = getattr(self.model, "processor", None)
align = getattr(processor, "_align_to_total_patch_size", None)
if callable(align):
height, width = align(int(height), int(width))
rows, _, _ = self.model.split_tiles(int(height), False)
cols, _, _ = self.model.split_tiles(int(width), False)
return len(rows) * len(cols)
def _encoder_tiling_context(
self,
height: int,
width: int,
) -> AbstractContextManager:
parallel_size = int(getattr(self, "parallel_size", 1))
if parallel_size <= 1:
return nullcontext()
num_tiles = self._encoder_tile_count(height, width)
if num_tiles >= parallel_size:
return nullcontext()
logger.warning_once(
"MiniMax-H3 VAE encode splits a %dx%d input into %d tile(s) but "
"the tile group has %d ranks; encoding rank-locally for this "
"shape instead, which avoids ranks without tiles hanging the "
"collective.",
width,
height,
num_tiles,
parallel_size,
)
return self._rank_local_tiling()
@contextmanager
def _rank_local_tiling(self) -> Iterator[None]:
"""Run one decode with tiling kept on this rank, then restore the group.
Used only when there are fewer tiles than ranks. Every rank then decodes
every tile, which is slower than sharing the work but is correct and
involves no collective.
"""
state = self._native_parallel_state()
saved_state = dict(state)
saved_tiling = self.model.parallel_tiling
@@ -448,15 +599,24 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
@torch.inference_mode()
def encode_image(self, image: Image.Image) -> torch.Tensor:
if getattr(self, "decode_only", False):
raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode images")
previous_parallel = self.model.parallel_tiling
self.model.parallel_tiling = False
if int(getattr(self, "parallel_size", 1)) <= 1:
self.model.parallel_tiling = False
parameter = next(self.parameters())
previous_dtype = parameter.dtype
if previous_dtype != torch.float32:
self.to(torch.float32)
devices = [parameter.device] if parameter.device.type != "cpu" else []
try:
with torch.random.fork_rng(devices=devices, device_type=parameter.device.type):
with (
self._encoder_tiling_context(image.height, image.width),
torch.random.fork_rng(
devices=devices,
device_type=parameter.device.type,
),
):
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
for device in devices:
with self.device_module.device(device):
@@ -495,13 +655,26 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
self,
frames: Any,
) -> tuple[torch.Tensor, tuple[int, int, int]]:
if getattr(self, "decode_only", False):
raise RuntimeError("MiniMax H3 decode-only video VAE cannot encode videos")
parameter = next(self.parameters())
previous_dtype = parameter.dtype
if previous_dtype != torch.float32:
self.to(torch.float32)
devices = [parameter.device] if parameter.device.type != "cpu" else []
shape = getattr(frames, "shape", None)
if int(getattr(self, "parallel_size", 1)) > 1:
if shape is None or len(shape) != 4:
raise ValueError("parallel MiniMax H3 video encode requires a rank-4 frame array")
if int(shape[-1]) in {1, 3, 4}:
frame_height, frame_width = int(shape[-3]), int(shape[-2])
else:
frame_height, frame_width = int(shape[-2]), int(shape[-1])
tiling_context = self._encoder_tiling_context(frame_height, frame_width)
else:
tiling_context = nullcontext()
try:
with torch.random.fork_rng(devices=devices, device_type=parameter.device.type):
with tiling_context, torch.random.fork_rng(devices=devices, device_type=parameter.device.type):
torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
for device in devices:
with self.device_module.device(device):
@@ -561,6 +734,8 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
@torch.inference_mode()
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
if getattr(self, "encode_only", False):
raise RuntimeError("MiniMax H3 encode-only video VAE cannot decode latents")
with self._decode_tiling_context(latent):
decoded = self.model.decode_base(self._denormalize_latent(latent))
return self._normalize_decoded_frames(self.model.processor.revert_tensor(decoded))
@@ -598,6 +773,8 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
the complete decode path. After a callback failure, the remaining
chunks are decoded and discarded before the exception is re-raised.
"""
if getattr(self, "encode_only", False):
raise RuntimeError("MiniMax H3 encode-only video VAE cannot decode latents")
if not callable(on_chunk):
raise TypeError("on_chunk must be callable")
if not callable(getattr(self.model, "_adaptive_decode", None)):
@@ -622,16 +799,24 @@ class MiniMaxH3AudioVAE(nn.Module):
*,
device: torch.device,
load_device: torch.device | None = None,
encode_only: bool = False,
decode_only: bool = False,
trust_remote_code: bool = False,
) -> None:
super().__init__()
self._device_target = device
self.encode_only = bool(encode_only)
self.decode_only = bool(decode_only)
if self.encode_only and self.decode_only:
raise ValueError("MiniMax H3 audio VAE cannot be both encode-only and decode-only")
self.config_dict = _load_component_config(component_path)
self.remote = _load_remote_component(
component_path,
self.config_dict,
trust_remote_code=trust_remote_code,
)
_check_trust_remote_code(component_path, self.config_dict, trust_remote_code)
if self.encode_only:
self.remote = _load_audio_vae_encoder(component_path, self.config_dict)
else:
self.remote = _load_remote_component(component_path, self.config_dict, trust_remote_code=trust_remote_code)
if self.decode_only:
_remove_modules(self.remote.model, ("encoder", "pre_block", "mean_proj"))
# The checkpoint's audio VAE contract is FP32 for both reference
# encoding and waveform decoding.
initial_device = load_device or device
@@ -674,6 +859,8 @@ class MiniMaxH3AudioVAE(nn.Module):
waveform: torch.Tensor,
sample_rate: int,
) -> tuple[torch.Tensor, int]:
if getattr(self, "decode_only", False):
raise RuntimeError("MiniMax H3 decode-only audio VAE cannot encode waveforms")
import torchaudio
waveform = waveform.float()
@@ -719,6 +906,8 @@ class MiniMaxH3AudioVAE(nn.Module):
@torch.inference_mode()
def decode_latent(self, latent: torch.Tensor) -> torch.Tensor:
if getattr(self, "encode_only", False):
raise RuntimeError("MiniMax H3 encode-only audio VAE cannot decode latents")
channels = int(self.config_dict["latent_channels"])
mean = torch.tensor(
self.config_dict["latents_mean"],