[Feature] Support MiniMax-H3 Turbo LoRA with the legacy manager (#6476)

Signed-off-by: mglyn <1203789601@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Mu GuanLinandHongsheng Liu authored and GitHub committed 2026-08-24 06:21:42 +08:00
1 parent 1c917b2751
commit 3d035bfa19
8 files changed
+956 -71

No files matched your search

@@ -0,0 +1,205 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import math
from pathlib import Path
import torch
from safetensors import safe_open
from vllm.lora.lora_model import LoRAModel
from vllm.lora.lora_weights import PackedLoRALayerWeights
from vllm.lora.peft_helper import PEFTHelper
from vllm.model_executor.models.utils import WeightsMapper
from vllm_omni.lora.request import LoRARequest
_TURBO_RANK = 128
_TURBO_ALPHA = 128
_TURBO_HIDDEN_SIZE = 5376
_TURBO_ATTENTION_INNER_SIZE = 7168
_TURBO_FFN_HIDDEN_SIZE = 14336
_TURBO_FILENAME = "minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"
_LORA_A_SUFFIX = ".lora_A.default.weight"
_LORA_B_SUFFIX = ".lora_B.default.weight"
_TURBO_TARGETS = frozenset({"to_q", "to_k", "to_v", "out_proj", "fc1", "fc2"})
_TURBO_RAW_TARGET_SUFFIXES = (
"attn.to_q",
"attn.to_k",
"attn.to_v",
"attn.to_out.0",
"ff.net.0.proj",
"ff.net.2",
)
_TURBO_TARGET_DIMS = {
"attn.to_q": (_TURBO_HIDDEN_SIZE, _TURBO_ATTENTION_INNER_SIZE),
"attn.to_k": (_TURBO_HIDDEN_SIZE, _TURBO_ATTENTION_INNER_SIZE),
"attn.to_v": (_TURBO_HIDDEN_SIZE, _TURBO_ATTENTION_INNER_SIZE),
"attn.to_out.0": (_TURBO_ATTENTION_INNER_SIZE, _TURBO_HIDDEN_SIZE),
"ff.net.0.proj": (_TURBO_HIDDEN_SIZE, 2 * _TURBO_FFN_HIDDEN_SIZE),
"ff.net.2": (_TURBO_FFN_HIDDEN_SIZE, _TURBO_HIDDEN_SIZE),
}
_TURBO_EXPECTED_RAW_TARGETS = frozenset(
f"{prefix}.{block_index}.{suffix}"
for prefix, block_count in (
("transformer_blocks", 50),
("token_refiner.refiner_blocks", 2),
)
for block_index in range(block_count)
for suffix in _TURBO_RAW_TARGET_SUFFIXES
)
_TURBO_TARGET_PATTERN = (
r"^transformer\.(?:token_refiner\.blocks|blocks)\.\d+\."
r"(?:attn\.(?:to_q|to_k|to_v|out_proj)|mlp\.(?:fc1|fc2))$"
)
_TURBO_WEIGHTS_MAPPER = WeightsMapper(
orig_to_new_substr={
"token_refiner.refiner_blocks.": "token_refiner.blocks.",
"transformer_blocks.": "blocks.",
".attn.to_out.0.": ".attn.out_proj.",
".ff.net.0.proj.": ".mlp.fc1.",
".ff.net.2.": ".mlp.fc2.",
".lora_A.default.": ".lora_A.",
".lora_B.default.": ".lora_B.",
}
)
def _select_turbo_file(artifact_path: str | Path) -> Path | None:
path = Path(artifact_path)
if path.is_file():
return path if path.suffix == ".safetensors" else None
if not path.is_dir():
return None
candidate = path / _TURBO_FILENAME
return candidate if candidate.is_file() else None
def _validate_and_convert_tensors(checkpoint) -> dict[str, torch.Tensor]:
tensors: dict[str, torch.Tensor] = {}
pairs: dict[str, set[str]] = {}
raw_targets: set[str] = set()
for name in checkpoint.keys():
if name.endswith(_LORA_A_SUFFIX):
raw_target = name[: -len(_LORA_A_SUFFIX)]
side = "a"
elif name.endswith(_LORA_B_SUFFIX):
raw_target = name[: -len(_LORA_B_SUFFIX)]
side = "b"
else:
raise ValueError(f"Unconsumed MiniMax-H3 Turbo tensor: {name!r}")
raw_targets.add(raw_target)
mapped_name = _TURBO_WEIGHTS_MAPPER.apply_list([name])[0]
mapped_target = mapped_name.rsplit(".lora_", 1)[0]
if mapped_target.rsplit(".", 1)[-1] not in _TURBO_TARGETS:
raise ValueError(f"Unsupported MiniMax-H3 Turbo target: {raw_target!r}")
target_sides = pairs.setdefault(mapped_target, set())
if side in target_sides:
raise ValueError(f"Duplicate MiniMax-H3 Turbo tensor for {mapped_target}.{side}")
target_sides.add(side)
tensor = checkpoint.get_tensor(name)
if tensor.ndim != 2:
raise ValueError(f"MiniMax-H3 Turbo LoRA tensors must be matrices, got {name}={tuple(tensor.shape)}")
suffix = next((suffix for suffix in _TURBO_RAW_TARGET_SUFFIXES if raw_target.endswith(suffix)), None)
if suffix is None:
raise ValueError(f"MiniMax-H3 Turbo LoRA contains unsupported target: {raw_target}")
input_dim, output_dim = _TURBO_TARGET_DIMS[suffix]
expected_shape = (_TURBO_RANK, input_dim) if side == "a" else (output_dim, _TURBO_RANK)
if tuple(tensor.shape) != expected_shape:
raise ValueError(
f"MiniMax-H3 Turbo tensor has invalid global shape: {name}={tuple(tensor.shape)}, "
f"expected={expected_shape}"
)
if side == "b" and ".ff.net.0.proj." in name:
value, gate = tensor.chunk(2, dim=0)
tensor = torch.cat((gate, value), dim=0).contiguous()
tensors[name] = tensor
incomplete = sorted(target for target, sides in pairs.items() if sides != {"a", "b"})
if incomplete:
raise ValueError(f"Incomplete MiniMax-H3 Turbo LoRA pairs: {incomplete}")
missing = sorted(_TURBO_EXPECTED_RAW_TARGETS - raw_targets)
unexpected = sorted(raw_targets - _TURBO_EXPECTED_RAW_TARGETS)
if missing or unexpected:
raise ValueError(
"MiniMax-H3 Turbo target set does not match the supported v1.0 artifact: "
f"missing={len(missing)} {missing[:5]}, unexpected={len(unexpected)} {unexpected[:5]}"
)
return tensors
def _pack_h3_turbo_fc1(lora_model: LoRAModel) -> None:
"""Represent H3's fused gate/up projection without generic layout guesses."""
for module_name, weights in tuple(lora_model.loras.items()):
if not module_name.endswith(".mlp.fc1"):
continue
gate_b, up_b = weights.lora_b.chunk(2, dim=0)
lora_model.loras[module_name] = PackedLoRALayerWeights(
module_name=module_name,
rank=weights.rank,
lora_alphas=[weights.lora_alpha, weights.lora_alpha],
lora_a=[weights.lora_a, weights.lora_a],
lora_b=[gate_b.contiguous(), up_b.contiguous()],
scaling=[weights.scaling, weights.scaling],
)
def load_minimax_h3_turbo_lora(
*,
partition: str,
lora_request: LoRARequest,
lora_path: str | Path,
dtype: torch.dtype,
unsupported_offload_mode: str | None = None,
) -> tuple[LoRAModel, PEFTHelper] | None:
"""Load the published LightX2V Turbo v1.0 through the legacy manager."""
lora_file = _select_turbo_file(lora_path)
if lora_file is None:
return None
with safe_open(lora_file, framework="pt", device="cpu") as checkpoint:
metadata = checkpoint.metadata() or {}
if metadata.get("key_format") != "minimax-h3-diffusers":
if lora_file.name == _TURBO_FILENAME:
raise ValueError(
"MiniMax-H3 Turbo v1.0 requires safetensors metadata key_format='minimax-h3-diffusers'"
)
return None
if lora_file.name != _TURBO_FILENAME:
raise ValueError(f"MiniMax-H3 Turbo supports only {_TURBO_FILENAME!r}, got {lora_file.name!r}")
raw_alpha = metadata.get("alpha")
try:
alpha = float(raw_alpha) if raw_alpha is not None else math.nan
except ValueError as exc:
raise ValueError(f"MiniMax-H3 Turbo alpha must be numeric, got {raw_alpha!r}") from exc
if alpha != _TURBO_ALPHA:
raise ValueError(f"MiniMax-H3 Turbo v1.0 requires alpha={_TURBO_ALPHA}, got {raw_alpha!r}")
if partition == "ref2va":
raise ValueError("MiniMax-H3 Turbo LoRA supports FL2VA/T2VA only")
if unsupported_offload_mode is not None:
raise ValueError(f"MiniMax-H3 Turbo dynamic LoRA does not support {unsupported_offload_mode}")
tensors = _validate_and_convert_tensors(checkpoint)
peft_helper = PEFTHelper.from_dict(
{
"r": _TURBO_RANK,
"lora_alpha": _TURBO_ALPHA,
"target_modules": _TURBO_TARGET_PATTERN,
}
)
lora_model = LoRAModel.from_lora_tensors(
lora_model_id=lora_request.lora_int_id,
tensors=tensors,
peft_helper=peft_helper,
device="cpu",
dtype=dtype,
weights_mapper=_TURBO_WEIGHTS_MAPPER,
)
_pack_h3_turbo_fc1(lora_model)
return lora_model, peft_helper
@@ -954,6 +954,14 @@ class MiniMaxH3DiTModel(nn.Module):
# quantization or LoRA to map onto. Address the fused layers directly, e.g.
# ignored_layers=["blocks.0.attn.qkv_proj"].
packed_modules_mapping = {}
# Turbo LoRA checkpoints publish separate Q/K/V adapters. This declaration
# lets the legacy diffusion LoRA manager bind them to the packed QKV layer;
# it does not change the fused base-checkpoint loading path above.
stacked_params_mapping = (
(".attn.qkv_proj", ".attn.to_q", "q"),
(".attn.qkv_proj", ".attn.to_k", "k"),
(".attn.qkv_proj", ".attn.to_v", "v"),
)
def _validate_tp_config(self, *, arch: MiniMaxH3DiTArchConfig, tp_size: int) -> None:
if tp_size < 1:
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""vLLM-Omni pipeline for MiniMax H3 FL2VA and Ref2VA partitions."""
from __future__ import annotations
@@ -11,7 +12,7 @@ from collections.abc import Iterable, Mapping, Sequence
from contextlib import contextmanager, nullcontext
from itertools import groupby
from pathlib import Path
from typing import Any, ClassVar
from typing import TYPE_CHECKING, Any, ClassVar
import numpy as np
import torch
@@ -74,6 +75,7 @@ from .condition_noise import (
)
from .denoise_loop import MiniMaxH3DenoiseBranch, minimax_h3_denoise_loop
from .encoder import MiniMaxH3Qwen3VLEncoder
from .lora import load_minimax_h3_turbo_lora
from .minimax_h3_transformer import MiniMaxH3DiTModel
from .packed_sequence import (
minimax_h3_packed_sequence,
@@ -110,6 +112,12 @@ from .vae import MiniMaxH3AudioVAE, MiniMaxH3VideoVAE
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.lora.lora_model import LoRAModel
from vllm.lora.peft_helper import PEFTHelper
from vllm_omni.lora.request import LoRARequest
MINIMAX_H3_FPS = 24
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
MINIMAX_H3_IMGVID_COND_TIMESTEP = 0.999
@@ -130,6 +138,9 @@ MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
MINIMAX_H3_TURBO_SIGMA_POINTS = 5
MINIMAX_H3_TURBO_VIDEO_SHIFT = 6.0
MINIMAX_H3_TURBO_AUDIO_SHIFT = 3.0
MINIMAX_H3_DOWNLOAD_PATTERNS = [
"FL2VA/**",
"Ref2VA/model_index.json",
@@ -559,6 +570,80 @@ class MiniMaxH3Pipeline(
# uniform path available to partially constructed pipelines.
_base_schedule_by_partition: ClassVar[Mapping[str, DMD2SigmaSchedule | None]] = {}
def _load_diffusion_lora_adapter(
self,
*,
lora_request: LoRARequest,
lora_path: str | Path,
dtype: torch.dtype,
) -> tuple[LoRAModel, PEFTHelper] | None:
# A cache eviction may be followed by a different adapter reusing the
# same client-supplied ID. Every real load replaces the classification.
self._turbo_lora_adapter_ids.discard(lora_request.lora_int_id)
od_config = getattr(self, "od_config", None)
offload_modes = []
if getattr(od_config, "enable_cpu_offload", False):
offload_modes.append("model-level CPU offload (--enable-cpu-offload)")
if getattr(od_config, "enable_layerwise_offload", False):
offload_modes.append("layerwise offload (--enable-layerwise-offload)")
if getattr(od_config, "enable_distributed_layerwise_offload", False):
offload_modes.append("distributed layerwise offload (--enable-distributed-layerwise-offload)")
loaded = load_minimax_h3_turbo_lora(
partition=self.partition,
lora_request=lora_request,
lora_path=lora_path,
dtype=dtype,
unsupported_offload_mode=" or ".join(offload_modes) or None,
)
if loaded is not None:
self._turbo_lora_adapter_ids.add(lora_request.lora_int_id)
return loaded
def _validate_diffusion_lora_binding(
self,
*,
lora_model: LoRAModel,
bound_lora_names: frozenset[str],
) -> None:
if lora_model.id not in self._turbo_lora_adapter_ids:
return
missing = sorted(set(lora_model.loras) - bound_lora_names)
if missing:
raise ValueError(
"MiniMax-H3 Turbo LoRA binding is incomplete: "
f"bound={len(bound_lora_names)}/{len(lora_model.loras)}, missing={missing[:5]}"
)
def _has_active_turbo_lora(self, sampling: Any) -> bool:
lora_request = sampling.lora_request
return (
lora_request is not None
and not math.isclose(0.0, float(sampling.lora_scale))
and lora_request.lora_int_id in self._turbo_lora_adapter_ids
)
def _validate_turbo_sampling(self, sampling: Any) -> None:
extra = sampling.extra_args or {}
sigma_points = sampling.num_inference_steps
if sigma_points != MINIMAX_H3_TURBO_SIGMA_POINTS:
raise OmniClientError(
"MiniMax-H3 Turbo requires num_inference_steps=5 (five sigma points produce four denoiser evaluations)"
)
try:
video_shift = float(extra.get("flow_shift", self.default_video_shift))
except (TypeError, ValueError) as exc:
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}") from exc
if not math.isclose(video_shift, MINIMAX_H3_TURBO_VIDEO_SHIFT):
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}")
try:
audio_shift = float(extra.get("audio_flow_shift", self.default_audio_shift))
except (TypeError, ValueError) as exc:
raise OmniClientError(
f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}"
) from exc
if not math.isclose(audio_shift, MINIMAX_H3_TURBO_AUDIO_SHIFT):
raise OmniClientError(f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}")
def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
"""Adopt runner-installed generic Cache-DiT for request transitions."""
@@ -589,6 +674,7 @@ class MiniMaxH3Pipeline(
getattr(od_config, "task_type", None),
str(od_config.model),
)
self._turbo_lora_adapter_ids: set[int] = set()
model_root = _resolve_minimax_h3_model_root(
str(od_config.model),
od_config.revision,
@@ -781,6 +867,8 @@ class MiniMaxH3Pipeline(
self,
requested: str | None,
multi_modal_data: dict[str, Any],
*,
has_turbo_lora: bool = False,
) -> str:
if requested is None:
# A Ref2VA-only startup has no FL2VA transformer; preserve its
@@ -798,6 +886,8 @@ class MiniMaxH3Pipeline(
raise OmniClientError(
f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}"
)
if task == "ref2va" and has_turbo_lora:
raise OmniClientError("MiniMax-H3 Turbo LoRA supports T2VA/FL2VA requests only")
return task
def _resolve_shape(
@@ -1655,7 +1745,14 @@ class MiniMaxH3Pipeline(
quality = sampling.quality
logger.debug("MiniMax H3 request quality=%s", quality)
extra = sampling.extra_args or {}
task = self._resolve_task(extra.get("task"), multi_modal_data)
has_turbo_lora = self._has_active_turbo_lora(sampling)
task = self._resolve_task(
extra.get("task"),
multi_modal_data,
has_turbo_lora=has_turbo_lora,
)
if has_turbo_lora:
self._validate_turbo_sampling(sampling)
raw_image = multi_modal_data.get("image")
raw_videos = multi_modal_data.get("video")