[Feature] Support MiniMax-H3 Turbo LoRA with the legacy manager (#6476)
Signed-off-by: mglyn <1203789601@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
1c917b2751
commit
3d035bfa19
8 files changed
+956
-71
No files matched your search
@@ -0,0 +1,205 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from vllm.lora.lora_model import LoRAModel
|
||||
from vllm.lora.lora_weights import PackedLoRALayerWeights
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
|
||||
_TURBO_RANK = 128
|
||||
_TURBO_ALPHA = 128
|
||||
_TURBO_HIDDEN_SIZE = 5376
|
||||
_TURBO_ATTENTION_INNER_SIZE = 7168
|
||||
_TURBO_FFN_HIDDEN_SIZE = 14336
|
||||
_TURBO_FILENAME = "minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"
|
||||
_LORA_A_SUFFIX = ".lora_A.default.weight"
|
||||
_LORA_B_SUFFIX = ".lora_B.default.weight"
|
||||
_TURBO_TARGETS = frozenset({"to_q", "to_k", "to_v", "out_proj", "fc1", "fc2"})
|
||||
_TURBO_RAW_TARGET_SUFFIXES = (
|
||||
"attn.to_q",
|
||||
"attn.to_k",
|
||||
"attn.to_v",
|
||||
"attn.to_out.0",
|
||||
"ff.net.0.proj",
|
||||
"ff.net.2",
|
||||
)
|
||||
_TURBO_TARGET_DIMS = {
|
||||
"attn.to_q": (_TURBO_HIDDEN_SIZE, _TURBO_ATTENTION_INNER_SIZE),
|
||||
"attn.to_k": (_TURBO_HIDDEN_SIZE, _TURBO_ATTENTION_INNER_SIZE),
|
||||
"attn.to_v": (_TURBO_HIDDEN_SIZE, _TURBO_ATTENTION_INNER_SIZE),
|
||||
"attn.to_out.0": (_TURBO_ATTENTION_INNER_SIZE, _TURBO_HIDDEN_SIZE),
|
||||
"ff.net.0.proj": (_TURBO_HIDDEN_SIZE, 2 * _TURBO_FFN_HIDDEN_SIZE),
|
||||
"ff.net.2": (_TURBO_FFN_HIDDEN_SIZE, _TURBO_HIDDEN_SIZE),
|
||||
}
|
||||
_TURBO_EXPECTED_RAW_TARGETS = frozenset(
|
||||
f"{prefix}.{block_index}.{suffix}"
|
||||
for prefix, block_count in (
|
||||
("transformer_blocks", 50),
|
||||
("token_refiner.refiner_blocks", 2),
|
||||
)
|
||||
for block_index in range(block_count)
|
||||
for suffix in _TURBO_RAW_TARGET_SUFFIXES
|
||||
)
|
||||
_TURBO_TARGET_PATTERN = (
|
||||
r"^transformer\.(?:token_refiner\.blocks|blocks)\.\d+\."
|
||||
r"(?:attn\.(?:to_q|to_k|to_v|out_proj)|mlp\.(?:fc1|fc2))$"
|
||||
)
|
||||
|
||||
_TURBO_WEIGHTS_MAPPER = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"token_refiner.refiner_blocks.": "token_refiner.blocks.",
|
||||
"transformer_blocks.": "blocks.",
|
||||
".attn.to_out.0.": ".attn.out_proj.",
|
||||
".ff.net.0.proj.": ".mlp.fc1.",
|
||||
".ff.net.2.": ".mlp.fc2.",
|
||||
".lora_A.default.": ".lora_A.",
|
||||
".lora_B.default.": ".lora_B.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _select_turbo_file(artifact_path: str | Path) -> Path | None:
|
||||
path = Path(artifact_path)
|
||||
if path.is_file():
|
||||
return path if path.suffix == ".safetensors" else None
|
||||
if not path.is_dir():
|
||||
return None
|
||||
|
||||
candidate = path / _TURBO_FILENAME
|
||||
return candidate if candidate.is_file() else None
|
||||
|
||||
|
||||
def _validate_and_convert_tensors(checkpoint) -> dict[str, torch.Tensor]:
|
||||
tensors: dict[str, torch.Tensor] = {}
|
||||
pairs: dict[str, set[str]] = {}
|
||||
raw_targets: set[str] = set()
|
||||
for name in checkpoint.keys():
|
||||
if name.endswith(_LORA_A_SUFFIX):
|
||||
raw_target = name[: -len(_LORA_A_SUFFIX)]
|
||||
side = "a"
|
||||
elif name.endswith(_LORA_B_SUFFIX):
|
||||
raw_target = name[: -len(_LORA_B_SUFFIX)]
|
||||
side = "b"
|
||||
else:
|
||||
raise ValueError(f"Unconsumed MiniMax-H3 Turbo tensor: {name!r}")
|
||||
raw_targets.add(raw_target)
|
||||
|
||||
mapped_name = _TURBO_WEIGHTS_MAPPER.apply_list([name])[0]
|
||||
mapped_target = mapped_name.rsplit(".lora_", 1)[0]
|
||||
if mapped_target.rsplit(".", 1)[-1] not in _TURBO_TARGETS:
|
||||
raise ValueError(f"Unsupported MiniMax-H3 Turbo target: {raw_target!r}")
|
||||
target_sides = pairs.setdefault(mapped_target, set())
|
||||
if side in target_sides:
|
||||
raise ValueError(f"Duplicate MiniMax-H3 Turbo tensor for {mapped_target}.{side}")
|
||||
target_sides.add(side)
|
||||
|
||||
tensor = checkpoint.get_tensor(name)
|
||||
if tensor.ndim != 2:
|
||||
raise ValueError(f"MiniMax-H3 Turbo LoRA tensors must be matrices, got {name}={tuple(tensor.shape)}")
|
||||
suffix = next((suffix for suffix in _TURBO_RAW_TARGET_SUFFIXES if raw_target.endswith(suffix)), None)
|
||||
if suffix is None:
|
||||
raise ValueError(f"MiniMax-H3 Turbo LoRA contains unsupported target: {raw_target}")
|
||||
input_dim, output_dim = _TURBO_TARGET_DIMS[suffix]
|
||||
expected_shape = (_TURBO_RANK, input_dim) if side == "a" else (output_dim, _TURBO_RANK)
|
||||
if tuple(tensor.shape) != expected_shape:
|
||||
raise ValueError(
|
||||
f"MiniMax-H3 Turbo tensor has invalid global shape: {name}={tuple(tensor.shape)}, "
|
||||
f"expected={expected_shape}"
|
||||
)
|
||||
if side == "b" and ".ff.net.0.proj." in name:
|
||||
value, gate = tensor.chunk(2, dim=0)
|
||||
tensor = torch.cat((gate, value), dim=0).contiguous()
|
||||
tensors[name] = tensor
|
||||
|
||||
incomplete = sorted(target for target, sides in pairs.items() if sides != {"a", "b"})
|
||||
if incomplete:
|
||||
raise ValueError(f"Incomplete MiniMax-H3 Turbo LoRA pairs: {incomplete}")
|
||||
missing = sorted(_TURBO_EXPECTED_RAW_TARGETS - raw_targets)
|
||||
unexpected = sorted(raw_targets - _TURBO_EXPECTED_RAW_TARGETS)
|
||||
if missing or unexpected:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Turbo target set does not match the supported v1.0 artifact: "
|
||||
f"missing={len(missing)} {missing[:5]}, unexpected={len(unexpected)} {unexpected[:5]}"
|
||||
)
|
||||
return tensors
|
||||
|
||||
|
||||
def _pack_h3_turbo_fc1(lora_model: LoRAModel) -> None:
|
||||
"""Represent H3's fused gate/up projection without generic layout guesses."""
|
||||
|
||||
for module_name, weights in tuple(lora_model.loras.items()):
|
||||
if not module_name.endswith(".mlp.fc1"):
|
||||
continue
|
||||
gate_b, up_b = weights.lora_b.chunk(2, dim=0)
|
||||
lora_model.loras[module_name] = PackedLoRALayerWeights(
|
||||
module_name=module_name,
|
||||
rank=weights.rank,
|
||||
lora_alphas=[weights.lora_alpha, weights.lora_alpha],
|
||||
lora_a=[weights.lora_a, weights.lora_a],
|
||||
lora_b=[gate_b.contiguous(), up_b.contiguous()],
|
||||
scaling=[weights.scaling, weights.scaling],
|
||||
)
|
||||
|
||||
|
||||
def load_minimax_h3_turbo_lora(
|
||||
*,
|
||||
partition: str,
|
||||
lora_request: LoRARequest,
|
||||
lora_path: str | Path,
|
||||
dtype: torch.dtype,
|
||||
unsupported_offload_mode: str | None = None,
|
||||
) -> tuple[LoRAModel, PEFTHelper] | None:
|
||||
"""Load the published LightX2V Turbo v1.0 through the legacy manager."""
|
||||
|
||||
lora_file = _select_turbo_file(lora_path)
|
||||
if lora_file is None:
|
||||
return None
|
||||
with safe_open(lora_file, framework="pt", device="cpu") as checkpoint:
|
||||
metadata = checkpoint.metadata() or {}
|
||||
if metadata.get("key_format") != "minimax-h3-diffusers":
|
||||
if lora_file.name == _TURBO_FILENAME:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Turbo v1.0 requires safetensors metadata key_format='minimax-h3-diffusers'"
|
||||
)
|
||||
return None
|
||||
if lora_file.name != _TURBO_FILENAME:
|
||||
raise ValueError(f"MiniMax-H3 Turbo supports only {_TURBO_FILENAME!r}, got {lora_file.name!r}")
|
||||
raw_alpha = metadata.get("alpha")
|
||||
try:
|
||||
alpha = float(raw_alpha) if raw_alpha is not None else math.nan
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"MiniMax-H3 Turbo alpha must be numeric, got {raw_alpha!r}") from exc
|
||||
if alpha != _TURBO_ALPHA:
|
||||
raise ValueError(f"MiniMax-H3 Turbo v1.0 requires alpha={_TURBO_ALPHA}, got {raw_alpha!r}")
|
||||
if partition == "ref2va":
|
||||
raise ValueError("MiniMax-H3 Turbo LoRA supports FL2VA/T2VA only")
|
||||
if unsupported_offload_mode is not None:
|
||||
raise ValueError(f"MiniMax-H3 Turbo dynamic LoRA does not support {unsupported_offload_mode}")
|
||||
tensors = _validate_and_convert_tensors(checkpoint)
|
||||
|
||||
peft_helper = PEFTHelper.from_dict(
|
||||
{
|
||||
"r": _TURBO_RANK,
|
||||
"lora_alpha": _TURBO_ALPHA,
|
||||
"target_modules": _TURBO_TARGET_PATTERN,
|
||||
}
|
||||
)
|
||||
lora_model = LoRAModel.from_lora_tensors(
|
||||
lora_model_id=lora_request.lora_int_id,
|
||||
tensors=tensors,
|
||||
peft_helper=peft_helper,
|
||||
device="cpu",
|
||||
dtype=dtype,
|
||||
weights_mapper=_TURBO_WEIGHTS_MAPPER,
|
||||
)
|
||||
_pack_h3_turbo_fc1(lora_model)
|
||||
return lora_model, peft_helper
|
||||
@@ -954,6 +954,14 @@ class MiniMaxH3DiTModel(nn.Module):
|
||||
# quantization or LoRA to map onto. Address the fused layers directly, e.g.
|
||||
# ignored_layers=["blocks.0.attn.qkv_proj"].
|
||||
packed_modules_mapping = {}
|
||||
# Turbo LoRA checkpoints publish separate Q/K/V adapters. This declaration
|
||||
# lets the legacy diffusion LoRA manager bind them to the packed QKV layer;
|
||||
# it does not change the fused base-checkpoint loading path above.
|
||||
stacked_params_mapping = (
|
||||
(".attn.qkv_proj", ".attn.to_q", "q"),
|
||||
(".attn.qkv_proj", ".attn.to_k", "k"),
|
||||
(".attn.qkv_proj", ".attn.to_v", "v"),
|
||||
)
|
||||
|
||||
def _validate_tp_config(self, *, arch: MiniMaxH3DiTArchConfig, tp_size: int) -> None:
|
||||
if tp_size < 1:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""vLLM-Omni pipeline for MiniMax H3 FL2VA and Ref2VA partitions."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +12,7 @@ from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from itertools import groupby
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -74,6 +75,7 @@ from .condition_noise import (
|
||||
)
|
||||
from .denoise_loop import MiniMaxH3DenoiseBranch, minimax_h3_denoise_loop
|
||||
from .encoder import MiniMaxH3Qwen3VLEncoder
|
||||
from .lora import load_minimax_h3_turbo_lora
|
||||
from .minimax_h3_transformer import MiniMaxH3DiTModel
|
||||
from .packed_sequence import (
|
||||
minimax_h3_packed_sequence,
|
||||
@@ -110,6 +112,12 @@ from .vae import MiniMaxH3AudioVAE, MiniMaxH3VideoVAE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.lora.lora_model import LoRAModel
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
|
||||
MINIMAX_H3_FPS = 24
|
||||
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
|
||||
MINIMAX_H3_IMGVID_COND_TIMESTEP = 0.999
|
||||
@@ -130,6 +138,9 @@ MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
|
||||
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
|
||||
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
|
||||
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
|
||||
MINIMAX_H3_TURBO_SIGMA_POINTS = 5
|
||||
MINIMAX_H3_TURBO_VIDEO_SHIFT = 6.0
|
||||
MINIMAX_H3_TURBO_AUDIO_SHIFT = 3.0
|
||||
MINIMAX_H3_DOWNLOAD_PATTERNS = [
|
||||
"FL2VA/**",
|
||||
"Ref2VA/model_index.json",
|
||||
@@ -559,6 +570,80 @@ class MiniMaxH3Pipeline(
|
||||
# uniform path available to partially constructed pipelines.
|
||||
_base_schedule_by_partition: ClassVar[Mapping[str, DMD2SigmaSchedule | None]] = {}
|
||||
|
||||
def _load_diffusion_lora_adapter(
|
||||
self,
|
||||
*,
|
||||
lora_request: LoRARequest,
|
||||
lora_path: str | Path,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[LoRAModel, PEFTHelper] | None:
|
||||
# A cache eviction may be followed by a different adapter reusing the
|
||||
# same client-supplied ID. Every real load replaces the classification.
|
||||
self._turbo_lora_adapter_ids.discard(lora_request.lora_int_id)
|
||||
od_config = getattr(self, "od_config", None)
|
||||
offload_modes = []
|
||||
if getattr(od_config, "enable_cpu_offload", False):
|
||||
offload_modes.append("model-level CPU offload (--enable-cpu-offload)")
|
||||
if getattr(od_config, "enable_layerwise_offload", False):
|
||||
offload_modes.append("layerwise offload (--enable-layerwise-offload)")
|
||||
if getattr(od_config, "enable_distributed_layerwise_offload", False):
|
||||
offload_modes.append("distributed layerwise offload (--enable-distributed-layerwise-offload)")
|
||||
loaded = load_minimax_h3_turbo_lora(
|
||||
partition=self.partition,
|
||||
lora_request=lora_request,
|
||||
lora_path=lora_path,
|
||||
dtype=dtype,
|
||||
unsupported_offload_mode=" or ".join(offload_modes) or None,
|
||||
)
|
||||
if loaded is not None:
|
||||
self._turbo_lora_adapter_ids.add(lora_request.lora_int_id)
|
||||
return loaded
|
||||
|
||||
def _validate_diffusion_lora_binding(
|
||||
self,
|
||||
*,
|
||||
lora_model: LoRAModel,
|
||||
bound_lora_names: frozenset[str],
|
||||
) -> None:
|
||||
if lora_model.id not in self._turbo_lora_adapter_ids:
|
||||
return
|
||||
missing = sorted(set(lora_model.loras) - bound_lora_names)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Turbo LoRA binding is incomplete: "
|
||||
f"bound={len(bound_lora_names)}/{len(lora_model.loras)}, missing={missing[:5]}"
|
||||
)
|
||||
|
||||
def _has_active_turbo_lora(self, sampling: Any) -> bool:
|
||||
lora_request = sampling.lora_request
|
||||
return (
|
||||
lora_request is not None
|
||||
and not math.isclose(0.0, float(sampling.lora_scale))
|
||||
and lora_request.lora_int_id in self._turbo_lora_adapter_ids
|
||||
)
|
||||
|
||||
def _validate_turbo_sampling(self, sampling: Any) -> None:
|
||||
extra = sampling.extra_args or {}
|
||||
sigma_points = sampling.num_inference_steps
|
||||
if sigma_points != MINIMAX_H3_TURBO_SIGMA_POINTS:
|
||||
raise OmniClientError(
|
||||
"MiniMax-H3 Turbo requires num_inference_steps=5 (five sigma points produce four denoiser evaluations)"
|
||||
)
|
||||
try:
|
||||
video_shift = float(extra.get("flow_shift", self.default_video_shift))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}") from exc
|
||||
if not math.isclose(video_shift, MINIMAX_H3_TURBO_VIDEO_SHIFT):
|
||||
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}")
|
||||
try:
|
||||
audio_shift = float(extra.get("audio_flow_shift", self.default_audio_shift))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise OmniClientError(
|
||||
f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}"
|
||||
) from exc
|
||||
if not math.isclose(audio_shift, MINIMAX_H3_TURBO_AUDIO_SHIFT):
|
||||
raise OmniClientError(f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}")
|
||||
|
||||
def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
|
||||
"""Adopt runner-installed generic Cache-DiT for request transitions."""
|
||||
|
||||
@@ -589,6 +674,7 @@ class MiniMaxH3Pipeline(
|
||||
getattr(od_config, "task_type", None),
|
||||
str(od_config.model),
|
||||
)
|
||||
self._turbo_lora_adapter_ids: set[int] = set()
|
||||
model_root = _resolve_minimax_h3_model_root(
|
||||
str(od_config.model),
|
||||
od_config.revision,
|
||||
@@ -781,6 +867,8 @@ class MiniMaxH3Pipeline(
|
||||
self,
|
||||
requested: str | None,
|
||||
multi_modal_data: dict[str, Any],
|
||||
*,
|
||||
has_turbo_lora: bool = False,
|
||||
) -> str:
|
||||
if requested is None:
|
||||
# A Ref2VA-only startup has no FL2VA transformer; preserve its
|
||||
@@ -798,6 +886,8 @@ class MiniMaxH3Pipeline(
|
||||
raise OmniClientError(
|
||||
f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}"
|
||||
)
|
||||
if task == "ref2va" and has_turbo_lora:
|
||||
raise OmniClientError("MiniMax-H3 Turbo LoRA supports T2VA/FL2VA requests only")
|
||||
return task
|
||||
|
||||
def _resolve_shape(
|
||||
@@ -1655,7 +1745,14 @@ class MiniMaxH3Pipeline(
|
||||
quality = sampling.quality
|
||||
logger.debug("MiniMax H3 request quality=%s", quality)
|
||||
extra = sampling.extra_args or {}
|
||||
task = self._resolve_task(extra.get("task"), multi_modal_data)
|
||||
has_turbo_lora = self._has_active_turbo_lora(sampling)
|
||||
task = self._resolve_task(
|
||||
extra.get("task"),
|
||||
multi_modal_data,
|
||||
has_turbo_lora=has_turbo_lora,
|
||||
)
|
||||
if has_turbo_lora:
|
||||
self._validate_turbo_sampling(sampling)
|
||||
|
||||
raw_image = multi_modal_data.get("image")
|
||||
raw_videos = multi_modal_data.get("video")
|
||||
|
||||
Reference in new issue
Block a user