[Feature][MiniMax-H3] Support the full LightX2V Turbo LoRA matrix (#7062)

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
汪志鹏andClaude Opus 5 authored and GitHub committed 2026-09-06 20:23:36 +08:00
1 parent 039808e0d9
commit af74a5a155
10 files changed
+557 -141

No files matched your search

+145 -30
View File
@@ -1,20 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Loader for the LightX2V Turbo MiniMax-H3 LoRA.
"""Loader for the LightX2V Turbo MiniMax-H3 LoRA family.
Keyed by ``key_format=minimax-h3-diffusers``. The native FlashGen contract lives
in ``.npu.lora``; the two artifacts differ in rank, target naming and QKV
layout, so each owns its own parsing and packing.
The published artifacts differ in task family, step count, training resolution
and LoRA alpha, so :class:`TurboSpec` carries the contract of the file actually
loaded and the pipeline validates each request against it. Only the
Diffusers-PEFT exports are served. The native FlashGen contract lives in
``.npu.lora``.
"""
from __future__ import annotations
import math
from dataclasses import dataclass, replace
from pathlib import Path
import regex as re
import torch
from safetensors import safe_open
from vllm.logger import init_logger
from vllm.lora.lora_model import LoRAModel
from vllm.lora.lora_weights import PackedLoRALayerWeights
from vllm.lora.peft_helper import PEFTHelper
@@ -22,12 +27,79 @@ from vllm.model_executor.models.utils import WeightsMapper
from vllm_omni.lora.request import LoRARequest
logger = init_logger(__name__)
_TURBO_RANK = 128
_TURBO_ALPHA = 128
_TURBO_HIDDEN_SIZE = 5376
_TURBO_ATTENTION_INNER_SIZE = 7168
_TURBO_FFN_HIDDEN_SIZE = 14336
_TURBO_FILENAME = "minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"
# ``minimax_h3_<task>_turbo_<n>step_v<major.minor>[_768p][_bf16]``. The name is
# the only place the sampler contract is recorded.
_TURBO_NAME_RE = re.compile(
r"^minimax_h3_(?P<task>fl2v|ref2v)_turbo_(?P<steps>\d+)step"
r"_v\d+\.\d+(?P<res>_768p)?(?:_bf16)?\.safetensors$"
)
# The 768p retrains moved to a shorter video flow shift; the 544p artifacts keep
# the base model's.
_TURBO_VIDEO_SHIFT_768P = 6.0
_TURBO_VIDEO_SHIFT_544P = 12.0
_TURBO_AUDIO_SHIFT = 3.0
# Only ``4step_v0.1`` declares no alpha. LightX2V's reference script never reads
# the metadata: it applies ``scale * alpha / rank`` with ``--lora-alpha``
# defaulting to 8, and its documented v0.1 command does not override that. So an
# artifact that declares nothing is driven at 8, not at its rank.
_TURBO_DEFAULT_ALPHA = 8.0
@dataclass(frozen=True)
class TurboSpec:
"""The sampler contract one Turbo artifact was distilled for."""
filename: str
task_family: str
"""``fl2v`` (serves t2va and fl2va) or ``ref2v`` (serves ref2va)."""
denoise_steps: int
video_shift: float
audio_shift: float
rank: int
alpha: float
@property
def sigma_points(self) -> int:
"""Sigma points the API contract expects: one more than the forwards."""
return self.denoise_steps + 1
@property
def supported_tasks(self) -> frozenset[str]:
if self.task_family == "ref2v":
return frozenset({"ref2va"})
return frozenset({"t2va", "fl2va"})
def parse_turbo_filename(name: str) -> TurboSpec | None:
"""Return the contract a Turbo filename encodes, or ``None``.
``alpha`` is the rank-matched default; the loader replaces it with the
artifact's declared value.
"""
match = _TURBO_NAME_RE.match(name)
if match is None:
return None
steps = int(match.group("steps"))
if steps <= 0:
return None
return TurboSpec(
filename=name,
task_family=match.group("task"),
denoise_steps=steps,
video_shift=_TURBO_VIDEO_SHIFT_768P if match.group("res") else _TURBO_VIDEO_SHIFT_544P,
audio_shift=_TURBO_AUDIO_SHIFT,
rank=_TURBO_RANK,
alpha=_TURBO_DEFAULT_ALPHA,
)
_LORA_A_SUFFIX = ".lora_A.default.weight"
_LORA_B_SUFFIX = ".lora_B.default.weight"
_TURBO_TARGETS = frozenset({"to_q", "to_k", "to_v", "out_proj", "fc1", "fc2"})
@@ -81,11 +153,20 @@ def _select_turbo_file(artifact_path: str | Path) -> Path | None:
if not path.is_dir():
return None
candidate = path / _TURBO_FILENAME
return candidate if candidate.is_file() else None
candidates = sorted(child for child in path.glob("*.safetensors") if parse_turbo_filename(child.name))
if len(candidates) == 1:
return candidates[0]
if len(candidates) > 1:
raise ValueError(
f"{path} holds {len(candidates)} MiniMax-H3 Turbo artifacts "
f"({[c.name for c in candidates[:4]]}); point --lora-path at one file."
)
return None
def _validate_and_convert_tensors(checkpoint) -> dict[str, torch.Tensor]:
"""Validate a Diffusers-layout Turbo tensor set and pack it for the manager."""
tensors: dict[str, torch.Tensor] = {}
pairs: dict[str, set[str]] = {}
raw_targets: set[str] = set()
@@ -134,7 +215,7 @@ def _validate_and_convert_tensors(checkpoint) -> dict[str, torch.Tensor]:
unexpected = sorted(raw_targets - _TURBO_EXPECTED_RAW_TARGETS)
if missing or unexpected:
raise ValueError(
"MiniMax-H3 Turbo target set does not match the supported v1.0 artifact: "
"MiniMax-H3 Turbo target set does not match the published artifact layout: "
f"missing={len(missing)} {missing[:5]}, unexpected={len(unexpected)} {unexpected[:5]}"
)
return tensors
@@ -164,39 +245,61 @@ def load_minimax_h3_turbo_lora(
lora_path: str | Path,
dtype: torch.dtype,
unsupported_offload_mode: str | None = None,
) -> tuple[LoRAModel, PEFTHelper] | None:
"""Load the published LightX2V Turbo v1.0 through the legacy manager."""
) -> tuple[LoRAModel, PEFTHelper, TurboSpec] | None:
"""Load any published LightX2V Turbo artifact through the legacy manager."""
lora_file = _select_turbo_file(lora_path)
if lora_file is None:
return None
spec = parse_turbo_filename(lora_file.name)
if spec is None:
# Not a Turbo artifact by name; leave it to the next loader.
return None
with safe_open(lora_file, framework="pt", device="cpu") as checkpoint:
# Three published artifacts declare no key_format, so absence is not an
# error; a declaration naming a different format is.
metadata = checkpoint.metadata() or {}
if metadata.get("key_format") != "minimax-h3-diffusers":
if lora_file.name == _TURBO_FILENAME:
raise ValueError(
"MiniMax-H3 Turbo v1.0 requires safetensors metadata key_format='minimax-h3-diffusers'"
)
return None
if lora_file.name != _TURBO_FILENAME:
raise ValueError(f"MiniMax-H3 Turbo supports only {_TURBO_FILENAME!r}, got {lora_file.name!r}")
declared_format = metadata.get("key_format")
if declared_format not in (None, "minimax-h3-diffusers"):
raise ValueError(
f"{lora_file.name} declares key_format={declared_format!r}, expected 'minimax-h3-diffusers'"
)
raw_alpha = metadata.get("alpha")
try:
alpha = float(raw_alpha) if raw_alpha is not None else math.nan
except ValueError as exc:
raise ValueError(f"MiniMax-H3 Turbo alpha must be numeric, got {raw_alpha!r}") from exc
if alpha != _TURBO_ALPHA:
raise ValueError(f"MiniMax-H3 Turbo v1.0 requires alpha={_TURBO_ALPHA}, got {raw_alpha!r}")
if partition == "ref2va":
raise ValueError("MiniMax-H3 Turbo LoRA supports FL2VA/T2VA only")
if raw_alpha is None:
logger.warning(
"MiniMax-H3 Turbo artifact %s declares no alpha; using the LightX2V reference "
"default alpha=%g (scale %g). Override with the request-level LoRA scale.",
spec.filename,
spec.alpha,
spec.alpha / spec.rank,
)
else:
try:
spec = replace(spec, alpha=float(raw_alpha))
except (TypeError, ValueError) as exc:
raise ValueError(f"MiniMax-H3 Turbo alpha must be numeric, got {raw_alpha!r}") from exc
if not math.isfinite(spec.alpha) or spec.alpha <= 0:
raise ValueError(f"MiniMax-H3 Turbo alpha must be a positive number, got {raw_alpha!r}")
if partition == "ref2va" and spec.task_family == "fl2v":
raise ValueError(f"{spec.filename} is an FL2VA/T2VA Turbo artifact; a Ref2VA-only server cannot serve it.")
# ``combined`` serves ref2va from ``transformers_ref``, but the LoRA
# target pattern only injects into ``transformer``: the adapter would
# bind to the stack that never runs.
if spec.task_family == "ref2v" and partition != "ref2va":
raise ValueError(
f"{spec.filename} is a Ref2VA Turbo artifact; start the server with --task-type ref2va "
f"(task_type={partition!r} serves ref2va from a DiT the adapter cannot bind to)"
)
if unsupported_offload_mode is not None:
raise ValueError(f"MiniMax-H3 Turbo dynamic LoRA does not support {unsupported_offload_mode}")
tensors = _validate_and_convert_tensors(checkpoint)
peft_helper = PEFTHelper.from_dict(
{
"r": _TURBO_RANK,
"lora_alpha": _TURBO_ALPHA,
"r": spec.rank,
"lora_alpha": spec.alpha,
"target_modules": _TURBO_TARGET_PATTERN,
}
)
@@ -209,4 +312,16 @@ def load_minimax_h3_turbo_lora(
weights_mapper=_TURBO_WEIGHTS_MAPPER,
)
_pack_h3_turbo_fc1(lora_model)
return lora_model, peft_helper
logger.info(
"MiniMax-H3 Turbo adapter %s: task=%s, %d denoiser forwards (%d sigma points), "
"rank=%d alpha=%g, flow_shift=%g/%g",
spec.filename,
spec.task_family,
spec.denoise_steps,
spec.sigma_points,
spec.rank,
spec.alpha,
spec.video_shift,
spec.audio_shift,
)
return lora_model, peft_helper, spec
@@ -125,7 +125,7 @@ from .denoise_loop import (
)
from .encoder import MiniMaxH3Qwen3VLEncoder
from .fasth3 import FastH3WeightFusion, resolve_fasth3_fusion
from .lora import load_minimax_h3_turbo_lora
from .lora import TurboSpec, load_minimax_h3_turbo_lora
from .minimax_h3_transformer import (
MiniMaxH3Attention,
MiniMaxH3DiTModel,
@@ -188,9 +188,6 @@ MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
MINIMAX_H3_TURBO_SIGMA_POINTS = 5
MINIMAX_H3_TURBO_VIDEO_SHIFT = 6.0
MINIMAX_H3_TURBO_AUDIO_SHIFT = 3.0
MINIMAX_H3_DOWNLOAD_PATTERNS = [
"FL2VA/**",
"Ref2VA/model_index.json",
@@ -706,7 +703,7 @@ class MiniMaxH3Pipeline(
) -> tuple[LoRAModel, PEFTHelper] | None:
# A cache eviction may be followed by a different adapter reusing the
# same client-supplied ID. Every real load replaces the classification.
self._turbo_lora_adapter_ids.discard(lora_request.lora_int_id)
self._turbo_lora_specs.pop(lora_request.lora_int_id, None)
self._native_lora_adapter_ids.discard(lora_request.lora_int_id)
self._lora_sigma_schedules.pop(lora_request.lora_int_id, None)
od_config = getattr(self, "od_config", None)
@@ -726,8 +723,9 @@ class MiniMaxH3Pipeline(
unsupported_offload_mode=" or ".join(offload_modes) or None,
)
if loaded is not None:
self._turbo_lora_adapter_ids.add(lora_request.lora_int_id)
return loaded
lora_model, peft_helper, turbo_spec = loaded
self._turbo_lora_specs[lora_request.lora_int_id] = turbo_spec
return lora_model, peft_helper
# Selection is by the artifact's safetensors ``key_format``, not by the
# running platform: the native loader is checkpoint-format parsing with
@@ -753,7 +751,7 @@ class MiniMaxH3Pipeline(
lora_model: LoRAModel,
bound_lora_names: frozenset[str],
) -> None:
if lora_model.id in self._turbo_lora_adapter_ids:
if lora_model.id in self._turbo_lora_specs:
missing = sorted(set(lora_model.loras) - bound_lora_names)
if missing:
raise ValueError(
@@ -770,14 +768,6 @@ class MiniMaxH3Pipeline(
f"bound={len(bound_lora_names)}/{len(lora_model.loras)}, missing={missing[:5]}"
)
def _has_active_turbo_lora(self, sampling: Any) -> bool:
lora_request = sampling.lora_request
return (
lora_request is not None
and not math.isclose(0.0, float(sampling.lora_scale))
and lora_request.lora_int_id in self._turbo_lora_adapter_ids
)
def _has_active_native_lora(self, sampling: Any) -> bool:
lora_request = sampling.lora_request
return (
@@ -837,27 +827,46 @@ class MiniMaxH3Pipeline(
return adapter_schedule
return self._base_schedule_for_task(task)
def _validate_turbo_sampling(self, sampling: Any) -> None:
def _active_turbo_spec(self, sampling: Any) -> TurboSpec | None:
"""Return the spec of the Turbo adapter this request actually applies.
A recognized adapter at scale 0 contributes nothing, so it neither
constrains the task nor imposes its sampler contract.
"""
lora_request = sampling.lora_request
if lora_request is None or math.isclose(0.0, float(sampling.lora_scale)):
return None
return self._turbo_lora_specs.get(lora_request.lora_int_id)
def _validate_turbo_sampling(self, sampling: Any, spec: TurboSpec) -> None:
"""Hold a request to the contract of the artifact that is loaded.
Sigma-point count and both flow shifts vary across the Turbo family, so
each is checked against the adapter's own spec rather than a single
published configuration.
"""
extra = sampling.extra_args or {}
sigma_points = sampling.num_inference_steps
if sigma_points != MINIMAX_H3_TURBO_SIGMA_POINTS:
if sigma_points != spec.sigma_points:
raise OmniClientError(
"MiniMax-H3 Turbo requires num_inference_steps=5 (five sigma points produce four denoiser evaluations)"
f"{spec.filename} is a {spec.denoise_steps}-step artifact and requires "
f"num_inference_steps={spec.sigma_points} "
f"({spec.sigma_points} sigma points produce {spec.denoise_steps} denoiser evaluations)"
)
try:
video_shift = float(extra.get("flow_shift", self.default_video_shift))
except (TypeError, ValueError) as exc:
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}") from exc
if not math.isclose(video_shift, MINIMAX_H3_TURBO_VIDEO_SHIFT):
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}")
raise OmniClientError(f"{spec.filename} requires flow_shift={spec.video_shift:g}") from exc
if not math.isclose(video_shift, spec.video_shift):
raise OmniClientError(f"{spec.filename} requires flow_shift={spec.video_shift:g}")
try:
audio_shift = float(extra.get("audio_flow_shift", self.default_audio_shift))
except (TypeError, ValueError) as exc:
raise OmniClientError(
f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}"
) from exc
if not math.isclose(audio_shift, MINIMAX_H3_TURBO_AUDIO_SHIFT):
raise OmniClientError(f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}")
raise OmniClientError(f"{spec.filename} requires audio_flow_shift={spec.audio_shift:g}") from exc
if not math.isclose(audio_shift, spec.audio_shift):
raise OmniClientError(f"{spec.filename} requires audio_flow_shift={spec.audio_shift:g}")
def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
"""Adopt runner-installed generic Cache-DiT for request transitions."""
@@ -890,7 +899,7 @@ class MiniMaxH3Pipeline(
getattr(od_config, "task_type", None),
str(od_config.model),
)
self._turbo_lora_adapter_ids: set[int] = set()
self._turbo_lora_specs: dict[int, TurboSpec] = {}
self._native_lora_adapter_ids: set[int] = set()
self._lora_sigma_schedules: dict[int, DMD2SigmaSchedule] = {}
model_root = _resolve_minimax_h3_model_root(
@@ -1177,7 +1186,7 @@ class MiniMaxH3Pipeline(
requested: str | None,
multi_modal_data: dict[str, Any],
*,
has_turbo_lora: bool = False,
turbo_spec: TurboSpec | None = None,
has_native_lora: bool = False,
) -> str:
if requested is None:
@@ -1196,8 +1205,11 @@ class MiniMaxH3Pipeline(
raise OmniClientError(
f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}"
)
if task == "ref2va" and has_turbo_lora:
raise OmniClientError("MiniMax-H3 Turbo LoRA supports T2VA/FL2VA requests only")
if turbo_spec is not None and task not in turbo_spec.supported_tasks:
raise OmniClientError(
f"{turbo_spec.filename} is a {turbo_spec.task_family} Turbo artifact and serves "
f"{sorted(turbo_spec.supported_tasks)}, got task={task!r}"
)
if has_native_lora and task != "t2va":
raise OmniClientError("MiniMax-H3 native LoRA supports T2VA requests only")
if self._fasth3 is not None:
@@ -2216,16 +2228,16 @@ class MiniMaxH3Pipeline(
quality = sampling.quality
logger.debug("MiniMax H3 request quality=%s", quality)
extra = sampling.extra_args or {}
has_turbo_lora = self._has_active_turbo_lora(sampling)
turbo_spec = self._active_turbo_spec(sampling)
has_native_lora = self._has_active_native_lora(sampling)
task = self._resolve_task(
extra.get("task"),
multi_modal_data,
has_turbo_lora=has_turbo_lora,
turbo_spec=turbo_spec,
has_native_lora=has_native_lora,
)
if has_turbo_lora:
self._validate_turbo_sampling(sampling)
if turbo_spec is not None:
self._validate_turbo_sampling(sampling, turbo_spec)
if has_native_lora:
self._validate_native_sampling(sampling, task=task)
if self._fasth3 is not None: