[Feature][MiniMax-H3] Support the full LightX2V Turbo LoRA matrix (#7062)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
039808e0d9
commit
af74a5a155
10 files changed
+557
-141
No files matched your search
@@ -1,20 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Loader for the LightX2V Turbo MiniMax-H3 LoRA.
|
||||
"""Loader for the LightX2V Turbo MiniMax-H3 LoRA family.
|
||||
|
||||
Keyed by ``key_format=minimax-h3-diffusers``. The native FlashGen contract lives
|
||||
in ``.npu.lora``; the two artifacts differ in rank, target naming and QKV
|
||||
layout, so each owns its own parsing and packing.
|
||||
The published artifacts differ in task family, step count, training resolution
|
||||
and LoRA alpha, so :class:`TurboSpec` carries the contract of the file actually
|
||||
loaded and the pipeline validates each request against it. Only the
|
||||
Diffusers-PEFT exports are served. The native FlashGen contract lives in
|
||||
``.npu.lora``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from safetensors import safe_open
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.lora_model import LoRAModel
|
||||
from vllm.lora.lora_weights import PackedLoRALayerWeights
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
@@ -22,12 +27,79 @@ from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_TURBO_RANK = 128
|
||||
_TURBO_ALPHA = 128
|
||||
_TURBO_HIDDEN_SIZE = 5376
|
||||
_TURBO_ATTENTION_INNER_SIZE = 7168
|
||||
_TURBO_FFN_HIDDEN_SIZE = 14336
|
||||
_TURBO_FILENAME = "minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"
|
||||
# ``minimax_h3_<task>_turbo_<n>step_v<major.minor>[_768p][_bf16]``. The name is
|
||||
# the only place the sampler contract is recorded.
|
||||
_TURBO_NAME_RE = re.compile(
|
||||
r"^minimax_h3_(?P<task>fl2v|ref2v)_turbo_(?P<steps>\d+)step"
|
||||
r"_v\d+\.\d+(?P<res>_768p)?(?:_bf16)?\.safetensors$"
|
||||
)
|
||||
# The 768p retrains moved to a shorter video flow shift; the 544p artifacts keep
|
||||
# the base model's.
|
||||
_TURBO_VIDEO_SHIFT_768P = 6.0
|
||||
_TURBO_VIDEO_SHIFT_544P = 12.0
|
||||
_TURBO_AUDIO_SHIFT = 3.0
|
||||
# Only ``4step_v0.1`` declares no alpha. LightX2V's reference script never reads
|
||||
# the metadata: it applies ``scale * alpha / rank`` with ``--lora-alpha``
|
||||
# defaulting to 8, and its documented v0.1 command does not override that. So an
|
||||
# artifact that declares nothing is driven at 8, not at its rank.
|
||||
_TURBO_DEFAULT_ALPHA = 8.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TurboSpec:
|
||||
"""The sampler contract one Turbo artifact was distilled for."""
|
||||
|
||||
filename: str
|
||||
task_family: str
|
||||
"""``fl2v`` (serves t2va and fl2va) or ``ref2v`` (serves ref2va)."""
|
||||
denoise_steps: int
|
||||
video_shift: float
|
||||
audio_shift: float
|
||||
rank: int
|
||||
alpha: float
|
||||
|
||||
@property
|
||||
def sigma_points(self) -> int:
|
||||
"""Sigma points the API contract expects: one more than the forwards."""
|
||||
return self.denoise_steps + 1
|
||||
|
||||
@property
|
||||
def supported_tasks(self) -> frozenset[str]:
|
||||
if self.task_family == "ref2v":
|
||||
return frozenset({"ref2va"})
|
||||
return frozenset({"t2va", "fl2va"})
|
||||
|
||||
|
||||
def parse_turbo_filename(name: str) -> TurboSpec | None:
|
||||
"""Return the contract a Turbo filename encodes, or ``None``.
|
||||
|
||||
``alpha`` is the rank-matched default; the loader replaces it with the
|
||||
artifact's declared value.
|
||||
"""
|
||||
|
||||
match = _TURBO_NAME_RE.match(name)
|
||||
if match is None:
|
||||
return None
|
||||
steps = int(match.group("steps"))
|
||||
if steps <= 0:
|
||||
return None
|
||||
return TurboSpec(
|
||||
filename=name,
|
||||
task_family=match.group("task"),
|
||||
denoise_steps=steps,
|
||||
video_shift=_TURBO_VIDEO_SHIFT_768P if match.group("res") else _TURBO_VIDEO_SHIFT_544P,
|
||||
audio_shift=_TURBO_AUDIO_SHIFT,
|
||||
rank=_TURBO_RANK,
|
||||
alpha=_TURBO_DEFAULT_ALPHA,
|
||||
)
|
||||
|
||||
|
||||
_LORA_A_SUFFIX = ".lora_A.default.weight"
|
||||
_LORA_B_SUFFIX = ".lora_B.default.weight"
|
||||
_TURBO_TARGETS = frozenset({"to_q", "to_k", "to_v", "out_proj", "fc1", "fc2"})
|
||||
@@ -81,11 +153,20 @@ def _select_turbo_file(artifact_path: str | Path) -> Path | None:
|
||||
if not path.is_dir():
|
||||
return None
|
||||
|
||||
candidate = path / _TURBO_FILENAME
|
||||
return candidate if candidate.is_file() else None
|
||||
candidates = sorted(child for child in path.glob("*.safetensors") if parse_turbo_filename(child.name))
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
if len(candidates) > 1:
|
||||
raise ValueError(
|
||||
f"{path} holds {len(candidates)} MiniMax-H3 Turbo artifacts "
|
||||
f"({[c.name for c in candidates[:4]]}); point --lora-path at one file."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _validate_and_convert_tensors(checkpoint) -> dict[str, torch.Tensor]:
|
||||
"""Validate a Diffusers-layout Turbo tensor set and pack it for the manager."""
|
||||
|
||||
tensors: dict[str, torch.Tensor] = {}
|
||||
pairs: dict[str, set[str]] = {}
|
||||
raw_targets: set[str] = set()
|
||||
@@ -134,7 +215,7 @@ def _validate_and_convert_tensors(checkpoint) -> dict[str, torch.Tensor]:
|
||||
unexpected = sorted(raw_targets - _TURBO_EXPECTED_RAW_TARGETS)
|
||||
if missing or unexpected:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Turbo target set does not match the supported v1.0 artifact: "
|
||||
"MiniMax-H3 Turbo target set does not match the published artifact layout: "
|
||||
f"missing={len(missing)} {missing[:5]}, unexpected={len(unexpected)} {unexpected[:5]}"
|
||||
)
|
||||
return tensors
|
||||
@@ -164,39 +245,61 @@ def load_minimax_h3_turbo_lora(
|
||||
lora_path: str | Path,
|
||||
dtype: torch.dtype,
|
||||
unsupported_offload_mode: str | None = None,
|
||||
) -> tuple[LoRAModel, PEFTHelper] | None:
|
||||
"""Load the published LightX2V Turbo v1.0 through the legacy manager."""
|
||||
) -> tuple[LoRAModel, PEFTHelper, TurboSpec] | None:
|
||||
"""Load any published LightX2V Turbo artifact through the legacy manager."""
|
||||
|
||||
lora_file = _select_turbo_file(lora_path)
|
||||
if lora_file is None:
|
||||
return None
|
||||
spec = parse_turbo_filename(lora_file.name)
|
||||
if spec is None:
|
||||
# Not a Turbo artifact by name; leave it to the next loader.
|
||||
return None
|
||||
with safe_open(lora_file, framework="pt", device="cpu") as checkpoint:
|
||||
# Three published artifacts declare no key_format, so absence is not an
|
||||
# error; a declaration naming a different format is.
|
||||
metadata = checkpoint.metadata() or {}
|
||||
if metadata.get("key_format") != "minimax-h3-diffusers":
|
||||
if lora_file.name == _TURBO_FILENAME:
|
||||
raise ValueError(
|
||||
"MiniMax-H3 Turbo v1.0 requires safetensors metadata key_format='minimax-h3-diffusers'"
|
||||
)
|
||||
return None
|
||||
if lora_file.name != _TURBO_FILENAME:
|
||||
raise ValueError(f"MiniMax-H3 Turbo supports only {_TURBO_FILENAME!r}, got {lora_file.name!r}")
|
||||
declared_format = metadata.get("key_format")
|
||||
if declared_format not in (None, "minimax-h3-diffusers"):
|
||||
raise ValueError(
|
||||
f"{lora_file.name} declares key_format={declared_format!r}, expected 'minimax-h3-diffusers'"
|
||||
)
|
||||
raw_alpha = metadata.get("alpha")
|
||||
try:
|
||||
alpha = float(raw_alpha) if raw_alpha is not None else math.nan
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"MiniMax-H3 Turbo alpha must be numeric, got {raw_alpha!r}") from exc
|
||||
if alpha != _TURBO_ALPHA:
|
||||
raise ValueError(f"MiniMax-H3 Turbo v1.0 requires alpha={_TURBO_ALPHA}, got {raw_alpha!r}")
|
||||
if partition == "ref2va":
|
||||
raise ValueError("MiniMax-H3 Turbo LoRA supports FL2VA/T2VA only")
|
||||
if raw_alpha is None:
|
||||
logger.warning(
|
||||
"MiniMax-H3 Turbo artifact %s declares no alpha; using the LightX2V reference "
|
||||
"default alpha=%g (scale %g). Override with the request-level LoRA scale.",
|
||||
spec.filename,
|
||||
spec.alpha,
|
||||
spec.alpha / spec.rank,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
spec = replace(spec, alpha=float(raw_alpha))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"MiniMax-H3 Turbo alpha must be numeric, got {raw_alpha!r}") from exc
|
||||
if not math.isfinite(spec.alpha) or spec.alpha <= 0:
|
||||
raise ValueError(f"MiniMax-H3 Turbo alpha must be a positive number, got {raw_alpha!r}")
|
||||
|
||||
if partition == "ref2va" and spec.task_family == "fl2v":
|
||||
raise ValueError(f"{spec.filename} is an FL2VA/T2VA Turbo artifact; a Ref2VA-only server cannot serve it.")
|
||||
# ``combined`` serves ref2va from ``transformers_ref``, but the LoRA
|
||||
# target pattern only injects into ``transformer``: the adapter would
|
||||
# bind to the stack that never runs.
|
||||
if spec.task_family == "ref2v" and partition != "ref2va":
|
||||
raise ValueError(
|
||||
f"{spec.filename} is a Ref2VA Turbo artifact; start the server with --task-type ref2va "
|
||||
f"(task_type={partition!r} serves ref2va from a DiT the adapter cannot bind to)"
|
||||
)
|
||||
if unsupported_offload_mode is not None:
|
||||
raise ValueError(f"MiniMax-H3 Turbo dynamic LoRA does not support {unsupported_offload_mode}")
|
||||
|
||||
tensors = _validate_and_convert_tensors(checkpoint)
|
||||
|
||||
peft_helper = PEFTHelper.from_dict(
|
||||
{
|
||||
"r": _TURBO_RANK,
|
||||
"lora_alpha": _TURBO_ALPHA,
|
||||
"r": spec.rank,
|
||||
"lora_alpha": spec.alpha,
|
||||
"target_modules": _TURBO_TARGET_PATTERN,
|
||||
}
|
||||
)
|
||||
@@ -209,4 +312,16 @@ def load_minimax_h3_turbo_lora(
|
||||
weights_mapper=_TURBO_WEIGHTS_MAPPER,
|
||||
)
|
||||
_pack_h3_turbo_fc1(lora_model)
|
||||
return lora_model, peft_helper
|
||||
logger.info(
|
||||
"MiniMax-H3 Turbo adapter %s: task=%s, %d denoiser forwards (%d sigma points), "
|
||||
"rank=%d alpha=%g, flow_shift=%g/%g",
|
||||
spec.filename,
|
||||
spec.task_family,
|
||||
spec.denoise_steps,
|
||||
spec.sigma_points,
|
||||
spec.rank,
|
||||
spec.alpha,
|
||||
spec.video_shift,
|
||||
spec.audio_shift,
|
||||
)
|
||||
return lora_model, peft_helper, spec
|
||||
@@ -125,7 +125,7 @@ from .denoise_loop import (
|
||||
)
|
||||
from .encoder import MiniMaxH3Qwen3VLEncoder
|
||||
from .fasth3 import FastH3WeightFusion, resolve_fasth3_fusion
|
||||
from .lora import load_minimax_h3_turbo_lora
|
||||
from .lora import TurboSpec, load_minimax_h3_turbo_lora
|
||||
from .minimax_h3_transformer import (
|
||||
MiniMaxH3Attention,
|
||||
MiniMaxH3DiTModel,
|
||||
@@ -188,9 +188,6 @@ MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES = 30 * 1024 * 1024
|
||||
MINIMAX_H3_REFERENCE_IMAGE_FORMATS = frozenset({"jpeg", "png", "webp", "heic", "heif"})
|
||||
MINIMAX_H3_MIN_OUTPUT_SECONDS = 4.0
|
||||
MINIMAX_H3_MAX_OUTPUT_SECONDS = 15.0
|
||||
MINIMAX_H3_TURBO_SIGMA_POINTS = 5
|
||||
MINIMAX_H3_TURBO_VIDEO_SHIFT = 6.0
|
||||
MINIMAX_H3_TURBO_AUDIO_SHIFT = 3.0
|
||||
MINIMAX_H3_DOWNLOAD_PATTERNS = [
|
||||
"FL2VA/**",
|
||||
"Ref2VA/model_index.json",
|
||||
@@ -706,7 +703,7 @@ class MiniMaxH3Pipeline(
|
||||
) -> tuple[LoRAModel, PEFTHelper] | None:
|
||||
# A cache eviction may be followed by a different adapter reusing the
|
||||
# same client-supplied ID. Every real load replaces the classification.
|
||||
self._turbo_lora_adapter_ids.discard(lora_request.lora_int_id)
|
||||
self._turbo_lora_specs.pop(lora_request.lora_int_id, None)
|
||||
self._native_lora_adapter_ids.discard(lora_request.lora_int_id)
|
||||
self._lora_sigma_schedules.pop(lora_request.lora_int_id, None)
|
||||
od_config = getattr(self, "od_config", None)
|
||||
@@ -726,8 +723,9 @@ class MiniMaxH3Pipeline(
|
||||
unsupported_offload_mode=" or ".join(offload_modes) or None,
|
||||
)
|
||||
if loaded is not None:
|
||||
self._turbo_lora_adapter_ids.add(lora_request.lora_int_id)
|
||||
return loaded
|
||||
lora_model, peft_helper, turbo_spec = loaded
|
||||
self._turbo_lora_specs[lora_request.lora_int_id] = turbo_spec
|
||||
return lora_model, peft_helper
|
||||
|
||||
# Selection is by the artifact's safetensors ``key_format``, not by the
|
||||
# running platform: the native loader is checkpoint-format parsing with
|
||||
@@ -753,7 +751,7 @@ class MiniMaxH3Pipeline(
|
||||
lora_model: LoRAModel,
|
||||
bound_lora_names: frozenset[str],
|
||||
) -> None:
|
||||
if lora_model.id in self._turbo_lora_adapter_ids:
|
||||
if lora_model.id in self._turbo_lora_specs:
|
||||
missing = sorted(set(lora_model.loras) - bound_lora_names)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
@@ -770,14 +768,6 @@ class MiniMaxH3Pipeline(
|
||||
f"bound={len(bound_lora_names)}/{len(lora_model.loras)}, missing={missing[:5]}"
|
||||
)
|
||||
|
||||
def _has_active_turbo_lora(self, sampling: Any) -> bool:
|
||||
lora_request = sampling.lora_request
|
||||
return (
|
||||
lora_request is not None
|
||||
and not math.isclose(0.0, float(sampling.lora_scale))
|
||||
and lora_request.lora_int_id in self._turbo_lora_adapter_ids
|
||||
)
|
||||
|
||||
def _has_active_native_lora(self, sampling: Any) -> bool:
|
||||
lora_request = sampling.lora_request
|
||||
return (
|
||||
@@ -837,27 +827,46 @@ class MiniMaxH3Pipeline(
|
||||
return adapter_schedule
|
||||
return self._base_schedule_for_task(task)
|
||||
|
||||
def _validate_turbo_sampling(self, sampling: Any) -> None:
|
||||
def _active_turbo_spec(self, sampling: Any) -> TurboSpec | None:
|
||||
"""Return the spec of the Turbo adapter this request actually applies.
|
||||
|
||||
A recognized adapter at scale 0 contributes nothing, so it neither
|
||||
constrains the task nor imposes its sampler contract.
|
||||
"""
|
||||
|
||||
lora_request = sampling.lora_request
|
||||
if lora_request is None or math.isclose(0.0, float(sampling.lora_scale)):
|
||||
return None
|
||||
return self._turbo_lora_specs.get(lora_request.lora_int_id)
|
||||
|
||||
def _validate_turbo_sampling(self, sampling: Any, spec: TurboSpec) -> None:
|
||||
"""Hold a request to the contract of the artifact that is loaded.
|
||||
|
||||
Sigma-point count and both flow shifts vary across the Turbo family, so
|
||||
each is checked against the adapter's own spec rather than a single
|
||||
published configuration.
|
||||
"""
|
||||
|
||||
extra = sampling.extra_args or {}
|
||||
sigma_points = sampling.num_inference_steps
|
||||
if sigma_points != MINIMAX_H3_TURBO_SIGMA_POINTS:
|
||||
if sigma_points != spec.sigma_points:
|
||||
raise OmniClientError(
|
||||
"MiniMax-H3 Turbo requires num_inference_steps=5 (five sigma points produce four denoiser evaluations)"
|
||||
f"{spec.filename} is a {spec.denoise_steps}-step artifact and requires "
|
||||
f"num_inference_steps={spec.sigma_points} "
|
||||
f"({spec.sigma_points} sigma points produce {spec.denoise_steps} denoiser evaluations)"
|
||||
)
|
||||
try:
|
||||
video_shift = float(extra.get("flow_shift", self.default_video_shift))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}") from exc
|
||||
if not math.isclose(video_shift, MINIMAX_H3_TURBO_VIDEO_SHIFT):
|
||||
raise OmniClientError(f"MiniMax-H3 Turbo requires flow_shift={MINIMAX_H3_TURBO_VIDEO_SHIFT:g}")
|
||||
raise OmniClientError(f"{spec.filename} requires flow_shift={spec.video_shift:g}") from exc
|
||||
if not math.isclose(video_shift, spec.video_shift):
|
||||
raise OmniClientError(f"{spec.filename} requires flow_shift={spec.video_shift:g}")
|
||||
try:
|
||||
audio_shift = float(extra.get("audio_flow_shift", self.default_audio_shift))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise OmniClientError(
|
||||
f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}"
|
||||
) from exc
|
||||
if not math.isclose(audio_shift, MINIMAX_H3_TURBO_AUDIO_SHIFT):
|
||||
raise OmniClientError(f"MiniMax-H3 Turbo requires audio_flow_shift={MINIMAX_H3_TURBO_AUDIO_SHIFT:g}")
|
||||
raise OmniClientError(f"{spec.filename} requires audio_flow_shift={spec.audio_shift:g}") from exc
|
||||
if not math.isclose(audio_shift, spec.audio_shift):
|
||||
raise OmniClientError(f"{spec.filename} requires audio_flow_shift={spec.audio_shift:g}")
|
||||
|
||||
def adopt_cache_dit_backend(self, backend: CacheDiTBackend) -> None:
|
||||
"""Adopt runner-installed generic Cache-DiT for request transitions."""
|
||||
@@ -890,7 +899,7 @@ class MiniMaxH3Pipeline(
|
||||
getattr(od_config, "task_type", None),
|
||||
str(od_config.model),
|
||||
)
|
||||
self._turbo_lora_adapter_ids: set[int] = set()
|
||||
self._turbo_lora_specs: dict[int, TurboSpec] = {}
|
||||
self._native_lora_adapter_ids: set[int] = set()
|
||||
self._lora_sigma_schedules: dict[int, DMD2SigmaSchedule] = {}
|
||||
model_root = _resolve_minimax_h3_model_root(
|
||||
@@ -1177,7 +1186,7 @@ class MiniMaxH3Pipeline(
|
||||
requested: str | None,
|
||||
multi_modal_data: dict[str, Any],
|
||||
*,
|
||||
has_turbo_lora: bool = False,
|
||||
turbo_spec: TurboSpec | None = None,
|
||||
has_native_lora: bool = False,
|
||||
) -> str:
|
||||
if requested is None:
|
||||
@@ -1196,8 +1205,11 @@ class MiniMaxH3Pipeline(
|
||||
raise OmniClientError(
|
||||
f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}"
|
||||
)
|
||||
if task == "ref2va" and has_turbo_lora:
|
||||
raise OmniClientError("MiniMax-H3 Turbo LoRA supports T2VA/FL2VA requests only")
|
||||
if turbo_spec is not None and task not in turbo_spec.supported_tasks:
|
||||
raise OmniClientError(
|
||||
f"{turbo_spec.filename} is a {turbo_spec.task_family} Turbo artifact and serves "
|
||||
f"{sorted(turbo_spec.supported_tasks)}, got task={task!r}"
|
||||
)
|
||||
if has_native_lora and task != "t2va":
|
||||
raise OmniClientError("MiniMax-H3 native LoRA supports T2VA requests only")
|
||||
if self._fasth3 is not None:
|
||||
@@ -2216,16 +2228,16 @@ class MiniMaxH3Pipeline(
|
||||
quality = sampling.quality
|
||||
logger.debug("MiniMax H3 request quality=%s", quality)
|
||||
extra = sampling.extra_args or {}
|
||||
has_turbo_lora = self._has_active_turbo_lora(sampling)
|
||||
turbo_spec = self._active_turbo_spec(sampling)
|
||||
has_native_lora = self._has_active_native_lora(sampling)
|
||||
task = self._resolve_task(
|
||||
extra.get("task"),
|
||||
multi_modal_data,
|
||||
has_turbo_lora=has_turbo_lora,
|
||||
turbo_spec=turbo_spec,
|
||||
has_native_lora=has_native_lora,
|
||||
)
|
||||
if has_turbo_lora:
|
||||
self._validate_turbo_sampling(sampling)
|
||||
if turbo_spec is not None:
|
||||
self._validate_turbo_sampling(sampling, turbo_spec)
|
||||
if has_native_lora:
|
||||
self._validate_native_sampling(sampling, task=task)
|
||||
if self._fasth3 is not None:
|
||||
|
||||
Reference in new issue
Block a user