[Model] Optimize MiniMax-H3 DLO component lifecycle

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
hsliuustc0106 committed 2026-08-25 17:58:08 +08:00
1 parent 29e3165bb9
commit 1843f05ced
9 files changed
+701 -75

No files matched your search

@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""MiniMax H3 Qwen3-VL layer-50 text/vision encoder.
The encoder is reimplemented on top of vLLM-style tensor-parallel building
@@ -42,6 +43,10 @@ from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMetho
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm_omni.diffusion.layers.norm import RMSNorm as DiffusionRMSNorm
from vllm_omni.diffusion.offloader.module_residency import (
BoundedAllocatorCache,
PinnedModuleStager,
)
MINIMAX_H3_QWEN3VL_SELECTED_LM_LAYER = 50
MINIMAX_H3_QWEN3VL_HIDDEN_DIM = 5120
@@ -1211,12 +1216,18 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module):
if not self.is_loaded:
return
if getattr(self, "_omni_layerwise_enabled", False):
for name, child in self.vision.named_children():
if name != "blocks":
child.to(self.device_target)
for name, child in self.text_model.named_children():
if name != "layers":
child.to(self.device_target)
stager = getattr(self, "_omni_non_block_stager", None)
if stager is None:
hooks = getattr(self, "_omni_layerwise_hooks", ())
pin_memory = bool(getattr(hooks[0], "pin_memory", True)) if hooks else True
stager = PinnedModuleStager(
self._omni_non_block_modules(),
self.device_target,
pin_memory=pin_memory,
cache_retention=getattr(self, "_omni_component_cache", None),
)
self._omni_non_block_stager = stager
stager.load()
return
self.vision.to(self.device_target)
self.text_model.to(self.device_target)
@@ -1227,18 +1238,39 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module):
if getattr(self, "_omni_layerwise_enabled", False):
for hook in self._omni_layerwise_hooks:
hook.offload_layer()
for name, child in self.vision.named_children():
if name != "blocks":
stager = getattr(self, "_omni_non_block_stager", None)
if stager is not None:
stager.offload()
else:
# The initial DLO setup reaches this branch before the first
# load. These modules were constructed on CPU, so this is not
# a device-to-host rematerialization.
for child in self._omni_non_block_modules():
child.to("cpu")
for name, child in self.text_model.named_children():
if name != "layers":
child.to("cpu")
torch.accelerator.empty_cache()
self._release_omni_component_cache()
return
self.vision.to("cpu")
self.text_model.to("cpu")
torch.accelerator.empty_cache()
def _omni_non_block_modules(self) -> list[nn.Module]:
modules = [child for name, child in self.vision.named_children() if name != "blocks"]
modules.extend(child for name, child in self.text_model.named_children() if name != "layers")
return modules
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
self._omni_component_cache = cache
stager = getattr(self, "_omni_non_block_stager", None)
if stager is not None:
stager.set_cache_retention(cache)
def _release_omni_component_cache(self) -> None:
cache = getattr(self, "_omni_component_cache", None)
if cache is None:
torch.accelerator.empty_cache()
else:
cache.release_if_needed()
def enable_omni_layerwise_offload(self, *, pin_memory: bool = True) -> None:
"""Stream the TP-local Qwen vision/text blocks for low-HBM serving.
@@ -46,6 +46,7 @@ from vllm_omni.diffusion.models.interface import (
)
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.offloader import (
BoundedAllocatorCache,
OffloadPlan,
apply_sequential_offload,
remove_sequential_offload,
@@ -952,6 +953,12 @@ class MiniMaxH3Pipeline(
# Registry-side VAE patch-parallel discovery uses ``pipeline.vae``.
self.vae = self.video_vae
self._dlo_component_cache = None
if getattr(od_config, "enable_distributed_layerwise_offload", False):
self._dlo_component_cache = BoundedAllocatorCache(self.device)
for component in (self.text_encoder, self.video_vae, self.audio_vae):
component.set_omni_component_cache(self._dlo_component_cache)
self._quality_policy = MiniMaxH3QualityPolicy(od_config)
self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
@@ -1371,11 +1378,8 @@ class MiniMaxH3Pipeline(
):
# Layerwise DiT offload already provides the low-residency encoder
# phase used by the checkpoint reference.
self.text_encoder.load_to_device()
try:
with self._component_on_device(self.text_encoder):
return self.text_encoder.encode_ids(input_ids, **vision_kwargs)
finally:
self.text_encoder.offload_to_cpu()
# Keep both Qwen and DiT resident across requests. Moving either model
# here makes encoder latency include a tens-of-gigabytes PCIe transfer,
@@ -1433,13 +1437,35 @@ class MiniMaxH3Pipeline(
yield
return
staged = self._uses_manual_component_offload()
if staged:
component.load_to_device()
try:
yield
finally:
if staged:
component.offload_to_cpu()
component.load_to_device()
yield
except BaseException:
if staged:
try:
component.offload_to_cpu()
except BaseException:
logger.exception("Failed to release %s after component failure", component.__class__.__name__)
cache = getattr(self, "_dlo_component_cache", None)
if cache is not None:
try:
cache.release_if_needed(force=True)
except BaseException:
logger.exception("Failed to release retained allocator cache after component failure")
raise
else:
if staged:
try:
component.offload_to_cpu()
except BaseException:
cache = getattr(self, "_dlo_component_cache", None)
if cache is not None:
try:
cache.release_if_needed(force=True)
except BaseException:
logger.exception("Failed to release retained allocator cache after offload failure")
raise
def _encode_visual_conditions(
self,
+25 -3
View File
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""MiniMax H3 remote-code VAE adapters and exact latent contracts."""
from __future__ import annotations
@@ -21,7 +22,10 @@ from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor impor
DistributedVaeMixin,
)
from vllm_omni.diffusion.distributed.parallel_state import get_world_group
from vllm_omni.diffusion.offloader.module_residency import PinnedModuleStager
from vllm_omni.diffusion.offloader.module_residency import (
BoundedAllocatorCache,
PinnedModuleStager,
)
from .packed_tokens import minimax_h3_patchify_video_latent
@@ -148,12 +152,21 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
else:
self.remote.to(self._device_target)
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
self._omni_component_cache = cache
if self._stager is not None:
self._stager.set_cache_retention(cache)
def offload_to_cpu(self) -> None:
if self._stager is not None:
self._stager.offload()
else:
self.remote.to("cpu")
torch.accelerator.empty_cache()
cache = getattr(self, "_omni_component_cache", None)
if cache is None:
torch.accelerator.empty_cache()
else:
cache.release_if_needed()
def set_parallel_size(
self,
@@ -411,12 +424,21 @@ class MiniMaxH3AudioVAE(nn.Module):
else:
self.remote.to(self._device_target)
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
self._omni_component_cache = cache
if self._stager is not None:
self._stager.set_cache_retention(cache)
def offload_to_cpu(self) -> None:
if self._stager is not None:
self._stager.offload()
else:
self.remote.to("cpu")
torch.accelerator.empty_cache()
cache = getattr(self, "_omni_component_cache", None)
if cache is None:
torch.accelerator.empty_cache()
else:
cache.release_if_needed()
@torch.inference_mode()
def encode_waveform(