[Model] Optimize MiniMax-H3 DLO component lifecycle
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
29e3165bb9
commit
1843f05ced
9 files changed
+701
-75
No files matched your search
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MiniMax H3 Qwen3-VL layer-50 text/vision encoder.
|
||||
|
||||
The encoder is reimplemented on top of vLLM-style tensor-parallel building
|
||||
@@ -42,6 +43,10 @@ from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMetho
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm as DiffusionRMSNorm
|
||||
from vllm_omni.diffusion.offloader.module_residency import (
|
||||
BoundedAllocatorCache,
|
||||
PinnedModuleStager,
|
||||
)
|
||||
|
||||
MINIMAX_H3_QWEN3VL_SELECTED_LM_LAYER = 50
|
||||
MINIMAX_H3_QWEN3VL_HIDDEN_DIM = 5120
|
||||
@@ -1211,12 +1216,18 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module):
|
||||
if not self.is_loaded:
|
||||
return
|
||||
if getattr(self, "_omni_layerwise_enabled", False):
|
||||
for name, child in self.vision.named_children():
|
||||
if name != "blocks":
|
||||
child.to(self.device_target)
|
||||
for name, child in self.text_model.named_children():
|
||||
if name != "layers":
|
||||
child.to(self.device_target)
|
||||
stager = getattr(self, "_omni_non_block_stager", None)
|
||||
if stager is None:
|
||||
hooks = getattr(self, "_omni_layerwise_hooks", ())
|
||||
pin_memory = bool(getattr(hooks[0], "pin_memory", True)) if hooks else True
|
||||
stager = PinnedModuleStager(
|
||||
self._omni_non_block_modules(),
|
||||
self.device_target,
|
||||
pin_memory=pin_memory,
|
||||
cache_retention=getattr(self, "_omni_component_cache", None),
|
||||
)
|
||||
self._omni_non_block_stager = stager
|
||||
stager.load()
|
||||
return
|
||||
self.vision.to(self.device_target)
|
||||
self.text_model.to(self.device_target)
|
||||
@@ -1227,18 +1238,39 @@ class MiniMaxH3Qwen3VLEncoder(nn.Module):
|
||||
if getattr(self, "_omni_layerwise_enabled", False):
|
||||
for hook in self._omni_layerwise_hooks:
|
||||
hook.offload_layer()
|
||||
for name, child in self.vision.named_children():
|
||||
if name != "blocks":
|
||||
stager = getattr(self, "_omni_non_block_stager", None)
|
||||
if stager is not None:
|
||||
stager.offload()
|
||||
else:
|
||||
# The initial DLO setup reaches this branch before the first
|
||||
# load. These modules were constructed on CPU, so this is not
|
||||
# a device-to-host rematerialization.
|
||||
for child in self._omni_non_block_modules():
|
||||
child.to("cpu")
|
||||
for name, child in self.text_model.named_children():
|
||||
if name != "layers":
|
||||
child.to("cpu")
|
||||
torch.accelerator.empty_cache()
|
||||
self._release_omni_component_cache()
|
||||
return
|
||||
self.vision.to("cpu")
|
||||
self.text_model.to("cpu")
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def _omni_non_block_modules(self) -> list[nn.Module]:
|
||||
modules = [child for name, child in self.vision.named_children() if name != "blocks"]
|
||||
modules.extend(child for name, child in self.text_model.named_children() if name != "layers")
|
||||
return modules
|
||||
|
||||
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
|
||||
self._omni_component_cache = cache
|
||||
stager = getattr(self, "_omni_non_block_stager", None)
|
||||
if stager is not None:
|
||||
stager.set_cache_retention(cache)
|
||||
|
||||
def _release_omni_component_cache(self) -> None:
|
||||
cache = getattr(self, "_omni_component_cache", None)
|
||||
if cache is None:
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
cache.release_if_needed()
|
||||
|
||||
def enable_omni_layerwise_offload(self, *, pin_memory: bool = True) -> None:
|
||||
"""Stream the TP-local Qwen vision/text blocks for low-HBM serving.
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from vllm_omni.diffusion.models.interface import (
|
||||
)
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.offloader import (
|
||||
BoundedAllocatorCache,
|
||||
OffloadPlan,
|
||||
apply_sequential_offload,
|
||||
remove_sequential_offload,
|
||||
@@ -952,6 +953,12 @@ class MiniMaxH3Pipeline(
|
||||
# Registry-side VAE patch-parallel discovery uses ``pipeline.vae``.
|
||||
self.vae = self.video_vae
|
||||
|
||||
self._dlo_component_cache = None
|
||||
if getattr(od_config, "enable_distributed_layerwise_offload", False):
|
||||
self._dlo_component_cache = BoundedAllocatorCache(self.device)
|
||||
for component in (self.text_encoder, self.video_vae, self.audio_vae):
|
||||
component.set_omni_component_cache(self._dlo_component_cache)
|
||||
|
||||
self._quality_policy = MiniMaxH3QualityPolicy(od_config)
|
||||
self._cache_dit_runtime = RequestScopedCacheDiTRuntime(self)
|
||||
|
||||
@@ -1371,11 +1378,8 @@ class MiniMaxH3Pipeline(
|
||||
):
|
||||
# Layerwise DiT offload already provides the low-residency encoder
|
||||
# phase used by the checkpoint reference.
|
||||
self.text_encoder.load_to_device()
|
||||
try:
|
||||
with self._component_on_device(self.text_encoder):
|
||||
return self.text_encoder.encode_ids(input_ids, **vision_kwargs)
|
||||
finally:
|
||||
self.text_encoder.offload_to_cpu()
|
||||
|
||||
# Keep both Qwen and DiT resident across requests. Moving either model
|
||||
# here makes encoder latency include a tens-of-gigabytes PCIe transfer,
|
||||
@@ -1433,13 +1437,35 @@ class MiniMaxH3Pipeline(
|
||||
yield
|
||||
return
|
||||
staged = self._uses_manual_component_offload()
|
||||
if staged:
|
||||
component.load_to_device()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if staged:
|
||||
component.offload_to_cpu()
|
||||
component.load_to_device()
|
||||
yield
|
||||
except BaseException:
|
||||
if staged:
|
||||
try:
|
||||
component.offload_to_cpu()
|
||||
except BaseException:
|
||||
logger.exception("Failed to release %s after component failure", component.__class__.__name__)
|
||||
cache = getattr(self, "_dlo_component_cache", None)
|
||||
if cache is not None:
|
||||
try:
|
||||
cache.release_if_needed(force=True)
|
||||
except BaseException:
|
||||
logger.exception("Failed to release retained allocator cache after component failure")
|
||||
raise
|
||||
else:
|
||||
if staged:
|
||||
try:
|
||||
component.offload_to_cpu()
|
||||
except BaseException:
|
||||
cache = getattr(self, "_dlo_component_cache", None)
|
||||
if cache is not None:
|
||||
try:
|
||||
cache.release_if_needed(force=True)
|
||||
except BaseException:
|
||||
logger.exception("Failed to release retained allocator cache after offload failure")
|
||||
raise
|
||||
|
||||
def _encode_visual_conditions(
|
||||
self,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MiniMax H3 remote-code VAE adapters and exact latent contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -21,7 +22,10 @@ from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor impor
|
||||
DistributedVaeMixin,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import get_world_group
|
||||
from vllm_omni.diffusion.offloader.module_residency import PinnedModuleStager
|
||||
from vllm_omni.diffusion.offloader.module_residency import (
|
||||
BoundedAllocatorCache,
|
||||
PinnedModuleStager,
|
||||
)
|
||||
|
||||
from .packed_tokens import minimax_h3_patchify_video_latent
|
||||
|
||||
@@ -148,12 +152,21 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
else:
|
||||
self.remote.to(self._device_target)
|
||||
|
||||
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
|
||||
self._omni_component_cache = cache
|
||||
if self._stager is not None:
|
||||
self._stager.set_cache_retention(cache)
|
||||
|
||||
def offload_to_cpu(self) -> None:
|
||||
if self._stager is not None:
|
||||
self._stager.offload()
|
||||
else:
|
||||
self.remote.to("cpu")
|
||||
torch.accelerator.empty_cache()
|
||||
cache = getattr(self, "_omni_component_cache", None)
|
||||
if cache is None:
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
cache.release_if_needed()
|
||||
|
||||
def set_parallel_size(
|
||||
self,
|
||||
@@ -411,12 +424,21 @@ class MiniMaxH3AudioVAE(nn.Module):
|
||||
else:
|
||||
self.remote.to(self._device_target)
|
||||
|
||||
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
|
||||
self._omni_component_cache = cache
|
||||
if self._stager is not None:
|
||||
self._stager.set_cache_retention(cache)
|
||||
|
||||
def offload_to_cpu(self) -> None:
|
||||
if self._stager is not None:
|
||||
self._stager.offload()
|
||||
else:
|
||||
self.remote.to("cpu")
|
||||
torch.accelerator.empty_cache()
|
||||
cache = getattr(self, "_omni_component_cache", None)
|
||||
if cache is None:
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
cache.release_if_needed()
|
||||
|
||||
@torch.inference_mode()
|
||||
def encode_waveform(
|
||||
|
||||
Reference in new issue
Block a user