[Model][MiniMax-H3] Multi-GPU serving fixes and layer-wise offload component selection (#7047)
Signed-off-by: heyuanliu-intel <heyuan.liu@intel.com>
This commit is contained in:
1 parent
2f1a18e69f
commit
40f8e94400
21 files changed
+1857
-60
No files matched your search
@@ -44,6 +44,10 @@ from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelInput,
|
||||
SequenceParallelOutput,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import (
|
||||
get_forward_context,
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm_omni.diffusion.layers.activation import SiluAndMul
|
||||
from vllm_omni.diffusion.layers.fused_qk_norm_rope import fused_qk_norm_rope
|
||||
from vllm_omni.diffusion.layers.norm import RMSNorm
|
||||
@@ -67,6 +71,15 @@ logger = init_logger(__name__)
|
||||
# packing, and ``_run_packed_attention`` re-checks it per forward; a name-only
|
||||
# gate would let FLASH_ATTN's NPU/XPU code paths through even though those
|
||||
# variants would silently attend across request boundaries.
|
||||
def _ring_sequence_parallel_is_active(attention_layer: Attention) -> bool:
|
||||
"""Match :meth:`Attention._get_active_parallel_strategy` for Ring."""
|
||||
if not getattr(attention_layer, "use_ring", False) or getattr(attention_layer, "skip_sequence_parallel", False):
|
||||
return False
|
||||
if is_forward_context_available() and not get_forward_context().sp_active:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _attention_isolates_packed_requests(attention_layer: Any) -> bool:
|
||||
"""True if this attention layer keeps N-document packed boundaries.
|
||||
|
||||
@@ -78,7 +91,7 @@ def _attention_isolates_packed_requests(attention_layer: Any) -> bool:
|
||||
backend = getattr(attention_layer, "attn_backend", None)
|
||||
if backend is None or not backend.supports_multi_doc_packed_varlen():
|
||||
return False
|
||||
return not getattr(attention_layer, "use_ring", False)
|
||||
return not _ring_sequence_parallel_is_active(attention_layer)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -508,6 +521,7 @@ class MiniMaxH3Attention(nn.Module):
|
||||
)
|
||||
attn_mask = None
|
||||
mask_free_packed_padding = False
|
||||
use_ring = _ring_sequence_parallel_is_active(self.attention)
|
||||
if num_requests > 1:
|
||||
# A step-mode batch packs one document per request, so its valid
|
||||
# rows are block-diagonal rather than a prefix: neither a KV prefix
|
||||
@@ -534,12 +548,15 @@ class MiniMaxH3Attention(nn.Module):
|
||||
# supports_packed_mask_free: backend consumes the packed metadata
|
||||
# without ever reading attn_mask (CUDA packed varlen, NPU
|
||||
# npu_attn_varlen opt-in with its own fallback rebuild).
|
||||
use_ring = getattr(self.attention, "use_ring", False)
|
||||
mask_free_packed_padding = not use_ring and self.attention.attn_backend.supports_packed_mask_free()
|
||||
no_mask = not use_ring and (
|
||||
self.attention.attn_backend.supports_prefix_kv_slicing or mask_free_packed_padding
|
||||
)
|
||||
if used < packed_total and not no_mask:
|
||||
# Hybrid Ulysses reshards Q to one ring partition before the ring
|
||||
# kernel runs, so a global [packed_total] mask cannot pass its
|
||||
# query-length check. Ring consumes valid_kv_length directly and
|
||||
# trims the circulated K/V blocks instead.
|
||||
if used < packed_total and not no_mask and not use_ring:
|
||||
attn_mask = torch.arange(packed_total, device=q.device)[None] < used
|
||||
metadata = AttentionMetadata(
|
||||
attn_mask=attn_mask,
|
||||
@@ -563,7 +580,7 @@ class MiniMaxH3Attention(nn.Module):
|
||||
# quadratic full_qk mask is never materialized. Ring attention
|
||||
# is excluded: it keeps the aligned padding rows for its
|
||||
# fixed-size P2P buffers and still needs the mask.
|
||||
"npu_attn_varlen": not getattr(self.attention, "use_ring", False),
|
||||
"npu_attn_varlen": not use_ring,
|
||||
# fp16-range protection for the ascend_laser_attention kernel
|
||||
# (see MINIMAX_H3_LASER_INPUT_SCALE). Ignored by every other
|
||||
# backend/path.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""MiniMax H3 packed-sequence materialization for FL2VA and Ref2VA tasks.
|
||||
|
||||
Layout: [text L | imgvid_cond C | audio A(=t*2ch) | video_target V | pad P].
|
||||
@@ -18,6 +19,8 @@ from collections.abc import Mapping, Sequence
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.errors import OmniClientError
|
||||
|
||||
MINIMAX_H3_FL2VA_KEYFRAME_SIGNATURES: tuple[tuple[int, ...], ...] = (
|
||||
(0,),
|
||||
(-1,),
|
||||
@@ -38,13 +41,42 @@ _INTERP = 32
|
||||
_T_GROUP = 5
|
||||
_FRAME_PER_TOKEN = (1, 4, 4, 4, 4)
|
||||
_FRAME_RESCALE = 5.0 / 3.0
|
||||
_SEQ_ALIGN = 64
|
||||
MINIMAX_H3_SEQ_ALIGN = 64
|
||||
"""Row alignment of the packed sequence.
|
||||
|
||||
An explicit ``seq_len`` must be a multiple of this, so that pinning a length
|
||||
lands on a bucket the default rounding could also produce.
|
||||
"""
|
||||
|
||||
MINIMAX_H3_MAX_PAD_SEQ_LEN = 1 << 20
|
||||
"""Ceiling for a request-pinned packed length.
|
||||
|
||||
Every packed row costs a fixed number of bytes across the structural tensors
|
||||
built below, so an unbounded request value would size those allocations. The
|
||||
largest shape this pipeline is documented against -- 672x384 at 209 frames --
|
||||
packs about 16k rows, so this ceiling leaves roughly two orders of magnitude of
|
||||
headroom while keeping the structural tensors in the tens of megabytes.
|
||||
"""
|
||||
# MiniMax H3 packs video latents with a fixed [1, 2, 2] (t, h, w) patch.
|
||||
_PATCH_T = 1
|
||||
_PATCH_H = 2
|
||||
_PATCH_W = 2
|
||||
|
||||
|
||||
def _resolve_padded_seq_len(used: int, seq_len: int | None) -> int:
|
||||
"""Resolve the padded row count for a packed sequence.
|
||||
|
||||
Without an explicit ``seq_len`` the used rows are rounded up to the next
|
||||
:data:`MINIMAX_H3_SEQ_ALIGN` boundary. An explicit value is honored as-is
|
||||
and must cover the used rows.
|
||||
"""
|
||||
if seq_len is None:
|
||||
return ((used + MINIMAX_H3_SEQ_ALIGN - 1) // MINIMAX_H3_SEQ_ALIGN) * MINIMAX_H3_SEQ_ALIGN
|
||||
if seq_len < used:
|
||||
raise OmniClientError(f"seq_len {seq_len} < used rows {used}")
|
||||
return seq_len
|
||||
|
||||
|
||||
def _keyframe_cond_frame_indices(
|
||||
*,
|
||||
include_keyframe_cond: bool,
|
||||
@@ -124,10 +156,15 @@ def minimax_h3_packed_sequence(
|
||||
include_keyframe_cond: bool,
|
||||
keyframe_frame_indices: list[int] | tuple[int, ...] | None = None,
|
||||
frame_count: int | None = None,
|
||||
seq_len: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Build the packed-sequence structural fields for one CFG branch.
|
||||
|
||||
The used length is padded up to a multiple of 64.
|
||||
The used length is padded up to a multiple of
|
||||
:data:`MINIMAX_H3_SEQ_ALIGN`, or to an explicit ``seq_len`` that covers the
|
||||
used rows -- the same contract
|
||||
:func:`minimax_h3_packed_sequence_ref2va_blocks` already offers. Pinning
|
||||
the length keeps requests of different prompt lengths on one packed shape.
|
||||
"""
|
||||
ph, pw = latent_h // _PATCH_H, latent_w // _PATCH_W
|
||||
frame_rows = ph * pw
|
||||
@@ -143,7 +180,7 @@ def minimax_h3_packed_sequence(
|
||||
video_rows = latent_t * frame_rows
|
||||
audio_rows = audio_t * audio_channel
|
||||
used = text_len + cond_rows + audio_rows + video_rows
|
||||
seq_len = ((used + _SEQ_ALIGN - 1) // _SEQ_ALIGN) * _SEQ_ALIGN
|
||||
seq_len = _resolve_padded_seq_len(used, seq_len)
|
||||
|
||||
text_sl = slice(0, text_len)
|
||||
cond_sl = slice(text_len, text_len + cond_rows)
|
||||
@@ -376,10 +413,7 @@ def minimax_h3_packed_sequence_ref2va_blocks(
|
||||
audio_rows = audio_t * audio_channel
|
||||
ref_rows = ref_visual_rows + ref_audio_rows
|
||||
used = text_len + ref_rows + audio_rows + video_rows
|
||||
if seq_len is None:
|
||||
seq_len = ((used + _SEQ_ALIGN - 1) // _SEQ_ALIGN) * _SEQ_ALIGN
|
||||
if seq_len < used:
|
||||
raise ValueError(f"seq_len {seq_len} < used rows {used}")
|
||||
seq_len = _resolve_padded_seq_len(used, seq_len)
|
||||
|
||||
text_sl = slice(0, text_len)
|
||||
cursor = text_len
|
||||
|
||||
@@ -137,6 +137,8 @@ from .npu.lora import (
|
||||
load_minimax_h3_native_lora,
|
||||
)
|
||||
from .packed_sequence import (
|
||||
MINIMAX_H3_MAX_PAD_SEQ_LEN,
|
||||
MINIMAX_H3_SEQ_ALIGN,
|
||||
minimax_h3_packed_sequence,
|
||||
minimax_h3_packed_sequence_ref2va_blocks,
|
||||
)
|
||||
@@ -288,6 +290,7 @@ _MINIMAX_H3_DENOISE_INPUT_KEYS = (
|
||||
"visual_condition_shapes",
|
||||
"audio_condition_lengths",
|
||||
"keyframe_frame_indices",
|
||||
"pad_seq_len",
|
||||
)
|
||||
|
||||
# ``StepRequestState.extra`` keys owned by the step-execution path.
|
||||
@@ -543,6 +546,28 @@ def _resolve_minimax_h3_num_outputs(value: Any) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def _resolve_pad_seq_len(value: object) -> int | None:
|
||||
"""Validate the optional packed-length pin from ``extra_args``.
|
||||
|
||||
The packer itself only needs a value that covers the used rows. A request
|
||||
field needs two more guards: an unaligned value would silently open yet
|
||||
another compiled shape, and an unbounded one would size every structural
|
||||
tensor the packer allocates.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, np.integer)):
|
||||
raise OmniClientError("MiniMax H3 pad_seq_len must be an integer")
|
||||
pinned = int(value)
|
||||
if pinned <= 0:
|
||||
raise OmniClientError(f"MiniMax H3 pad_seq_len must be positive, got {pinned}")
|
||||
if pinned % MINIMAX_H3_SEQ_ALIGN:
|
||||
raise OmniClientError(f"MiniMax H3 pad_seq_len must be a multiple of {MINIMAX_H3_SEQ_ALIGN}, got {pinned}")
|
||||
if pinned > MINIMAX_H3_MAX_PAD_SEQ_LEN:
|
||||
raise OmniClientError(f"MiniMax H3 pad_seq_len must be at most {MINIMAX_H3_MAX_PAD_SEQ_LEN}, got {pinned}")
|
||||
return pinned
|
||||
|
||||
|
||||
def _minimax_h3_output_seeds(seed: int, num_outputs: int) -> list[int]:
|
||||
return [int(seed) + output_index for output_index in range(int(num_outputs))]
|
||||
|
||||
@@ -1077,11 +1102,13 @@ class MiniMaxH3Pipeline(
|
||||
os.path.join(model_path, "video_vae"),
|
||||
device=self.device,
|
||||
load_device=component_load_device,
|
||||
trust_remote_code=od_config.trust_remote_code,
|
||||
)
|
||||
self.audio_vae = MiniMaxH3AudioVAE(
|
||||
os.path.join(model_path, "audio_vae"),
|
||||
device=self.device,
|
||||
load_device=component_load_device,
|
||||
trust_remote_code=od_config.trust_remote_code,
|
||||
)
|
||||
# Registry-side VAE patch-parallel discovery uses ``pipeline.vae``.
|
||||
self.vae = self.video_vae
|
||||
@@ -1674,6 +1701,31 @@ class MiniMaxH3Pipeline(
|
||||
logger.exception("Failed to release retained allocator cache after offload failure")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _is_output_owner_rank() -> bool:
|
||||
"""Whether this rank's output is returned by the diffusion executor."""
|
||||
return not dist.is_available() or not dist.is_initialized() or dist.get_rank() == 0
|
||||
|
||||
def _offload_model_cpu_stage_output(self, tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Release a decoded output's storage before a later seed reloads the DiT.
|
||||
|
||||
Model-level CPU offload makes the VAE hook evict the DiT before decode.
|
||||
For a multi-output request, however, the next seed reloads the DiT while
|
||||
the preceding decoded output would otherwise still occupy the device.
|
||||
The reply rank performs the D2H copy early; ranks whose outputs are not
|
||||
consumed retain only a metadata placeholder for the final concatenation.
|
||||
|
||||
Call this on the tensor that is actually returned to the engine. Audio is
|
||||
released inside ``decode``; video is released by the callers of ``decode``
|
||||
only after ``_prepare_minimax_h3_video_output`` has quantized it, so the
|
||||
early D2H copy carries ``uint8`` frames rather than the decoded floats.
|
||||
"""
|
||||
if not getattr(self, "_model_cpu_offload_modules", None):
|
||||
return tensor
|
||||
if self._is_output_owner_rank():
|
||||
return tensor.cpu()
|
||||
return torch.empty(tensor.shape, dtype=tensor.dtype, device="meta")
|
||||
|
||||
def _encode_visual_conditions(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
@@ -1932,6 +1984,7 @@ class MiniMaxH3Pipeline(
|
||||
visual_condition_shapes: list[tuple[int, int, int]] | None = None,
|
||||
audio_condition_lengths: list[int] | None = None,
|
||||
keyframe_frame_indices: list[int] | None = None,
|
||||
pad_seq_len: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the packed layout, initial rows, anchors, and sigma schedules.
|
||||
|
||||
@@ -1961,6 +2014,7 @@ class MiniMaxH3Pipeline(
|
||||
latent_w=latent_w,
|
||||
audio_t=audio_t,
|
||||
ref_blocks=ref_blocks,
|
||||
seq_len=pad_seq_len,
|
||||
)
|
||||
else:
|
||||
packed = minimax_h3_packed_sequence(
|
||||
@@ -1972,7 +2026,19 @@ class MiniMaxH3Pipeline(
|
||||
include_keyframe_cond=task == "fl2va",
|
||||
keyframe_frame_indices=keyframe_frame_indices if task == "fl2va" else None,
|
||||
frame_count=num_frames if task == "fl2va" else None,
|
||||
seq_len=pad_seq_len,
|
||||
)
|
||||
# Report the effective shape at info level exactly when a request pins
|
||||
# it, so a deployment can confirm the pin landed without raising the
|
||||
# log level for every request.
|
||||
log = logger.info if pad_seq_len is not None else logger.debug
|
||||
log(
|
||||
"MiniMax H3 packed sequence: task=%s pad_seq_len=%s used=%d seq_len=%d",
|
||||
task,
|
||||
pad_seq_len,
|
||||
int(packed["cu_seqlens"][1]),
|
||||
int(packed["seq_len"]),
|
||||
)
|
||||
|
||||
tags = packed["token_tags"].clone()
|
||||
tags[packed["text_pos"]] = text_tags.cpu()
|
||||
@@ -2108,6 +2174,7 @@ class MiniMaxH3Pipeline(
|
||||
visual_condition_shapes: list[tuple[int, int, int]] | None = None,
|
||||
audio_condition_lengths: list[int] | None = None,
|
||||
keyframe_frame_indices: list[int] | None = None,
|
||||
pad_seq_len: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
inputs = self._build_denoise_inputs(
|
||||
task=task,
|
||||
@@ -2131,6 +2198,7 @@ class MiniMaxH3Pipeline(
|
||||
visual_condition_shapes=visual_condition_shapes,
|
||||
audio_condition_lengths=audio_condition_lengths,
|
||||
keyframe_frame_indices=keyframe_frame_indices,
|
||||
pad_seq_len=pad_seq_len,
|
||||
)
|
||||
branch = inputs["branch"]
|
||||
transformer = self._transformer_for_task(task)
|
||||
@@ -2253,6 +2321,7 @@ class MiniMaxH3Pipeline(
|
||||
video = video[..., :height, :width].contiguous()
|
||||
with self._component_on_device(self.audio_vae):
|
||||
audio = self.audio_vae.decode_latent(audio_latent)
|
||||
audio = self._offload_model_cpu_stage_output(audio)
|
||||
return video, audio
|
||||
|
||||
@staticmethod
|
||||
@@ -2561,6 +2630,7 @@ class MiniMaxH3Pipeline(
|
||||
base_schedule, num_steps = self._resolve_sigma_positions(task, sampling)
|
||||
video_shift = float(extra.get("flow_shift", self.default_video_shift))
|
||||
audio_shift = float(extra.get("audio_flow_shift", self.default_audio_shift))
|
||||
pad_seq_len = _resolve_pad_seq_len(extra.get("pad_seq_len"))
|
||||
quality_plan = self._quality_policy.resolve(
|
||||
quality=quality,
|
||||
num_inference_steps=num_steps,
|
||||
@@ -2587,6 +2657,7 @@ class MiniMaxH3Pipeline(
|
||||
"visual_condition_shapes": visual_shapes,
|
||||
"audio_condition_lengths": audio_lengths,
|
||||
"keyframe_frame_indices": keyframe_frame_indices,
|
||||
"pad_seq_len": pad_seq_len,
|
||||
"seed": seed,
|
||||
"num_steps": num_steps,
|
||||
"video_shift": video_shift,
|
||||
@@ -2643,7 +2714,13 @@ class MiniMaxH3Pipeline(
|
||||
height=context["height"],
|
||||
width=context["width"],
|
||||
)
|
||||
videos.append(_prepare_minimax_h3_video_output(video))
|
||||
# Rebind rather than append the expression: the local would
|
||||
# otherwise keep the decoded frames' device storage alive for the
|
||||
# rest of the iteration, and the next seed's diffuse() -- the DiT
|
||||
# reload this release exists to make room for -- runs before the
|
||||
# loop rebinds it. post_decode() rebinds for the same reason.
|
||||
video = self._offload_model_cpu_stage_output(_prepare_minimax_h3_video_output(video))
|
||||
videos.append(video)
|
||||
audios.append(audio)
|
||||
if videos and isinstance(videos[0], bytes):
|
||||
video = videos[0] if len(videos) == 1 else videos
|
||||
@@ -2966,7 +3043,7 @@ class MiniMaxH3Pipeline(
|
||||
height=shape["height"],
|
||||
width=shape["width"],
|
||||
)
|
||||
video = _prepare_minimax_h3_video_output(video)
|
||||
video = self._offload_model_cpu_stage_output(_prepare_minimax_h3_video_output(video))
|
||||
return DiffusionOutput(
|
||||
output=(video, audio),
|
||||
post_process_func=get_minimax_h3_post_process_func(self.od_config),
|
||||
|
||||
@@ -75,11 +75,23 @@ def _load_component_config(component_path: str) -> dict[str, Any]:
|
||||
def _load_remote_component(
|
||||
component_path: str,
|
||||
config: dict[str, Any],
|
||||
*,
|
||||
trust_remote_code: bool,
|
||||
) -> nn.Module:
|
||||
auto_map = config.get("auto_map") or {}
|
||||
class_reference = auto_map.get("AutoModel")
|
||||
if not isinstance(class_reference, str):
|
||||
raise ValueError(f"{component_path}/config.json must define auto_map.AutoModel")
|
||||
if not trust_remote_code:
|
||||
raise ValueError(
|
||||
f"Loading {component_path} executes the modeling code shipped with "
|
||||
f"the checkpoint (auto_map.AutoModel = {class_reference}). Pass "
|
||||
"--trust-remote-code (or trust_remote_code=True) to allow it."
|
||||
)
|
||||
# ``trust_remote_code`` is checked here rather than forwarded to
|
||||
# ``get_class_from_dynamic_module``: that helper takes no such argument and
|
||||
# would silently absorb it into ``**kwargs``, so forwarding it would read
|
||||
# like a gate while executing the remote code unconditionally.
|
||||
component_cls = get_class_from_dynamic_module(
|
||||
class_reference,
|
||||
component_path,
|
||||
@@ -144,6 +156,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
*,
|
||||
device: torch.device,
|
||||
load_device: torch.device | None = None,
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._device_target = device
|
||||
@@ -151,6 +164,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
self.remote = _load_remote_component(
|
||||
component_path,
|
||||
self.config_dict,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
# Match the reference loader contract before installing inference-only
|
||||
# decoder fast paths. Keyframe encoding remains FP32; decoder Linear
|
||||
@@ -176,6 +190,11 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
self.use_slicing = False
|
||||
self.parallel_size = 1
|
||||
self.device_module = torch.get_device_module()
|
||||
self._tile_gather_workspace: torch.Tensor | None = None
|
||||
self._tile_gather_stats = {"hits": 0, "allocs": 0, "workspace_bytes": 0}
|
||||
self._checkpoint_tile_gather = None
|
||||
if self._tile_gather_reuse_enabled():
|
||||
self._install_persistent_tile_gather()
|
||||
|
||||
def load_to_device(self) -> None:
|
||||
if self._stager is not None:
|
||||
@@ -241,6 +260,145 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
parallel_module = importlib.import_module(f"{package}.parallel")
|
||||
return parallel_module.get_parallel_state()
|
||||
|
||||
def _tile_gather_reuse_enabled(self) -> bool:
|
||||
"""Whether this device needs the tiled-VAE gather address kept stable.
|
||||
|
||||
XPU only. The accumulation this guards against is an XCCL-side
|
||||
registration that is kept for every distinct receive-buffer address and
|
||||
never reclaimed; no other backend in tree does that, so everywhere else
|
||||
the checkpoint's own method is left in place and behaviour is unchanged.
|
||||
The stable address comes from one grow-only byte workspace, so what the
|
||||
adapter holds resident is one buffer for the largest geometry it has
|
||||
seen rather than one per geometry.
|
||||
"""
|
||||
|
||||
return self._device_target.type == "xpu"
|
||||
|
||||
def _install_persistent_tile_gather(self) -> None:
|
||||
"""Route the checkpoint's tiled-VAE gather through a bounded workspace.
|
||||
|
||||
The checkpoint's ``_all_gather_tiled_results`` allocates its gather
|
||||
output afresh on every call. Model-level offload calls ``empty_cache()``
|
||||
once per request, so the next request's output lands on a new device
|
||||
address; XCCL registers a non-reclaimable resource per new receive
|
||||
address, and the registrations accumulate until the card is full. The
|
||||
replacement below keeps a single byte workspace alive on this adapter
|
||||
and carves every gather out of its front, so the address the collective
|
||||
writes to is the same one every request.
|
||||
|
||||
The workspace only ever grows: a geometry that fits in the current
|
||||
block reuses it as is, and a larger one replaces the block, which
|
||||
drops the previous allocation before allocating its own. Resident
|
||||
memory is therefore bounded by one buffer for the largest geometry the
|
||||
adapter has seen, not by one buffer per distinct geometry.
|
||||
|
||||
The override is bound on the checkpoint *instance*, not its class: the
|
||||
class is remote code shared by every component loaded from the same
|
||||
checkpoint, and only this adapter knows the device it runs on.
|
||||
"""
|
||||
|
||||
self._checkpoint_tile_gather = self.model._all_gather_tiled_results
|
||||
self.model._all_gather_tiled_results = self._persistent_tile_gather
|
||||
logger.info(
|
||||
"[H3_VAE_GATHER] persistent tile gather installed device=%s",
|
||||
self._device_target.type,
|
||||
)
|
||||
|
||||
def _persistent_tile_gather(
|
||||
self,
|
||||
tasks: list[torch.Tensor],
|
||||
num_tiles: int,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Equal-shape replacement for the checkpoint's tiled-result gather.
|
||||
|
||||
Contract kept identical to the checkpoint's method: return a list of
|
||||
``num_tiles`` tensors in global tile order, and raise on an empty local
|
||||
share so a rank that owns no tile cannot silently skip the collective.
|
||||
|
||||
Tile ownership is round-robin (``range(sp_rank, num_tiles, sp_size)``),
|
||||
so every rank can compute every other rank's task count from
|
||||
``num_tiles`` and ``sp_size`` alone. That makes the per-rank payloads
|
||||
equal once the leading task dimension is padded to ``max_tasks``, which
|
||||
is what lets a single ``all_gather_into_tensor`` into a stable buffer
|
||||
replace the variable-shape gather.
|
||||
|
||||
The landing buffer is a view onto the front of one byte workspace that
|
||||
this adapter keeps alive and only ever grows, so its address stays
|
||||
stable across requests while resident memory stays bounded by the
|
||||
largest geometry seen rather than growing per geometry. A byte
|
||||
workspace is dtype-agnostic, so mixed-precision requests share it too.
|
||||
|
||||
Returned tiles are cloned out of the buffer: the buffer is overwritten
|
||||
by the next call and callers hold the tiles past that point.
|
||||
"""
|
||||
|
||||
state = self._native_parallel_state()
|
||||
group = state["sp_process_group"]
|
||||
sp_size = int(state["sp_size"])
|
||||
sp_rank = int(state["sp_rank"])
|
||||
|
||||
if not tasks:
|
||||
raise ValueError(f"Found empty tasks on sp rank {sp_rank}")
|
||||
|
||||
max_tasks = -(-num_tiles // sp_size)
|
||||
if len(tasks) > max_tasks:
|
||||
raise ValueError(
|
||||
f"sp rank {sp_rank} holds {len(tasks)} tiles but round-robin "
|
||||
f"ownership of {num_tiles} tiles across {sp_size} ranks allows "
|
||||
f"at most {max_tasks}"
|
||||
)
|
||||
if len(tasks) == max_tasks:
|
||||
stacked = torch.stack(tasks, dim=0)
|
||||
else:
|
||||
# Pad the leading (task) dimension only. The padded slots belong to
|
||||
# ranks whose share is short by construction, and the unpacking loop
|
||||
# below never reads them back.
|
||||
stacked = tasks[0].new_empty((max_tasks, *tasks[0].shape))
|
||||
torch.stack(tasks, dim=0, out=stacked[: len(tasks)])
|
||||
stacked[len(tasks) :].zero_()
|
||||
|
||||
need_bytes = sp_size * stacked.numel() * stacked.element_size()
|
||||
workspace = self._tile_gather_workspace
|
||||
if workspace is None or workspace.device != stacked.device or workspace.numel() < need_bytes:
|
||||
# Drop the previous block before allocating its replacement so the
|
||||
# two are never resident at once; only one workspace is ever held.
|
||||
workspace = None
|
||||
self._tile_gather_workspace = None
|
||||
workspace = torch.empty(need_bytes, dtype=torch.uint8, device=stacked.device)
|
||||
self._tile_gather_workspace = workspace
|
||||
self._tile_gather_stats["allocs"] += 1
|
||||
self._tile_gather_stats["workspace_bytes"] = need_bytes
|
||||
reuse = "alloc"
|
||||
else:
|
||||
self._tile_gather_stats["hits"] += 1
|
||||
reuse = "hit"
|
||||
buffer = workspace[:need_bytes].view(stacked.dtype).view(sp_size, *stacked.shape)
|
||||
# Quantities, not just presence: a line that only says "installed"
|
||||
# cannot distinguish a buffer that is being reused from one that is
|
||||
# reallocated every request, which is the whole failure being fixed.
|
||||
# Debug level, because this fires once per decoder tile batch (12 times
|
||||
# per request on the canonical 1344x768 geometry) and the one-shot
|
||||
# "installed" line above is what an operator needs at info level.
|
||||
logger.debug(
|
||||
"[H3_VAE_GATHER] reuse=%s shape=%s/%s need_mib=%.2f workspace_mib=%.2f buf_ptr=0x%x hits=%d allocs=%d",
|
||||
reuse,
|
||||
tuple(stacked.shape),
|
||||
stacked.dtype,
|
||||
need_bytes / (1024.0 * 1024.0),
|
||||
self._tile_gather_stats["workspace_bytes"] / (1024.0 * 1024.0),
|
||||
workspace.data_ptr(),
|
||||
self._tile_gather_stats["hits"],
|
||||
self._tile_gather_stats["allocs"],
|
||||
)
|
||||
dist.all_gather_into_tensor(buffer, stacked, group=group)
|
||||
|
||||
results: list[torch.Tensor] = [None] * num_tiles # type: ignore[list-item]
|
||||
for rank in range(sp_size):
|
||||
num_rank_tasks = -(-(num_tiles - rank) // sp_size)
|
||||
for k in range(num_rank_tasks):
|
||||
results[k * sp_size + rank] = buffer[rank][k].clone()
|
||||
return results
|
||||
|
||||
def _decoder_tile_count(self, latent: torch.Tensor) -> int:
|
||||
"""Number of decoder tiles the checkpoint will split ``latent`` into.
|
||||
|
||||
@@ -464,6 +622,7 @@ class MiniMaxH3AudioVAE(nn.Module):
|
||||
*,
|
||||
device: torch.device,
|
||||
load_device: torch.device | None = None,
|
||||
trust_remote_code: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._device_target = device
|
||||
@@ -471,6 +630,7 @@ class MiniMaxH3AudioVAE(nn.Module):
|
||||
self.remote = _load_remote_component(
|
||||
component_path,
|
||||
self.config_dict,
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
# The checkpoint's audio VAE contract is FP32 for both reference
|
||||
# encoding and waveform decoding.
|
||||
|
||||
Reference in new issue
Block a user