[Perf][MiniMax-H3] Cut video VAE encode/decode memory peaks for low-memory serving (#7241)
Signed-off-by: brandneway <gyuan4892@gmail.com> Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com>
This commit is contained in:
1 parent
a9abd563f9
commit
b2b6c6c7a9
10 files changed
+1584
-29
No files matched your search
@@ -8,7 +8,7 @@ import json
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import fields, replace
|
||||
from itertools import groupby
|
||||
from pathlib import Path
|
||||
@@ -138,9 +138,11 @@ from .time_request import (
|
||||
MINIMAX_H3_SHAPE_PLANNER,
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
from .vae import MiniMaxH3AudioVAE, MiniMaxH3VideoVAE
|
||||
from .vae import MiniMaxH3AudioVAE, MiniMaxH3VideoVAE, _VideoVAEPartProxy
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.worker.input_batch import InputBatch
|
||||
from vllm_omni.diffusion.worker.utils import StepRequestState
|
||||
|
||||
@@ -376,12 +378,20 @@ def _minimax_h3_post_process(output, output_type: str = "np"):
|
||||
|
||||
def _prepare_minimax_h3_video_output(video: torch.Tensor) -> torch.Tensor:
|
||||
"""Quantize decoded frames in place before worker-to-engine transfer."""
|
||||
video = video.detach().float()
|
||||
video = video.detach()
|
||||
if video.dtype == torch.uint8:
|
||||
# Streaming decode already quantized and clamped; only the transfer
|
||||
# layout remains.
|
||||
return video.permute(0, 2, 3, 4, 1).contiguous()
|
||||
video = video.float()
|
||||
video.clamp_(0, 1).mul_(255).round_()
|
||||
return video.permute(0, 2, 3, 4, 1).to(
|
||||
dtype=torch.uint8,
|
||||
memory_format=torch.contiguous_format,
|
||||
)
|
||||
permuted = video.permute(0, 2, 3, 4, 1)
|
||||
out = torch.empty(permuted.shape, dtype=torch.uint8, device=video.device)
|
||||
# copy_ fuses the layout change and the cast into one kernel;
|
||||
# ``.to(dtype=uint8, memory_format=contiguous_format)`` materializes a
|
||||
# contiguous FP32 intermediate first (~4.2GB for a 15s clip).
|
||||
out.copy_(permuted)
|
||||
return out
|
||||
|
||||
|
||||
def _register_dlo_component_cache(cache: BoundedAllocatorCache, *components: Any) -> None:
|
||||
@@ -1342,6 +1352,15 @@ class MiniMaxH3Pipeline(
|
||||
@contextmanager
|
||||
def _component_on_device(self, component: nn.Module):
|
||||
if getattr(self, "_model_cpu_offload_modules", None):
|
||||
# Sequential offload hooks whole modules (enable_omni_model_cpu_offload
|
||||
# registers them on the discovered components, e.g. the real
|
||||
# video_vae). Split-residency proxies carry no hook, so unwrap to the
|
||||
# hooked module before entering the context — whole-module movement
|
||||
# has no half-residency benefit anyway. The check is a type check, not
|
||||
# getattr: Mock components auto-create any attribute, which would
|
||||
# unwrap them to a child mock and break scope tracking.
|
||||
if isinstance(component, _VideoVAEPartProxy):
|
||||
component = component.sequential_offload_target
|
||||
with sequential_offload_component(component):
|
||||
yield
|
||||
return
|
||||
@@ -1381,6 +1400,50 @@ class MiniMaxH3Pipeline(
|
||||
"""Whether this rank's output is returned by the diffusion executor."""
|
||||
return not dist.is_available() or not dist.is_initialized() or dist.get_rank() == 0
|
||||
|
||||
def _encode_visual_conditions(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
prepared_videos: list[dict[str, Any]] | None,
|
||||
*,
|
||||
video_count: int,
|
||||
) -> tuple[torch.Tensor | None, list[tuple[int, int, int]]]:
|
||||
rows: list[torch.Tensor] = []
|
||||
shapes: list[tuple[int, int, int]] = []
|
||||
_, rank, _ = _dit_rank_world()
|
||||
# Keep image and video references in one residency window when both
|
||||
# appear in a request; otherwise the video branch would reload the VAE.
|
||||
# Encoding touches only the CNN encoder half, so the 9GB ViT decoder
|
||||
# stays off the device for the whole window.
|
||||
needs_video_vae = video_count > 0 or (rank == 0 and bool(images))
|
||||
video_vae_context = (
|
||||
self._component_on_device(self.video_vae.encoder_component) if needs_video_vae else nullcontext()
|
||||
)
|
||||
with video_vae_context:
|
||||
if images:
|
||||
image_rows = None
|
||||
if rank == 0:
|
||||
image_rows = torch.cat([self.video_vae.encode_image(image) for image in images])
|
||||
rows.append(
|
||||
_broadcast_tensor(
|
||||
image_rows,
|
||||
dtype=torch.float32,
|
||||
device=self.device,
|
||||
)
|
||||
)
|
||||
shapes.extend((1, image.height // 16, image.width // 16) for image in images)
|
||||
if video_count:
|
||||
video_rows, video_shapes = self._encode_video_conditions_resident(
|
||||
prepared_videos,
|
||||
count=video_count,
|
||||
)
|
||||
rows.append(video_rows)
|
||||
shapes.extend(video_shapes)
|
||||
# The latents are extracted; the encode's input/staging pages are idle
|
||||
# and must not stay mapped through the denoise and decode peaks.
|
||||
if needs_video_vae:
|
||||
self._release_stage_cache()
|
||||
return (torch.cat(rows) if rows else None), shapes
|
||||
|
||||
def _offload_model_cpu_stage_output(self, tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Release a decoded output's storage before a later seed reloads the DiT.
|
||||
|
||||
@@ -1768,6 +1831,26 @@ class MiniMaxH3Pipeline(
|
||||
raise ValueError("MiniMax H3 chunked MP4 encoding currently expects one output per decoder")
|
||||
return videos[0]
|
||||
|
||||
def _release_stage_cache(self) -> None:
|
||||
"""Return idle allocator pages to the device at stage boundaries.
|
||||
|
||||
The bounded component cache releases only past its idle-cache bound
|
||||
(>25% of device capacity), which on large devices lets a finished
|
||||
stage's freed activations -- the DiT's denoise buffers, an encode
|
||||
input, a decoded frame tensor -- stay physically mapped across the
|
||||
next stage's peak. Forcing the release at a boundary is exact (only
|
||||
free pages are returned; live tensors are untouched) and costs one
|
||||
remap per later allocation.
|
||||
"""
|
||||
cache = getattr(self, "_dlo_component_cache", None)
|
||||
if cache is not None:
|
||||
try:
|
||||
cache.release_if_needed(force=True)
|
||||
except BaseException:
|
||||
logger.exception("Failed to release retained allocator cache at stage boundary")
|
||||
return
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
def decode(
|
||||
self,
|
||||
video_latent: torch.Tensor,
|
||||
@@ -1776,7 +1859,11 @@ class MiniMaxH3Pipeline(
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
with self._component_on_device(self.video_vae):
|
||||
# Denoise just ended: its freed activation pages must not stay mapped
|
||||
# through the VAE decode peak. Decoding needs only the ViT decoder
|
||||
# half of the VAE, so the CNN encoder stays off the device.
|
||||
self._release_stage_cache()
|
||||
with self._component_on_device(self.video_vae.decoder_component):
|
||||
with current_omni_platform.create_autocast_context(
|
||||
device_type=self.device.type,
|
||||
dtype=torch.float16,
|
||||
@@ -2199,6 +2286,11 @@ class MiniMaxH3Pipeline(
|
||||
# loop rebinds it. post_decode() rebinds for the same reason.
|
||||
video = self._offload_model_cpu_stage_output(_prepare_minimax_h3_video_output(video))
|
||||
videos.append(video)
|
||||
# The FP32 decoded frames are quantized into the appended uint8
|
||||
# tensor; drop the reference and return the idle pages instead of
|
||||
# holding them through the next output's denoise/decode.
|
||||
del video
|
||||
self._release_stage_cache()
|
||||
audios.append(audio)
|
||||
if videos and isinstance(videos[0], bytes):
|
||||
video = videos[0] if len(videos) == 1 else videos
|
||||
@@ -2518,6 +2610,7 @@ class MiniMaxH3Pipeline(
|
||||
width=shape["width"],
|
||||
)
|
||||
video = self._offload_model_cpu_stage_output(_prepare_minimax_h3_video_output(video))
|
||||
self._release_stage_cache()
|
||||
return DiffusionOutput(
|
||||
output=(video, audio),
|
||||
post_process_func=get_minimax_h3_post_process_func(self.od_config),
|
||||
|
||||
@@ -6,11 +6,13 @@ from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
@@ -33,11 +35,15 @@ from vllm_omni.diffusion.offloader.module_residency import (
|
||||
from .chunked_decode import decode_h3_chunks
|
||||
from .ops import install_h3_vae_optimizations
|
||||
from .packed_tokens import minimax_h3_patchify_video_latent
|
||||
from .vae_temporal import install_temporal_stream_patches
|
||||
|
||||
MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42
|
||||
MINIMAX_H3_AUDIO_SAMPLE_RATE = 32000
|
||||
MINIMAX_H3_AUDIO_CHANNELS = 2
|
||||
|
||||
# Escape hatch back to the checkpoint's whole-video numpy preparation path.
|
||||
_VAE_ENCODE_LEGACY_PREP_ENV = "VLLM_OMNI_VAE_ENCODE_LEGACY_PREP"
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -63,6 +69,11 @@ def _minimax_h3_keyframe_encode_context(
|
||||
yield
|
||||
|
||||
|
||||
def _legacy_encode_prep_enabled() -> bool:
|
||||
value = os.environ.get(_VAE_ENCODE_LEGACY_PREP_ENV, "0")
|
||||
return value.strip().lower() not in ("", "0", "false", "off")
|
||||
|
||||
|
||||
def _load_component_config(component_path: str) -> dict[str, Any]:
|
||||
config_path = Path(component_path) / "config.json"
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
@@ -264,6 +275,49 @@ class _AudioVAEDeterminismContext(AbstractContextManager):
|
||||
return False
|
||||
|
||||
|
||||
class _VideoVAEPartProxy(nn.Module):
|
||||
"""Residency proxy for one half of the split video VAE.
|
||||
|
||||
The pipeline's ``_component_on_device`` drives whatever object it is
|
||||
given through ``load_to_device``/``offload_to_cpu``. The checkpoint's
|
||||
ViT decoder holds ~9GB of FP32 weights while the CNN encoder holds
|
||||
~0.7GB, and each is used by exactly one direction, so routing those
|
||||
calls per half keeps the decoder off the device while reference videos
|
||||
encode and the encoder off while latents decode. ``object.__setattr__``
|
||||
keeps the back-reference out of the module registry (registering the
|
||||
adapter as a submodule would recurse through ``parameters()``).
|
||||
"""
|
||||
|
||||
def __init__(self, vae: MiniMaxH3VideoVAE, part: str) -> None:
|
||||
super().__init__()
|
||||
object.__setattr__(self, "_vae", vae)
|
||||
object.__setattr__(self, "_part", part)
|
||||
|
||||
def load_to_device(self) -> None:
|
||||
self._vae._load_part_to_device(self._part)
|
||||
|
||||
def offload_to_cpu(self) -> None:
|
||||
self._vae._offload_part_to_cpu(self._part)
|
||||
|
||||
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
|
||||
self._vae.set_omni_component_cache(cache)
|
||||
|
||||
@property
|
||||
def sequential_offload_target(self) -> MiniMaxH3VideoVAE:
|
||||
"""The module model-level CPU offload actually hooks.
|
||||
|
||||
``enable_omni_model_cpu_offload`` registers the sequential hook on the
|
||||
real ``video_vae``, and the hook moves the whole module through
|
||||
``parameters()`` — entering the sequential context through this proxy
|
||||
would find no ``_hook_registry`` and raise, and whole-module movement
|
||||
has no half-residency benefit anyway. ``_component_on_device`` unwraps
|
||||
through this property before entering its sequential-offload branch;
|
||||
the manual per-half staging above stays proxy-driven, where split
|
||||
residency is the whole point.
|
||||
"""
|
||||
return self._vae
|
||||
|
||||
|
||||
class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
"""Adapter around the checkpoint's native parallel-tiled video VAE."""
|
||||
|
||||
@@ -303,14 +357,15 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
decoder,
|
||||
device=device,
|
||||
)
|
||||
self._stager = None
|
||||
if initial_device.type == "cpu" and device.type not in ("cpu", "meta"):
|
||||
self._stager = PinnedModuleStager(
|
||||
self.remote,
|
||||
device,
|
||||
pin_memory=True,
|
||||
)
|
||||
install_temporal_stream_patches(self.remote.model)
|
||||
self.model = self.remote.model
|
||||
self._stager = None
|
||||
self._encoder_stager = None
|
||||
self._decoder_stager = None
|
||||
if initial_device.type == "cpu" and device.type not in ("cpu", "meta"):
|
||||
self._build_residency_stagers(device)
|
||||
self.encoder_component = _VideoVAEPartProxy(self, "encoder")
|
||||
self.decoder_component = _VideoVAEPartProxy(self, "decoder")
|
||||
self.use_tiling = True
|
||||
self.use_slicing = False
|
||||
self.parallel_size = 1
|
||||
@@ -321,27 +376,97 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
if self._tile_gather_reuse_enabled():
|
||||
self._install_persistent_tile_gather()
|
||||
|
||||
def load_to_device(self) -> None:
|
||||
def _build_residency_stagers(self, device: torch.device) -> None:
|
||||
"""Stage the encode and decode halves of the remote separately.
|
||||
|
||||
The encode path touches only ``encoder`` + ``quant_conv`` and the
|
||||
decode path only ``post_quant_conv`` + ``decoder`` (verified against
|
||||
the checkpoint's ``AutoencoderKLLegacy``: the one cross reference,
|
||||
``tiled_decode``'s ``getattr(self.encoder, "mask_enabled")``, is
|
||||
short-circuited by ``self.training`` at inference). No storage is
|
||||
shared across the two groups, so independent staging is exact. When
|
||||
the checkpoint's structure is not discoverable, fall back to
|
||||
whole-module staging (the previous single-stager behavior).
|
||||
"""
|
||||
part_names = ("encoder", "quant_conv", "post_quant_conv", "decoder")
|
||||
if all(isinstance(getattr(self.model, name, None), nn.Module) for name in part_names):
|
||||
self._encoder_stager = PinnedModuleStager(
|
||||
[self.model.encoder, self.model.quant_conv],
|
||||
device,
|
||||
pin_memory=True,
|
||||
)
|
||||
self._decoder_stager = PinnedModuleStager(
|
||||
[self.model.post_quant_conv, self.model.decoder],
|
||||
device,
|
||||
pin_memory=True,
|
||||
)
|
||||
return
|
||||
self._stager = PinnedModuleStager(
|
||||
self.remote,
|
||||
device,
|
||||
pin_memory=True,
|
||||
)
|
||||
|
||||
def _part_stager(self, part: str) -> PinnedModuleStager | None:
|
||||
if part == "encoder":
|
||||
return self._encoder_stager
|
||||
if part == "decoder":
|
||||
return self._decoder_stager
|
||||
raise ValueError(f"unknown video VAE part {part!r}")
|
||||
|
||||
def _load_part_to_device(self, part: str) -> None:
|
||||
stager = self._part_stager(part)
|
||||
if stager is not None:
|
||||
stager.load()
|
||||
elif self._stager is not None:
|
||||
self._stager.load()
|
||||
else:
|
||||
# No staged residency (the component lives on the device already
|
||||
# or is fully CPU-resident): fall back to whole-module placement.
|
||||
self.remote.to(self._device_target)
|
||||
|
||||
def _offload_part_to_cpu(self, part: str) -> None:
|
||||
stager = self._part_stager(part)
|
||||
if stager is not None:
|
||||
stager.offload()
|
||||
return
|
||||
if self._stager is not None:
|
||||
self._stager.offload()
|
||||
return
|
||||
self.remote.to("cpu")
|
||||
self._release_component_cache()
|
||||
|
||||
def _release_component_cache(self) -> None:
|
||||
cache = getattr(self, "_omni_component_cache", None)
|
||||
if cache is None:
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
cache.release_if_needed()
|
||||
|
||||
def load_to_device(self) -> None:
|
||||
if self._encoder_stager is not None:
|
||||
self._encoder_stager.load()
|
||||
self._decoder_stager.load()
|
||||
elif self._stager is not None:
|
||||
self._stager.load()
|
||||
else:
|
||||
self.remote.to(self._device_target)
|
||||
|
||||
def set_omni_component_cache(self, cache: BoundedAllocatorCache | None) -> None:
|
||||
self._omni_component_cache = cache
|
||||
if self._stager is not None:
|
||||
self._stager.set_cache_retention(cache)
|
||||
for stager in (self._encoder_stager, self._decoder_stager, self._stager):
|
||||
if stager is not None:
|
||||
stager.set_cache_retention(cache)
|
||||
|
||||
def offload_to_cpu(self) -> None:
|
||||
if self._stager is not None:
|
||||
if self._encoder_stager is not None:
|
||||
self._encoder_stager.offload()
|
||||
self._decoder_stager.offload()
|
||||
elif self._stager is not None:
|
||||
self._stager.offload()
|
||||
else:
|
||||
self.remote.to("cpu")
|
||||
cache = getattr(self, "_omni_component_cache", None)
|
||||
if cache is None:
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
cache.release_if_needed()
|
||||
self._release_component_cache()
|
||||
|
||||
def set_parallel_size(
|
||||
self,
|
||||
@@ -650,6 +775,91 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
patch_size=(1, 2, 2),
|
||||
).float()
|
||||
|
||||
def _stream_prepare_video_tensor(
|
||||
self,
|
||||
frames: np.ndarray,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
"""Upload a uint8 ``(T, H, W, 3)`` video as one normalized FP32 tensor.
|
||||
|
||||
The checkpoint's numpy path uploads the whole video as FP32, then
|
||||
``transform_tensor`` materializes a second normalized copy, then
|
||||
``encode_temporal`` pads through a full-video ``torch.cat``: three
|
||||
resident pixel-scale copies on the device. Here the padding is
|
||||
replicated on the host, a single preallocated ``(3, T', H, W)`` tensor
|
||||
receives clip-by-clip uploads, and the ``÷255 → (x-mean)/std`` chain
|
||||
runs in place on each clip's staging buffer in the same op order as
|
||||
``convert_numpy_to_tensor`` → ``transform_tensor``, so peak device
|
||||
memory is the output tensor plus one clip instead of three copies.
|
||||
|
||||
Returns ``None`` whenever the checkpoint contract this mirrors (uint8
|
||||
frames, ``clip_length`` alignment, processor ``transform`` constants)
|
||||
is not discoverable; callers fall back to the legacy path.
|
||||
"""
|
||||
if frames.dtype != np.uint8 or frames.ndim != 4 or frames.shape[-1] != 3:
|
||||
return None
|
||||
if int(frames.shape[0]) == 0:
|
||||
return None
|
||||
model = self.model
|
||||
clip_length = getattr(model, "clip_length", None)
|
||||
transform = getattr(getattr(model, "processor", None), "transform", None)
|
||||
mean = getattr(transform, "mean", None)
|
||||
std = getattr(transform, "std", None)
|
||||
if not isinstance(clip_length, int) or clip_length <= 0:
|
||||
return None
|
||||
if mean is None or std is None or len(mean) != 3 or len(std) != 3:
|
||||
return None
|
||||
# Mirror the checkpoint's temporal alignment so the device-side
|
||||
# ``get_suitable_video_length`` trim is a no-op and the
|
||||
# ``encode_temporal`` padding ``torch.cat`` never triggers. A
|
||||
# trim-stable length is ``k * clip_length + tail`` (tail = frame
|
||||
# overlap plus the isolated last frame); it also satisfies
|
||||
# encode_temporal only when ``tail % clip_length == offset_frame``.
|
||||
# With an asymmetric checkpoint (isolated last frame but no isolated
|
||||
# first frame) no trim-stable length can satisfy encode_temporal, so
|
||||
# the remote re-pads through a whole-video cat regardless and any
|
||||
# host-side pad would be trimmed away before that -- keep the legacy
|
||||
# frame count there instead of uploading dead frames.
|
||||
isolated_first_frame = bool(getattr(model, "isolated_first_frame", False))
|
||||
frame_pre_padding = int(getattr(model, "frame_pre_padding", 0) or 0)
|
||||
offset = 1 if isolated_first_frame and frame_pre_padding == 0 else 0
|
||||
processor = getattr(model, "processor", None)
|
||||
tail = int(getattr(processor, "frame_overlap", 0) or 0)
|
||||
if bool(getattr(processor, "isolated_last_frame", False)):
|
||||
tail += 1
|
||||
num_frames = int(frames.shape[0])
|
||||
pad = 0
|
||||
if tail % clip_length == offset:
|
||||
align = getattr(processor, "align_video_length", None)
|
||||
if callable(align):
|
||||
pad = max(0, int(align(num_frames, mode="pad", granularity="chunk")))
|
||||
else:
|
||||
chunks = -(-(num_frames - tail) // clip_length)
|
||||
pad = max(max(chunks, 1) * clip_length + tail - num_frames, 0)
|
||||
if pad:
|
||||
frames = np.concatenate([frames, np.repeat(frames[-1:], pad, axis=0)])
|
||||
num_frames += pad
|
||||
mean_t = torch.as_tensor(mean, dtype=torch.float32, device=device).view(3, 1, 1, 1)
|
||||
std_t = torch.as_tensor(std, dtype=torch.float32, device=device).view(3, 1, 1, 1)
|
||||
height, width = int(frames.shape[1]), int(frames.shape[2])
|
||||
out = torch.empty(
|
||||
(3, num_frames, height, width),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
for start in range(0, num_frames, clip_length):
|
||||
end = min(start + clip_length, num_frames)
|
||||
# Frame slices of a C-contiguous (T, H, W, 3) array stay
|
||||
# contiguous, so each clip uploads at uint8 width (4x less host
|
||||
# traffic than the legacy FP32 upload) before the in-place
|
||||
# normalization chain. permute lands on the checkpoint's
|
||||
# (3, T, H, W) layout directly.
|
||||
chunk = torch.from_numpy(frames[start:end]).to(device=device)
|
||||
chunk = chunk.permute(3, 0, 1, 2).to(torch.float32).div_(255.0)
|
||||
chunk = chunk.sub_(mean_t).div_(std_t)
|
||||
out[:, start:end].copy_(chunk)
|
||||
return out
|
||||
|
||||
@torch.inference_mode()
|
||||
def encode_video(
|
||||
self,
|
||||
@@ -679,6 +889,14 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
for device in devices:
|
||||
with self.device_module.device(device):
|
||||
self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)
|
||||
prepared = None
|
||||
if isinstance(frames, np.ndarray) and not _legacy_encode_prep_enabled():
|
||||
prepared = self._stream_prepare_video_tensor(frames, parameter.device)
|
||||
if prepared is not None:
|
||||
# Tensor inputs already carry the checkpoint's expected
|
||||
# (3, T, H, W) normalized-FP32 contract, so encode_videos
|
||||
# skips its own convert/transform/pad whole-video copies.
|
||||
frames = [prepared]
|
||||
latent = self.model.encode_videos(
|
||||
frames,
|
||||
use_fp16_latent=True,
|
||||
@@ -738,7 +956,12 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
raise RuntimeError("MiniMax H3 encode-only video VAE cannot decode latents")
|
||||
with self._decode_tiling_context(latent):
|
||||
decoded = self.model.decode_base(self._denormalize_latent(latent))
|
||||
return self._normalize_decoded_frames(self.model.processor.revert_tensor(decoded))
|
||||
if decoded.dtype == torch.uint8:
|
||||
# The streaming uint8 write-back already ran the revert and the
|
||||
# output quantizer inside write_part; nothing remains but the
|
||||
# shape contract.
|
||||
return self._normalize_decoded_frames(decoded)
|
||||
return self._normalize_decoded_frames(self._revert_decoded_inplace(decoded))
|
||||
|
||||
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
channels = int(self.config_dict["latent_channels"])
|
||||
@@ -755,7 +978,7 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
frames = frames.unsqueeze(0).transpose(1, 2)
|
||||
if frames.ndim != 5:
|
||||
raise ValueError(f"unexpected decoded video shape {tuple(frames.shape)}")
|
||||
return frames.float()
|
||||
return frames if frames.dtype == torch.uint8 else frames.float()
|
||||
|
||||
# Chunks are reverted through the checkpoint's processor, which
|
||||
# denormalizes and clamps into the unit interval.
|
||||
@@ -791,6 +1014,33 @@ class MiniMaxH3VideoVAE(nn.Module, DistributedVaeMixin):
|
||||
with self._decode_tiling_context(z):
|
||||
decode_h3_chunks(self, z, on_chunk, group=group)
|
||||
|
||||
def _revert_decoded_inplace(self, decoded: torch.Tensor) -> torch.Tensor:
|
||||
"""In-place counterpart of the processor's ``revert_tensor``.
|
||||
|
||||
The checkpoint version materializes a denormalized copy, a clamped
|
||||
copy, and a contiguous copy of the whole decoded video -- three
|
||||
pixel-scale tensors resident at the decode peak. Decoding owns the
|
||||
tensor here, so the same op order (torchvision ``Normalize`` is
|
||||
``(x - mean) / std``, then ``clamp(0, 1)``) runs in place on the
|
||||
original ``(B, C, T, H, W)`` layout, which is bit-identical
|
||||
elementwise to normalizing the ``(b t) c h w`` rearrangement and
|
||||
returns with zero whole-video copies.
|
||||
|
||||
Falls back to ``processor.revert_tensor`` when the checkpoint's
|
||||
denormalization constants are not discoverable.
|
||||
"""
|
||||
processor = getattr(self.model, "processor", None)
|
||||
transform_rev = getattr(processor, "transform_rev", None)
|
||||
mean = getattr(transform_rev, "mean", None)
|
||||
std = getattr(transform_rev, "std", None)
|
||||
if mean is None or std is None or len(mean) != 3 or len(std) != 3:
|
||||
return processor.revert_tensor(decoded)
|
||||
if bool(getattr(processor, "use_3d_conv", True)) and decoded.ndim == 4:
|
||||
decoded = decoded.unsqueeze(2)
|
||||
mean_t = torch.as_tensor(mean, dtype=decoded.dtype, device=decoded.device).view(1, 3, 1, 1, 1)
|
||||
std_t = torch.as_tensor(std, dtype=decoded.dtype, device=decoded.device).view(1, 3, 1, 1, 1)
|
||||
return decoded.sub_(mean_t).div_(std_t).clamp_(0.0, 1.0)
|
||||
|
||||
|
||||
class MiniMaxH3AudioVAE(nn.Module):
|
||||
def __init__(
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Memory-optimized forks of the checkpoint's temporal VAE loops.
|
||||
|
||||
Two method replacements installed on the checkpoint's ``AutoencoderKLLegacy``
|
||||
at adapter construction:
|
||||
|
||||
* ``encode_temporal`` pads misaligned frame counts by concatenating repeated
|
||||
last frames onto the *whole* video before the chunk loop (~4GB copy for a
|
||||
15s clip). The fork pads inside the tail chunk instead: the chunks see
|
||||
bit-identical frames while the copy shrinks to a single clip.
|
||||
|
||||
* ``_decode_temporal_streaming`` accumulates the whole decoded video in a
|
||||
floating-point buffer that is denormalized, clamped, and quantized to
|
||||
uint8 only after the final chunk lands. The fork fuses the adapter's
|
||||
in-place revert and the output quantizer into ``write_part`` and
|
||||
accumulates directly into a uint8 buffer -- the same per-element op order
|
||||
(``(x - mean) / std -> clamp(0, 1) -> *255 -> round -> uint8``), so the
|
||||
delivered video is bit-identical while the resident decode buffer shrinks
|
||||
4x.
|
||||
|
||||
Set ``VLLM_OMNI_VAE_LEGACY_TEMPORAL=1`` to keep the checkpoint's methods.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import MethodType
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_LEGACY_TEMPORAL_ENV = "VLLM_OMNI_VAE_LEGACY_TEMPORAL"
|
||||
|
||||
|
||||
def _legacy_temporal_enabled() -> bool:
|
||||
value = os.environ.get(_LEGACY_TEMPORAL_ENV, "0")
|
||||
return value.strip().lower() not in ("", "0", "false", "off")
|
||||
|
||||
|
||||
def _encode_temporal_tail_pad(self, x):
|
||||
"""Drop-in fork of the checkpoint's ``encode_temporal``.
|
||||
|
||||
Frame handling is identical to the original; only the padding site
|
||||
moves from a whole-video ``torch.cat`` into the tail chunk.
|
||||
"""
|
||||
offset_frame = 1 if self.isolated_first_frame and self.frame_pre_padding == 0 else 0
|
||||
|
||||
orig_frames = int(x.shape[2])
|
||||
pad_size = (offset_frame - orig_frames) % self.clip_length
|
||||
padded_frames = orig_frames + pad_size
|
||||
num_chunks = (padded_frames - offset_frame) // self.clip_length
|
||||
|
||||
z_list = []
|
||||
for i in range(num_chunks):
|
||||
start_idx = i * self.clip_length + offset_frame
|
||||
end_idx = (i + 1) * self.clip_length + offset_frame
|
||||
clip_x = x[:, :, start_idx : min(end_idx, orig_frames), :, :]
|
||||
if end_idx > orig_frames:
|
||||
# The original pads the whole video up front by repeating the
|
||||
# last frame; repeating it inside the tail chunk feeds the
|
||||
# identical frames to the identical encodes.
|
||||
clip_x = torch.cat(
|
||||
[clip_x, clip_x[:, :, -1:].repeat(1, 1, end_idx - orig_frames, 1, 1)],
|
||||
dim=2,
|
||||
)
|
||||
|
||||
if self.isolated_key_frame:
|
||||
key_frame = clip_x[:, :, :1, :, :]
|
||||
z_key = self._adaptive_encode(key_frame)
|
||||
|
||||
if clip_x.shape[2] > 1:
|
||||
video_frames = clip_x[:, :, 1:, :, :]
|
||||
z_video = self._adaptive_encode(video_frames)
|
||||
z = torch.cat([z_key, z_video], dim=2)
|
||||
else:
|
||||
z = z_key
|
||||
else:
|
||||
z = self._adaptive_encode(clip_x)
|
||||
|
||||
z_list.append(z)
|
||||
|
||||
z = torch.cat(z_list, dim=2)
|
||||
if self.token_drop > 0:
|
||||
z = z[:, :, : -self.token_drop]
|
||||
|
||||
if self.isolated_first_frame:
|
||||
input_first_frame = x[:, :, :1, :, :]
|
||||
z_first_frame = self._adaptive_encode(input_first_frame)
|
||||
|
||||
if self.frame_pre_padding == 0:
|
||||
z = torch.cat([z_first_frame, z], dim=2)
|
||||
else:
|
||||
z = torch.cat([z_first_frame, z[:, :, 1:, :, :]], dim=2)
|
||||
|
||||
if self.isolated_last_frame:
|
||||
# The original indexes into the padded video: pad frames replicate
|
||||
# the last real frame, out-of-range indexes produce an empty slice
|
||||
# (a no-op), and negative indexes wrap. Reproduce that mapping
|
||||
# against the unpadded tensor instead of materializing the pad.
|
||||
last_frame_idx = padded_frames - self.frame_drop + offset_frame
|
||||
if last_frame_idx < 0:
|
||||
last_frame_idx += padded_frames
|
||||
if 0 <= last_frame_idx < padded_frames:
|
||||
src_idx = min(last_frame_idx, orig_frames - 1)
|
||||
input_last_frame = x[:, :, src_idx : src_idx + 1, :, :]
|
||||
z_last_frame = self._adaptive_encode(input_last_frame)
|
||||
z = torch.cat([z, z_last_frame], dim=2)
|
||||
|
||||
return z
|
||||
|
||||
|
||||
def _decode_temporal_streaming_uint8(self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype):
|
||||
"""Drop-in fork of the checkpoint's ``_decode_temporal_streaming``.
|
||||
|
||||
The chunk loop, overlap blending, frame plan, and assertions are
|
||||
identical to the original. ``write_part`` additionally runs the
|
||||
denormalize -> clamp -> quantize chain in place (the same op order the
|
||||
pipeline applies after decode) and accumulates into a uint8 buffer.
|
||||
"""
|
||||
total_frames, pad_frames, output_frames = self._decode_temporal_output_frame_plan(
|
||||
z, z_head, z_tail, num_chunks, pad_tokens
|
||||
)
|
||||
if output_frames <= 0:
|
||||
raise ValueError(
|
||||
f"decode_temporal streaming planned non-positive output_frames={output_frames} "
|
||||
f"total_frames={total_frames} pad_frames={pad_frames}"
|
||||
)
|
||||
|
||||
chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
|
||||
split_count = int(self.token_drop > 0) + 1
|
||||
dec = None
|
||||
dec_overlap = None
|
||||
norm = None
|
||||
write_pos = 0
|
||||
logical_frames = 0
|
||||
dropped_frames = 0
|
||||
decoded_count = 0
|
||||
|
||||
def write_part(part):
|
||||
nonlocal dec, dec_overlap, norm, write_pos, logical_frames, dropped_frames
|
||||
part_frames = int(part.shape[2])
|
||||
if part_frames <= 0:
|
||||
return
|
||||
logical_frames += part_frames
|
||||
if dec is None:
|
||||
out_shape = list(part.shape)
|
||||
out_shape[2] = output_frames
|
||||
dec = torch.empty(out_shape, dtype=torch.uint8, device=part.device)
|
||||
transform_rev = self.processor.transform_rev
|
||||
mean = torch.as_tensor(transform_rev.mean, dtype=part.dtype, device=part.device).view(1, 3, 1, 1, 1)
|
||||
std = torch.as_tensor(transform_rev.std, dtype=part.dtype, device=part.device).view(1, 3, 1, 1, 1)
|
||||
norm = (mean, std)
|
||||
# revert_tensor ((x - mean) / std, clamp) followed by the output
|
||||
# quantizer (*255, round, uint8), in place on the part: the blended
|
||||
# raw values are consumed before this point, and every write into
|
||||
# ``dec`` goes through here, so the final buffer matches the legacy
|
||||
# post-decode chain element for element. Denormalization and clamping
|
||||
# run in the decoded dtype exactly like the legacy adapter; the
|
||||
# quantizer then runs in FP32 because the legacy adapter returned
|
||||
# ``frames.float()`` before the pipeline multiplied and rounded —
|
||||
# rounding fp16/bf16 directly flips borderline pixels by one.
|
||||
mean_t, std_t = norm
|
||||
part.sub_(mean_t).div_(std_t).clamp_(0.0, 1.0)
|
||||
part = part.float()
|
||||
part.mul_(255.0).round_()
|
||||
|
||||
remaining = int(dec.shape[2]) - write_pos
|
||||
copy_frames = min(part_frames, max(0, remaining))
|
||||
if copy_frames > 0:
|
||||
dec[:, :, write_pos : write_pos + copy_frames, :, :].copy_(part[:, :, :copy_frames, :, :])
|
||||
write_pos += copy_frames
|
||||
dropped_frames += part_frames - copy_frames
|
||||
|
||||
for i in range(num_chunks):
|
||||
t_start_idx = i * self.tokens_chunk_size
|
||||
t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap
|
||||
clip_z = z[:, :, t_start_idx:t_end_idx, :, :]
|
||||
|
||||
if i == 0 and z_head is not None:
|
||||
clip_z = torch.cat([z_head, clip_z], dim=2)
|
||||
|
||||
if i == num_chunks - 1 and z_tail is not None:
|
||||
clip_z = torch.cat([clip_z, z_tail], dim=2)
|
||||
|
||||
clip_dec = self._adaptive_decode(clip_z)
|
||||
decoded_count += 1
|
||||
if temporal_cat_dtype is not None and clip_dec.dtype != temporal_cat_dtype:
|
||||
clip_dec = clip_dec.to(temporal_cat_dtype)
|
||||
if clip_dec.device != z.device:
|
||||
clip_dec = clip_dec.to(z.device)
|
||||
|
||||
dec_tail = None
|
||||
if i == 0 and z_head is not None:
|
||||
write_part(clip_dec[:, :, self.vae_ratio_t - 1 : self.vae_ratio_t, :, :])
|
||||
clip_dec = clip_dec[:, :, self.vae_ratio_t :, :, :]
|
||||
|
||||
if i == num_chunks - 1 and z_tail is not None:
|
||||
dec_tail = clip_dec[:, :, -1:, :, :]
|
||||
clip_dec = clip_dec[:, :, : -self.vae_ratio_t, :, :]
|
||||
|
||||
for j in range(split_count):
|
||||
f_start_idx = j * chunk_dec
|
||||
f_end_idx = min(f_start_idx + chunk_dec, clip_dec.shape[2])
|
||||
clip_dec_chunk = clip_dec[:, :, f_start_idx:f_end_idx, :, :]
|
||||
clip_dec_chunk = clip_dec_chunk[:, :, self.frame_pre_padding :, :, :]
|
||||
|
||||
if j == 0:
|
||||
if dec_overlap is not None:
|
||||
clip_dec_chunk = self.blend(dec_overlap, clip_dec_chunk, self.frame_overlap, dim=-3)
|
||||
dec_overlap = None
|
||||
write_part(clip_dec_chunk)
|
||||
else:
|
||||
# Break the view's reference to the full decoded clip so earlier
|
||||
# temporal chunks can be released before the final output exists.
|
||||
dec_overlap = clip_dec_chunk.contiguous()
|
||||
|
||||
if i == num_chunks - 1:
|
||||
if dec_overlap is not None:
|
||||
write_part(dec_overlap)
|
||||
dec_overlap = None
|
||||
if dec_tail is not None:
|
||||
write_part(dec_tail)
|
||||
|
||||
del clip_dec, clip_z
|
||||
|
||||
if dec is None:
|
||||
raise RuntimeError("decode_temporal streaming produced no output tensor")
|
||||
if logical_frames != total_frames or dropped_frames != pad_frames or write_pos != output_frames:
|
||||
raise RuntimeError(
|
||||
"decode_temporal streaming frame plan mismatch: "
|
||||
f"logical_frames={logical_frames} total_frames={total_frames} "
|
||||
f"dropped_frames={dropped_frames} pad_frames={pad_frames} "
|
||||
f"write_pos={write_pos} output_frames={output_frames}"
|
||||
)
|
||||
|
||||
return dec
|
||||
|
||||
|
||||
def _processor_denorm_available(model) -> bool:
|
||||
transform_rev = getattr(getattr(model, "processor", None), "transform_rev", None)
|
||||
mean = getattr(transform_rev, "mean", None)
|
||||
std = getattr(transform_rev, "std", None)
|
||||
return mean is not None and std is not None and len(mean) == 3 and len(std) == 3
|
||||
|
||||
|
||||
def install_temporal_stream_patches(model) -> None:
|
||||
"""Install the memory-optimized temporal forks on the checkpoint model.
|
||||
|
||||
Each fork is installed only when the checkpoint contract it mirrors is
|
||||
discoverable; otherwise the checkpoint's own method keeps running (the
|
||||
adapter's post-decode revert already covers that path).
|
||||
"""
|
||||
if _legacy_temporal_enabled():
|
||||
logger.info("MiniMax-H3 VAE temporal stream patches disabled by %s", _LEGACY_TEMPORAL_ENV)
|
||||
return
|
||||
if callable(getattr(model, "encode_temporal", None)) and hasattr(model, "clip_length"):
|
||||
model.encode_temporal = MethodType(_encode_temporal_tail_pad, model)
|
||||
logger.info("MiniMax-H3 VAE encode_temporal tail-padding patch installed")
|
||||
else:
|
||||
logger.warning("MiniMax-H3 VAE encode_temporal contract not found; keeping checkpoint method")
|
||||
if callable(getattr(model, "_decode_temporal_streaming", None)) and _processor_denorm_available(model):
|
||||
model._decode_temporal_streaming = MethodType(_decode_temporal_streaming_uint8, model)
|
||||
logger.info("MiniMax-H3 VAE streaming uint8 decode patch installed")
|
||||
else:
|
||||
logger.warning("MiniMax-H3 VAE streaming decode contract not found; keeping checkpoint method")
|
||||
|
||||
|
||||
__all__ = ["install_temporal_stream_patches"]
|
||||
Reference in new issue
Block a user