[diffusion][feature] Add component-selective offload policies (#5929)

Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
SYLAR authored and GitHub committed 2026-09-06 05:40:09 +08:00
1 parent 44d3ae100a
commit ea3cf99894
74 files changed
+5121 -1584

No files matched your search

@@ -487,6 +487,10 @@ def test_dlo_offload_plan_includes_token_refiner():
assert MiniMaxH3Pipeline._offload_plan.offload_submodules == {"token_refiner": "blocks"}
assert MiniMaxH3Pipeline._offload_plan.resident_dit_paths == frozenset({"transformer"})
assert MiniMaxH3Pipeline._offload_plan.encoder_component_types == {"text_encoder": "text_encoder"}
assert MiniMaxH3Pipeline._offload_plan.encoder_block_attrs == {
"text_encoder": ("vision.blocks", "text_model.layers")
}
def test_joint_postprocess_is_multiprocessing_picklable():
@@ -1185,6 +1189,12 @@ def test_text_encoder_stub_constructs_without_group_or_weights():
from vllm_omni.diffusion.models.minimax_h3.encoder import (
MiniMaxH3Qwen3VLEncoder,
)
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
MiniMaxH3Pipeline,
)
from vllm_omni.diffusion.offloader.component_utils import (
get_encoder_block_groups,
)
encoder = MiniMaxH3Qwen3VLEncoder(
"/nonexistent/text_encoder",
@@ -1197,6 +1207,25 @@ def test_text_encoder_stub_constructs_without_group_or_weights():
# The stub has no parameters, so it never contributes to the runner's
# strict missing-parameter check on non-encoder ranks.
assert list(encoder.named_parameters()) == []
assert (
get_encoder_block_groups(
encoder,
"text_encoder",
MiniMaxH3Pipeline._offload_plan,
strict=True,
)
== []
)
# An arbitrary empty module is not silently treated as a distributed
# stub; explicit plans still validate their declared paths.
with pytest.raises(ValueError, match=r"text_encoder\.vision\.blocks was not found"):
get_encoder_block_groups(
nn.Module(),
"text_encoder",
MiniMaxH3Pipeline._offload_plan,
strict=True,
)
def test_global_quant_config_is_shared_by_dit_and_encoder():
@@ -1303,10 +1332,38 @@ def test_model_offload_uses_hooked_text_encoder_call():
@pytest.mark.parametrize(
"offload_flag",
["enable_layerwise_offload", "enable_distributed_layerwise_offload"],
("enable_layerwise", "enable_distributed"),
[(True, False), (False, True)],
)
def test_layerwise_offload_releases_text_encoder(offload_flag):
def test_legacy_layer_offload_preserves_minimax_stage_lifecycle(enable_layerwise, enable_distributed):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
diffusion_offload_config=None,
enable_cpu_offload=False,
enable_layerwise_offload=enable_layerwise,
enable_distributed_layerwise_offload=enable_distributed,
)
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
vae = Mock()
with pipeline._component_on_device(vae):
pass
vae.load_to_device.assert_called_once_with()
vae.offload_to_cpu.assert_called_once_with()
def test_layerwise_encoder_selection_releases_text_encoder():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
@@ -1315,8 +1372,11 @@ def test_layerwise_offload_releases_text_encoder(offload_flag):
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=False,
diffusion_offload_config={
"mode": "layer",
"components": ["text_encoder"],
},
)
setattr(pipeline.od_config, offload_flag, True)
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
@@ -1328,7 +1388,7 @@ def test_layerwise_offload_releases_text_encoder(offload_flag):
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
def test_distributed_layerwise_offload_releases_text_encoder():
def test_layerwise_dit_only_keeps_text_encoder_resident():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
@@ -1336,7 +1396,8 @@ def test_distributed_layerwise_offload_releases_text_encoder():
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=True,
enable_distributed_layerwise_offload=False,
diffusion_offload_config={"mode": "layer", "components": ["dit"]},
)
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
@@ -1346,25 +1407,30 @@ def test_distributed_layerwise_offload_releases_text_encoder():
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
pipeline.text_encoder.offload_to_cpu.assert_not_called()
def test_distributed_layerwise_offload_stages_vae_component():
def test_dit_encoder_selection_keeps_vae_resident():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=True,
enable_distributed_layerwise_offload=False,
diffusion_offload_config={
"mode": "layer",
"components": ["dit", "text_encoder"],
},
)
component = Mock()
with pipeline._component_on_device(component):
component.load_to_device.assert_called_once_with()
component.offload_to_cpu.assert_not_called()
pass
component.offload_to_cpu.assert_called_once_with()
component.load_to_device.assert_not_called()
component.offload_to_cpu.assert_not_called()
def test_distributed_layerwise_resident_blocks_are_stage_scoped():
@@ -1413,57 +1479,6 @@ def test_distributed_layerwise_resident_blocks_can_be_skipped():
controller.offload_resident_layers.assert_not_called()
def test_encoder_layerwise_offload_keeps_tp_blocks_rank_local(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3.encoder import (
MiniMaxH3Qwen3VLEncoder,
)
class Stack(torch.nn.Module):
def __init__(self, count):
super().__init__()
self.blocks = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
class TextStack(torch.nn.Module):
def __init__(self, count):
super().__init__()
self.layers = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
hooks = []
def fake_apply(block, next_block, device, stream, pin_memory):
hook = SimpleNamespace(
block=block,
next_block=next_block,
device=device,
stream=stream,
pin_memory=pin_memory,
_prev_hook=None,
offload_layer=Mock(),
)
hooks.append(hook)
return hook
monkeypatch.setattr(
"vllm_omni.diffusion.offloader.layerwise_backend.apply_block_hook",
fake_apply,
)
monkeypatch.setattr(
"vllm_omni.platforms.current_omni_platform.Stream",
Mock(return_value="copy-stream"),
)
encoder = object.__new__(MiniMaxH3Qwen3VLEncoder)
torch.nn.Module.__init__(encoder)
encoder.device_target = torch.device("cpu")
encoder.vision = Stack(2)
encoder.text_model = TextStack(3)
encoder.enable_omni_layerwise_offload(pin_memory=False)
assert len(hooks) == 5
assert all(hook._prev_hook is not None for hook in hooks)
assert all(hook.device == torch.device("cpu") for hook in hooks)
def test_video_vae_keeps_reference_fp32_weights(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
import pytest
import torch
@@ -11,6 +13,44 @@ class _OffloadAbort(BaseException):
pass
def test_no_allgather_example_selects_distributed_backend():
from examples.offline_inference.minimax_h3.dlo_lifecycle import engine_kwargs
from vllm_omni.diffusion.offloader.base import OffloadConfig
from vllm_omni.diffusion.offloader.config import OffloadStrategy
args = SimpleNamespace(
model="/fake/MiniMax-H3/FL2VA",
mode="dlo-no-allgather",
dp_size=2,
tp_size=2,
batch_wait_ms=500.0,
init_timeout=1800.0,
)
kwargs = engine_kwargs(args)
od_config = SimpleNamespace(
diffusion_offload_config=None,
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=kwargs["enable_distributed_layerwise_offload"],
dlo_use_allgather=kwargs["dlo_use_allgather"],
dlo_resident_layers=kwargs["dlo_resident_layers"],
dlo_host_registration_limit_gib=0.0,
host_weight_runtime_mode="disabled",
pin_cpu_memory=True,
parallel_config=SimpleNamespace(
data_parallel_size=kwargs["data_parallel_size"],
sequence_parallel_size=1,
use_hsdp=False,
),
)
config = OffloadConfig.from_od_config(od_config)
assert config.strategy is OffloadStrategy.DISTRIBUTED_LAYER_WISE
assert config.dlo_resident_layers == 0
assert not config.uses_allgather("dit")
def test_encoder_non_block_children_use_one_shared_snapshot_stager(monkeypatch, mocker):
from vllm_omni.diffusion.models.minimax_h3 import encoder as encoder_module
@@ -61,6 +101,7 @@ def test_manual_component_failure_forces_retained_cache_release(mocker):
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = mocker.Mock()
pipeline.od_config.diffusion_offload_config = None
pipeline.od_config.enable_layerwise_offload = False
pipeline.od_config.enable_distributed_layerwise_offload = True
pipeline._model_cpu_offload_modules = []
@@ -83,6 +124,7 @@ def test_manual_component_offload_failure_forces_retained_cache_release(mocker):
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = mocker.Mock()
pipeline.od_config.diffusion_offload_config = None
pipeline.od_config.enable_layerwise_offload = False
pipeline.od_config.enable_distributed_layerwise_offload = True
pipeline._model_cpu_offload_modules = []
@@ -247,6 +247,55 @@ def test_h3_turbo_allows_distributed_layerwise_offload(monkeypatch):
assert captured["unsupported_offload_mode"] is None
@pytest.mark.parametrize(
("mode", "components", "expected_mode"),
[
("module", ["text_encoder"], None),
("layer", ["text_encoder"], None),
("module", ["dit"], "model-level CPU offload"),
("layer", ["dit"], "layerwise offload"),
],
)
def test_h3_turbo_offload_compatibility_follows_dit_selection(
monkeypatch,
mode,
components,
expected_mode,
):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.partition = "fl2va"
pipeline._turbo_lora_adapter_ids = set()
pipeline._native_lora_adapter_ids = set()
pipeline._lora_sigma_schedules = {}
pipeline.od_config = SimpleNamespace(
diffusion_offload_config={"mode": mode, "components": components},
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=False,
dlo_use_allgather=True,
dlo_resident_layers=0,
pin_cpu_memory=True,
)
captured = {}
def load_turbo(**kwargs):
captured.update(kwargs)
return object(), object()
monkeypatch.setattr(pipeline_module, "load_minimax_h3_turbo_lora", load_turbo)
assert pipeline._load_diffusion_lora_adapter(
lora_request=_request("turbo"),
lora_path="turbo",
dtype=torch.bfloat16,
)
assert captured["unsupported_offload_mode"] == expected_mode
def test_h3_turbo_accepts_only_the_declared_v1_artifact(tmp_path):
unsupported = tmp_path / "minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors"
_write_tiny_turbo(unsupported)
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import contextmanager
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock
import pytest
@@ -111,6 +112,66 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
remove_offload.assert_called_once_with([*dits, *stages])
def test_h3_model_cpu_offload_keeps_unselected_vaes_resident(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.transformer = torch.nn.Linear(2, 2)
pipeline.transformers_ref = torch.nn.Linear(2, 2)
pipeline.text_encoder = torch.nn.Linear(2, 2)
pipeline.video_vae = torch.nn.Linear(2, 2)
pipeline.audio_vae = torch.nn.Linear(2, 2)
apply_offload = Mock()
monkeypatch.setattr(module, "apply_sequential_offload", apply_offload)
pipeline.enable_omni_model_cpu_offload(
device=torch.device("cpu"),
pin_memory=False,
use_hsdp=False,
offload_components=frozenset({"dit", "text_encoder"}),
)
dits = [pipeline.transformer, pipeline.transformers_ref]
stages = [pipeline.text_encoder, pipeline.video_vae, pipeline.audio_vae]
apply_offload.assert_called_once_with(
dit_modules=dits,
encoder_modules=stages,
device=torch.device("cpu"),
pin_memory=False,
use_hsdp=False,
offload_initial_dits=True,
offload_dit_modules=dits,
offload_encoder_modules=[pipeline.text_encoder],
)
def test_h3_model_cpu_offload_rejects_unloaded_selected_encoder(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.transformer = torch.nn.Linear(2, 2)
pipeline.transformers_ref = torch.nn.Linear(2, 2)
pipeline.text_encoder = None
pipeline.video_vae = torch.nn.Linear(2, 2)
pipeline.audio_vae = torch.nn.Linear(2, 2)
apply_offload = Mock()
monkeypatch.setattr(module, "apply_sequential_offload", apply_offload)
with pytest.raises(ValueError, match="no loaded text encoder"):
pipeline.enable_omni_model_cpu_offload(
device=torch.device("cpu"),
pin_memory=False,
use_hsdp=False,
offload_components=frozenset({"text_encoder"}),
)
apply_offload.assert_not_called()
@pytest.mark.parametrize("decode_fails", [False, True])
def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
@@ -119,7 +180,7 @@ def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
component = torch.nn.Linear(2, 2)
events = []
events: list[tuple[Any, ...]] = []
@contextmanager
def record_component(value):
@@ -262,7 +323,7 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
torch.nn.Module.__init__(pipeline)
pipeline.audio_vae = Mock()
pipeline._model_cpu_offload_modules = [pipeline.audio_vae]
events = []
events: list[tuple[Any, ...]] = []
@contextmanager
def record_component(value):