[diffusion][feature] Add component-selective offload policies (#5929)
Signed-off-by: lishunyang12 <lishunyang12@163.com>
This commit is contained in:
1 parent
44d3ae100a
commit
ea3cf99894
74 files changed
+5121
-1584
No files matched your search
@@ -487,6 +487,10 @@ def test_dlo_offload_plan_includes_token_refiner():
|
||||
|
||||
assert MiniMaxH3Pipeline._offload_plan.offload_submodules == {"token_refiner": "blocks"}
|
||||
assert MiniMaxH3Pipeline._offload_plan.resident_dit_paths == frozenset({"transformer"})
|
||||
assert MiniMaxH3Pipeline._offload_plan.encoder_component_types == {"text_encoder": "text_encoder"}
|
||||
assert MiniMaxH3Pipeline._offload_plan.encoder_block_attrs == {
|
||||
"text_encoder": ("vision.blocks", "text_model.layers")
|
||||
}
|
||||
|
||||
|
||||
def test_joint_postprocess_is_multiprocessing_picklable():
|
||||
@@ -1185,6 +1189,12 @@ def test_text_encoder_stub_constructs_without_group_or_weights():
|
||||
from vllm_omni.diffusion.models.minimax_h3.encoder import (
|
||||
MiniMaxH3Qwen3VLEncoder,
|
||||
)
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
MiniMaxH3Pipeline,
|
||||
)
|
||||
from vllm_omni.diffusion.offloader.component_utils import (
|
||||
get_encoder_block_groups,
|
||||
)
|
||||
|
||||
encoder = MiniMaxH3Qwen3VLEncoder(
|
||||
"/nonexistent/text_encoder",
|
||||
@@ -1197,6 +1207,25 @@ def test_text_encoder_stub_constructs_without_group_or_weights():
|
||||
# The stub has no parameters, so it never contributes to the runner's
|
||||
# strict missing-parameter check on non-encoder ranks.
|
||||
assert list(encoder.named_parameters()) == []
|
||||
assert (
|
||||
get_encoder_block_groups(
|
||||
encoder,
|
||||
"text_encoder",
|
||||
MiniMaxH3Pipeline._offload_plan,
|
||||
strict=True,
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
||||
# An arbitrary empty module is not silently treated as a distributed
|
||||
# stub; explicit plans still validate their declared paths.
|
||||
with pytest.raises(ValueError, match=r"text_encoder\.vision\.blocks was not found"):
|
||||
get_encoder_block_groups(
|
||||
nn.Module(),
|
||||
"text_encoder",
|
||||
MiniMaxH3Pipeline._offload_plan,
|
||||
strict=True,
|
||||
)
|
||||
|
||||
|
||||
def test_global_quant_config_is_shared_by_dit_and_encoder():
|
||||
@@ -1303,10 +1332,38 @@ def test_model_offload_uses_hooked_text_encoder_call():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"offload_flag",
|
||||
["enable_layerwise_offload", "enable_distributed_layerwise_offload"],
|
||||
("enable_layerwise", "enable_distributed"),
|
||||
[(True, False), (False, True)],
|
||||
)
|
||||
def test_layerwise_offload_releases_text_encoder(offload_flag):
|
||||
def test_legacy_layer_offload_preserves_minimax_stage_lifecycle(enable_layerwise, enable_distributed):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
diffusion_offload_config=None,
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=enable_layerwise,
|
||||
enable_distributed_layerwise_offload=enable_distributed,
|
||||
)
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
|
||||
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
vae = Mock()
|
||||
with pipeline._component_on_device(vae):
|
||||
pass
|
||||
vae.load_to_device.assert_called_once_with()
|
||||
vae.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_layerwise_encoder_selection_releases_text_encoder():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
@@ -1315,8 +1372,11 @@ def test_layerwise_offload_releases_text_encoder(offload_flag):
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=False,
|
||||
diffusion_offload_config={
|
||||
"mode": "layer",
|
||||
"components": ["text_encoder"],
|
||||
},
|
||||
)
|
||||
setattr(pipeline.od_config, offload_flag, True)
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
@@ -1328,7 +1388,7 @@ def test_layerwise_offload_releases_text_encoder(offload_flag):
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_distributed_layerwise_offload_releases_text_encoder():
|
||||
def test_layerwise_dit_only_keeps_text_encoder_resident():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
@@ -1336,7 +1396,8 @@ def test_distributed_layerwise_offload_releases_text_encoder():
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=True,
|
||||
enable_distributed_layerwise_offload=False,
|
||||
diffusion_offload_config={"mode": "layer", "components": ["dit"]},
|
||||
)
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
@@ -1346,25 +1407,30 @@ def test_distributed_layerwise_offload_releases_text_encoder():
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
pipeline.text_encoder.offload_to_cpu.assert_not_called()
|
||||
|
||||
|
||||
def test_distributed_layerwise_offload_stages_vae_component():
|
||||
def test_dit_encoder_selection_keeps_vae_resident():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=True,
|
||||
enable_distributed_layerwise_offload=False,
|
||||
diffusion_offload_config={
|
||||
"mode": "layer",
|
||||
"components": ["dit", "text_encoder"],
|
||||
},
|
||||
)
|
||||
component = Mock()
|
||||
|
||||
with pipeline._component_on_device(component):
|
||||
component.load_to_device.assert_called_once_with()
|
||||
component.offload_to_cpu.assert_not_called()
|
||||
pass
|
||||
|
||||
component.offload_to_cpu.assert_called_once_with()
|
||||
component.load_to_device.assert_not_called()
|
||||
component.offload_to_cpu.assert_not_called()
|
||||
|
||||
|
||||
def test_distributed_layerwise_resident_blocks_are_stage_scoped():
|
||||
@@ -1413,57 +1479,6 @@ def test_distributed_layerwise_resident_blocks_can_be_skipped():
|
||||
controller.offload_resident_layers.assert_not_called()
|
||||
|
||||
|
||||
def test_encoder_layerwise_offload_keeps_tp_blocks_rank_local(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3.encoder import (
|
||||
MiniMaxH3Qwen3VLEncoder,
|
||||
)
|
||||
|
||||
class Stack(torch.nn.Module):
|
||||
def __init__(self, count):
|
||||
super().__init__()
|
||||
self.blocks = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
|
||||
|
||||
class TextStack(torch.nn.Module):
|
||||
def __init__(self, count):
|
||||
super().__init__()
|
||||
self.layers = torch.nn.ModuleList([torch.nn.Linear(2, 2) for _ in range(count)])
|
||||
|
||||
hooks = []
|
||||
|
||||
def fake_apply(block, next_block, device, stream, pin_memory):
|
||||
hook = SimpleNamespace(
|
||||
block=block,
|
||||
next_block=next_block,
|
||||
device=device,
|
||||
stream=stream,
|
||||
pin_memory=pin_memory,
|
||||
_prev_hook=None,
|
||||
offload_layer=Mock(),
|
||||
)
|
||||
hooks.append(hook)
|
||||
return hook
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.offloader.layerwise_backend.apply_block_hook",
|
||||
fake_apply,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.platforms.current_omni_platform.Stream",
|
||||
Mock(return_value="copy-stream"),
|
||||
)
|
||||
encoder = object.__new__(MiniMaxH3Qwen3VLEncoder)
|
||||
torch.nn.Module.__init__(encoder)
|
||||
encoder.device_target = torch.device("cpu")
|
||||
encoder.vision = Stack(2)
|
||||
encoder.text_model = TextStack(3)
|
||||
|
||||
encoder.enable_omni_layerwise_offload(pin_memory=False)
|
||||
|
||||
assert len(hooks) == 5
|
||||
assert all(hook._prev_hook is not None for hook in hooks)
|
||||
assert all(hook.device == torch.device("cpu") for hook in hooks)
|
||||
|
||||
|
||||
def test_video_vae_keeps_reference_fp32_weights(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -11,6 +13,44 @@ class _OffloadAbort(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
def test_no_allgather_example_selects_distributed_backend():
|
||||
from examples.offline_inference.minimax_h3.dlo_lifecycle import engine_kwargs
|
||||
from vllm_omni.diffusion.offloader.base import OffloadConfig
|
||||
from vllm_omni.diffusion.offloader.config import OffloadStrategy
|
||||
|
||||
args = SimpleNamespace(
|
||||
model="/fake/MiniMax-H3/FL2VA",
|
||||
mode="dlo-no-allgather",
|
||||
dp_size=2,
|
||||
tp_size=2,
|
||||
batch_wait_ms=500.0,
|
||||
init_timeout=1800.0,
|
||||
)
|
||||
kwargs = engine_kwargs(args)
|
||||
od_config = SimpleNamespace(
|
||||
diffusion_offload_config=None,
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=kwargs["enable_distributed_layerwise_offload"],
|
||||
dlo_use_allgather=kwargs["dlo_use_allgather"],
|
||||
dlo_resident_layers=kwargs["dlo_resident_layers"],
|
||||
dlo_host_registration_limit_gib=0.0,
|
||||
host_weight_runtime_mode="disabled",
|
||||
pin_cpu_memory=True,
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_size=kwargs["data_parallel_size"],
|
||||
sequence_parallel_size=1,
|
||||
use_hsdp=False,
|
||||
),
|
||||
)
|
||||
|
||||
config = OffloadConfig.from_od_config(od_config)
|
||||
|
||||
assert config.strategy is OffloadStrategy.DISTRIBUTED_LAYER_WISE
|
||||
assert config.dlo_resident_layers == 0
|
||||
assert not config.uses_allgather("dit")
|
||||
|
||||
|
||||
def test_encoder_non_block_children_use_one_shared_snapshot_stager(monkeypatch, mocker):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import encoder as encoder_module
|
||||
|
||||
@@ -61,6 +101,7 @@ def test_manual_component_failure_forces_retained_cache_release(mocker):
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = mocker.Mock()
|
||||
pipeline.od_config.diffusion_offload_config = None
|
||||
pipeline.od_config.enable_layerwise_offload = False
|
||||
pipeline.od_config.enable_distributed_layerwise_offload = True
|
||||
pipeline._model_cpu_offload_modules = []
|
||||
@@ -83,6 +124,7 @@ def test_manual_component_offload_failure_forces_retained_cache_release(mocker):
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = mocker.Mock()
|
||||
pipeline.od_config.diffusion_offload_config = None
|
||||
pipeline.od_config.enable_layerwise_offload = False
|
||||
pipeline.od_config.enable_distributed_layerwise_offload = True
|
||||
pipeline._model_cpu_offload_modules = []
|
||||
|
||||
@@ -247,6 +247,55 @@ def test_h3_turbo_allows_distributed_layerwise_offload(monkeypatch):
|
||||
assert captured["unsupported_offload_mode"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "components", "expected_mode"),
|
||||
[
|
||||
("module", ["text_encoder"], None),
|
||||
("layer", ["text_encoder"], None),
|
||||
("module", ["dit"], "model-level CPU offload"),
|
||||
("layer", ["dit"], "layerwise offload"),
|
||||
],
|
||||
)
|
||||
def test_h3_turbo_offload_compatibility_follows_dit_selection(
|
||||
monkeypatch,
|
||||
mode,
|
||||
components,
|
||||
expected_mode,
|
||||
):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.partition = "fl2va"
|
||||
pipeline._turbo_lora_adapter_ids = set()
|
||||
pipeline._native_lora_adapter_ids = set()
|
||||
pipeline._lora_sigma_schedules = {}
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
diffusion_offload_config={"mode": mode, "components": components},
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=False,
|
||||
dlo_use_allgather=True,
|
||||
dlo_resident_layers=0,
|
||||
pin_cpu_memory=True,
|
||||
)
|
||||
captured = {}
|
||||
|
||||
def load_turbo(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return object(), object()
|
||||
|
||||
monkeypatch.setattr(pipeline_module, "load_minimax_h3_turbo_lora", load_turbo)
|
||||
|
||||
assert pipeline._load_diffusion_lora_adapter(
|
||||
lora_request=_request("turbo"),
|
||||
lora_path="turbo",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
assert captured["unsupported_offload_mode"] == expected_mode
|
||||
|
||||
|
||||
def test_h3_turbo_accepts_only_the_declared_v1_artifact(tmp_path):
|
||||
unsupported = tmp_path / "minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors"
|
||||
_write_tiny_turbo(unsupported)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
@@ -111,6 +112,66 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
|
||||
remove_offload.assert_called_once_with([*dits, *stages])
|
||||
|
||||
|
||||
def test_h3_model_cpu_offload_keeps_unselected_vaes_resident(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.transformer = torch.nn.Linear(2, 2)
|
||||
pipeline.transformers_ref = torch.nn.Linear(2, 2)
|
||||
pipeline.text_encoder = torch.nn.Linear(2, 2)
|
||||
pipeline.video_vae = torch.nn.Linear(2, 2)
|
||||
pipeline.audio_vae = torch.nn.Linear(2, 2)
|
||||
apply_offload = Mock()
|
||||
monkeypatch.setattr(module, "apply_sequential_offload", apply_offload)
|
||||
|
||||
pipeline.enable_omni_model_cpu_offload(
|
||||
device=torch.device("cpu"),
|
||||
pin_memory=False,
|
||||
use_hsdp=False,
|
||||
offload_components=frozenset({"dit", "text_encoder"}),
|
||||
)
|
||||
|
||||
dits = [pipeline.transformer, pipeline.transformers_ref]
|
||||
stages = [pipeline.text_encoder, pipeline.video_vae, pipeline.audio_vae]
|
||||
apply_offload.assert_called_once_with(
|
||||
dit_modules=dits,
|
||||
encoder_modules=stages,
|
||||
device=torch.device("cpu"),
|
||||
pin_memory=False,
|
||||
use_hsdp=False,
|
||||
offload_initial_dits=True,
|
||||
offload_dit_modules=dits,
|
||||
offload_encoder_modules=[pipeline.text_encoder],
|
||||
)
|
||||
|
||||
|
||||
def test_h3_model_cpu_offload_rejects_unloaded_selected_encoder(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.transformer = torch.nn.Linear(2, 2)
|
||||
pipeline.transformers_ref = torch.nn.Linear(2, 2)
|
||||
pipeline.text_encoder = None
|
||||
pipeline.video_vae = torch.nn.Linear(2, 2)
|
||||
pipeline.audio_vae = torch.nn.Linear(2, 2)
|
||||
apply_offload = Mock()
|
||||
monkeypatch.setattr(module, "apply_sequential_offload", apply_offload)
|
||||
|
||||
with pytest.raises(ValueError, match="no loaded text encoder"):
|
||||
pipeline.enable_omni_model_cpu_offload(
|
||||
device=torch.device("cpu"),
|
||||
pin_memory=False,
|
||||
use_hsdp=False,
|
||||
offload_components=frozenset({"text_encoder"}),
|
||||
)
|
||||
|
||||
apply_offload.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_fails", [False, True])
|
||||
def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
@@ -119,7 +180,7 @@ def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
component = torch.nn.Linear(2, 2)
|
||||
events = []
|
||||
events: list[tuple[Any, ...]] = []
|
||||
|
||||
@contextmanager
|
||||
def record_component(value):
|
||||
@@ -262,7 +323,7 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.audio_vae = Mock()
|
||||
pipeline._model_cpu_offload_modules = [pipeline.audio_vae]
|
||||
events = []
|
||||
events: list[tuple[Any, ...]] = []
|
||||
|
||||
@contextmanager
|
||||
def record_component(value):
|
||||
|
||||
Reference in new issue
Block a user