[Diffusion] Integrate no-AllGather DLO with Host Weight Runtime (#6486)

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Signed-off-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
Co-authored-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
This commit is contained in:
Hongsheng Liuandhsliuustc0106 authored and GitHub committed 2026-08-25 08:03:53 +08:00
1 parent 8fdb719a41
commit 8360346a66
27 files changed
+1640 -51

No files matched your search

@@ -1,17 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Tests for the DiffusersPipelineLoader.
"""
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from huggingface_hub import snapshot_download
from safetensors.torch import save_file
from vllm.config.load import LoadConfig
from vllm_omni.diffusion.config import get_current_diffusion_config, get_current_diffusion_config_or_none
@@ -23,6 +25,7 @@ from vllm_omni.diffusion.model_loader.host_weight_plan import (
TensorBinding,
)
from vllm_omni.diffusion.models.helios import HeliosPipeline
from vllm_omni.diffusion.models.host_weight_contract import FinalLayoutModelContract
from vllm_omni.diffusion.registry import initialize_model
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@@ -73,6 +76,73 @@ class _DummyPipelineModel(nn.Module):
return loaded
class _HWRTransformer(nn.Module):
host_weight_restore_contract = FinalLayoutModelContract(
implementation_id="test-hwr-transformer",
version="1",
)
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.arange(4, dtype=torch.float32).to(torch.bfloat16).reshape(2, 2))
def validate_restored_host_weights(self):
assert self.weight.dtype is torch.bfloat16
class _HWRPipeline(nn.Module):
def __init__(self, source_root: Path):
super().__init__()
self.transformer = _HWRTransformer()
self.load_count = 0
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=str(source_root),
subfolder=None,
revision=None,
prefix="transformer.",
fall_back_to_pt=False,
)
]
def load_weights(self, weights):
loaded: set[str] = set()
params = dict(self.named_parameters())
for name, tensor in weights:
if name in params:
params[name].data.copy_(tensor.to(dtype=params[name].dtype))
loaded.add(name)
self.load_count += 1
return loaded
def _hwr_config(model: str | Path, root: Path, *, mode: str = "preferred") -> SimpleNamespace:
return SimpleNamespace(
model=str(model),
dtype=torch.bfloat16,
host_weight_runtime_mode=mode,
host_weight_runtime_root=str(root),
enable_distributed_layerwise_offload=True,
dlo_use_allgather=False,
lora_path=None,
quantization_config=None,
diffusion_attention_config=None,
parallel_config=SimpleNamespace(
use_hsdp=False,
data_parallel_size=1,
sequence_parallel_size=1,
tensor_parallel_size=1,
pipeline_parallel_size=1,
cfg_parallel_size=1,
enable_expert_parallel=False,
ulysses_degree=1,
ring_degree=1,
allgather_degree=1,
ulysses_mode="strict",
),
)
def _make_loader_with_weights(weight_names: list[str]) -> DiffusersPipelineLoader:
od_config = SimpleNamespace(
dtype=torch.float32,
@@ -92,6 +162,81 @@ def _make_loader_with_weights(weight_names: list[str]) -> DiffusersPipelineLoade
return loader
def test_hwr_cold_publication_and_warm_restore_skip_ordinary_dit_loading(
tmp_path: Path,
monkeypatch,
):
canonical_root = tmp_path / "canonical"
canonical_root.mkdir()
save_file(
{"weight": torch.arange(4, dtype=torch.float32).to(torch.bfloat16).reshape(2, 2)},
str(canonical_root / "model.safetensors"),
)
store_root = tmp_path / "hwr-store"
def make_loader() -> tuple[DiffusersPipelineLoader, _HWRPipeline]:
loader = DiffusersPipelineLoader(LoadConfig(), _hwr_config(canonical_root, store_root))
pipeline = _HWRPipeline(canonical_root)
monkeypatch.setattr(loader, "_init_from_load_format", lambda *args, **kwargs: pipeline)
return loader, pipeline
cold_loader, cold_model = make_loader()
cold = cold_loader.load_model(load_device="cpu", device=torch.device("cpu"))
assert cold is cold_model
assert cold_model.load_count == 1
assert cold_loader._hwr_state is not None
warm_loader, warm_model = make_loader()
monkeypatch.setattr(
warm_loader,
"_process_weights_after_loading",
lambda *args, **kwargs: pytest.fail("warm HWR restore re-entered byte-changing finalization"),
)
warm = warm_loader.load_model(load_device="cpu", device=torch.device("cpu"))
assert warm is warm_model
assert warm_model.load_count == 0
assert torch.equal(warm_model.transformer.weight, cold_model.transformer.weight)
from vllm_omni.diffusion.offloader.startup import take_offload_startup_state
startup_state = take_offload_startup_state(warm)
assert startup_state is not None
warm_plan = startup_state.host_weight_plan
assert warm_plan is not None
assert warm_plan.lease_carrier is not None
warm_plan.lease_carrier.close()
def test_hwr_commit_failure_discards_model_and_reloads_without_hwr_or_mmap(tmp_path: Path, monkeypatch):
from vllm_omni.diffusion.model_loader import diffusers_loader as loader_module
loader = DiffusersPipelineLoader(LoadConfig(), _hwr_config(tmp_path, tmp_path / "store"))
models: list[_DummyPipelineModel] = []
def init_model(*args, **kwargs):
del args, kwargs
model = _DummyPipelineModel(source_prefix="transformer.")
models.append(model)
return model
def commit_error(*args, **kwargs):
del args, kwargs
raise loader_module._HWRCommitError("restore commit failed")
monkeypatch.setattr(loader, "_init_from_load_format", init_model)
monkeypatch.setattr(loader, "_get_weight_sources", lambda _model: ())
monkeypatch.setattr(loader, "_resolve_hwr", commit_error)
monkeypatch.setattr(loader, "load_weights", lambda *args, **kwargs: None)
monkeypatch.setattr(loader, "_process_weights_after_loading", lambda *args, **kwargs: None)
monkeypatch.setattr(loader, "_apply_skip_softmax_calibration", lambda *args, **kwargs: None)
recovered = loader.load_model(load_device="cpu", device=torch.device("cpu"))
assert len(models) == 2
assert recovered is models[1]
assert loader.take_host_weight_plan() is None
def _make_dlo_online_quant_config() -> OmniDiffusionConfig:
return OmniDiffusionConfig(
model="",
@@ -103,6 +248,64 @@ def _make_dlo_online_quant_config() -> OmniDiffusionConfig:
)
@pytest.mark.parametrize(
("dist_offload", "use_allgather"),
[
(False, False),
(True, True),
],
)
def test_hwr_disabled_for_noneligible_dlo_paths_without_store_interaction(
monkeypatch,
tmp_path,
dist_offload,
use_allgather,
):
"""Disabled and AllGather paths must never construct or probe HWR."""
from vllm_omni.host_weight_runtime import HostWeightRuntime
root = tmp_path / "must-not-be-touched"
loader = DiffusersPipelineLoader(LoadConfig(), _hwr_config("dummy-model", root))
model = _DummyPipelineModel(source_prefix="transformer.")
modules = SimpleNamespace(dit_names=("transformer",), dits=(model.transformer,))
def unexpected_store_construction(*args, **kwargs):
raise AssertionError(f"HWR store interaction was not eligible: {args}, {kwargs}")
monkeypatch.setattr(HostWeightRuntime, "from_config", unexpected_store_construction)
assert (
loader._resolve_hwr(
model,
modules,
dist_offload=dist_offload,
use_allgather=use_allgather,
load_format="default",
sources=tuple(model.weights_sources),
)
is None
)
assert not root.exists()
def test_required_hwr_rejects_a_model_without_a_restore_contract(tmp_path):
loader = DiffusersPipelineLoader(
LoadConfig(),
_hwr_config("dummy-model", tmp_path / "store", mode="required"),
)
model = _DummyPipelineModel(source_prefix="transformer.")
modules = SimpleNamespace(dit_names=("transformer",), dits=(model.transformer,))
with pytest.raises(ValueError, match="restore contract"):
loader._resolve_hwr(
model,
modules,
dist_offload=True,
use_allgather=False,
load_format="default",
sources=tuple(model.weights_sources),
)
@pytest.mark.parametrize("offline", [False, True])
def test_prepare_weights_honors_component_index_and_explicit_override(tmp_path, mocker, offline):
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
@@ -726,6 +726,36 @@ def test_identity_uses_resolved_revision_and_exact_semantics(tmp_path: Path) ->
assert identity_v2.key != base_identity.key
@pytest.mark.parametrize(
"parallel",
[
FinalLayoutParallelIdentity(),
FinalLayoutParallelIdentity(tensor_parallel_size=2, tensor_parallel_rank=0),
FinalLayoutParallelIdentity(tensor_parallel_size=2, tensor_parallel_rank=1),
FinalLayoutParallelIdentity(sequence_parallel_size=2, ulysses_degree=2),
FinalLayoutParallelIdentity(sequence_parallel_size=2, ring_degree=2),
FinalLayoutParallelIdentity(
tensor_parallel_size=2,
tensor_parallel_rank=1,
sequence_parallel_size=2,
ulysses_degree=2,
),
],
)
def test_parallel_identity_matrix_is_exact_for_tp_and_sp(tmp_path: Path, parallel: FinalLayoutParallelIdentity) -> None:
"""TP rank/size and SP backend are identity coordinates for HWR artifacts."""
model = _TinyPipeline()
source = _prepared_source(tmp_path)
context = _identity(model, source, request=_request(parallel=parallel))
layout = context.identity.layout
assert layout.tensor_parallel_size == parallel.tensor_parallel_size
assert layout.tensor_parallel_rank == parallel.tensor_parallel_rank
assert layout.sequence_parallel_size == parallel.sequence_parallel_size
assert layout.sequence_parallel_backend == parallel.sequence_parallel_backend
assert "data_parallel" not in context.identity.layout.metadata.to_value().get("parallel", {})
@pytest.mark.parametrize(
("semantic_request", "message"),
[
@@ -38,6 +38,7 @@ from vllm_omni.diffusion.offloader.offload_plan import (
OffloadPlan,
get_offload_plan,
)
from vllm_omni.diffusion.offloader.startup import OffloadStartupState, attach_offload_startup_state
from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model]
@@ -695,6 +696,67 @@ class _SingleBlockModel(nn.Module):
self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)])
class _HWRPipeline(nn.Module):
def __init__(self):
super().__init__()
self.transformer = _SingleBlockModel(num_blocks=3)
class _FakeHostWeightLease:
def __init__(self, resolution_id: str):
self.closed = False
self.provenance = SimpleNamespace(resolution_id=resolution_id)
def close(self):
self.closed = True
class _FakeLeaseCarrier:
def __init__(self, lease: _FakeHostWeightLease):
self.lease = lease
self.taken = False
@property
def closed(self):
return self.taken or self.lease.closed
def take(self):
if self.taken:
raise RuntimeError("already taken")
self.taken = True
return self.lease
def close(self):
if not self.taken:
self.lease.close()
def _fake_hwr_plan(resolution_id: str = "hwr-test"):
lease = _FakeHostWeightLease(resolution_id)
carrier = _FakeLeaseCarrier(lease)
plan = HostWeightPlan(
backing_kind="host_weight_runtime",
bindings={},
lease_carrier=carrier,
)
return plan, carrier, lease
def _hwr_backend():
plan, carrier, lease = _fake_hwr_plan()
backend = DistributedLayerwiseOffloadBackend(
OffloadConfig(
strategy=OffloadStrategy.DISTRIBUTED_LAYER_WISE,
pin_cpu_memory=False,
dp_size=1,
dlo_use_allgather=False,
),
torch.device("cpu"),
host_weight_plan=plan,
)
return _HWRPipeline(), backend, carrier, lease
class _MultiBlockModel(nn.Module):
_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
@@ -838,6 +900,47 @@ class TestMmapWeightLoading:
assert all(hook.rank_local_mmap for group in backend._all_hook_groups for hook in group)
backend.disable()
def test_hwr_carrier_is_taken_and_released_after_bounded_staging(self, patched_offload_runtime):
"""A warm HWR plan uses the existing two-slot rank-local transport."""
pipeline, backend, carrier, lease = _hwr_backend()
model_bytes = sum(param.numel() * param.element_size() for param in pipeline.transformer.parameters())
backend.enable(pipeline)
assert carrier.taken
assert backend._using_rank_local_mmap
assert all(len(hook.cpu_staging_buffers) == 2 for group in backend._all_hook_groups for hook in group)
staging_bytes = sum(
buffer.numel() * buffer.element_size()
for buffer in backend._all_hook_groups[0][0].cpu_staging_buffers[0].values()
)
assert staging_bytes * 2 < model_bytes
assert not lease.closed
backend.disable()
assert lease.closed
def test_hwr_backend_failure_drains_partial_setup_before_lease_close(
self,
monkeypatch,
patched_offload_runtime,
):
pipeline, backend, carrier, lease = _hwr_backend()
monkeypatch.setattr(
backend,
"_allocate_shared_buffers",
lambda hooks: (_ for _ in ()).throw(RuntimeError("device buffer allocation failed")),
)
with pytest.raises(RuntimeError, match="device buffer allocation failed"):
backend.enable(pipeline)
assert carrier.taken
assert lease.closed
assert not backend.enabled
assert not backend._blocks
assert not backend._all_hook_groups
class TestGetBlocksFromDit:
def test_get_blocks_from_dit_single_block_attr(self):
@@ -1504,6 +1607,48 @@ class TestMmapValidation:
class TestConfigValidation:
"""Tests for configuration validation in OffloadConfig / execute_request."""
@pytest.mark.parametrize(
("dp", "sp", "tp", "use_allgather", "expected_dlo_group"),
[
(1, 1, 1, True, 1),
(2, 1, 1, True, 2),
(1, 4, 1, True, 4),
(2, 2, 2, True, 2),
(2, 2, 2, False, 1),
(1, 4, 2, False, 1),
],
)
def test_dlo_group_selection_covers_dp_sp_tp_matrix(
self,
dp,
sp,
tp,
use_allgather,
expected_dlo_group,
):
"""DLO sharding follows DP, falls back to SP, and never uses TP."""
config = OffloadConfig.from_od_config(
SimpleNamespace(
enable_cpu_offload=False,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=True,
dlo_use_allgather=use_allgather,
dlo_resident_layers=0,
pin_cpu_memory=False,
parallel_config=SimpleNamespace(
data_parallel_size=dp,
use_hsdp=False,
hsdp_shard_size=-1,
hsdp_replicate_size=1,
sequence_parallel_size=sp,
tensor_parallel_size=tp,
),
)
)
assert config.dp_size == expected_dlo_group
assert config.dlo_use_allgather is use_allgather
def test_loader_plan_cannot_be_silently_dropped_on_unsupported_platform(self, monkeypatch):
import vllm_omni.diffusion.offloader as offloader_module
@@ -1538,6 +1683,53 @@ class TestConfigValidation:
host_weight_plan=plan,
)
def test_startup_recovery_stays_inside_offloader_boundary(self, monkeypatch):
import vllm_omni.diffusion.offloader as offloader_module
class FailingBackend:
def __init__(self):
self.disabled = False
def enable(self, pipeline):
del pipeline
raise RuntimeError("initial prefetch failed")
def disable(self):
self.disabled = True
warm_pipeline = nn.Module()
canonical_pipeline = nn.Module()
plan, carrier, _lease = _fake_hwr_plan("hwr-recovery")
failing_backend = FailingBackend()
calls = []
def fake_backend(od_config, device, host_weight_plan):
del od_config, device
calls.append(host_weight_plan)
return failing_backend if host_weight_plan is plan else None
attach_offload_startup_state(
warm_pipeline,
OffloadStartupState(
host_weight_plan=plan,
fresh_model_loader=lambda: canonical_pipeline,
allow_fresh_retry=True,
),
)
monkeypatch.setattr(offloader_module, "get_offload_backend", fake_backend)
recovered, backend = offloader_module.enable_offload_backend(
SimpleNamespace(),
warm_pipeline,
device=torch.device("cpu"),
)
assert recovered is canonical_pipeline
assert backend is None
assert failing_backend.disabled
assert carrier.closed
assert calls == [plan, None]
def test_hsdp_with_allgather_rejected(self):
"""HSDP + DLO + AllGather should raise ValueError (double sharding)."""
from vllm_omni.diffusion.offloader.base import OffloadConfig
@@ -825,9 +825,6 @@ def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch):
del kwargs
return SimpleNamespace(transformer=torch.nn.Identity())
def take_host_weight_plan(self):
return None
class _DummyMemoryProfiler:
consumed_memory = 0
@@ -869,8 +866,8 @@ def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch):
monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _DummyMemoryProfiler)
monkeypatch.setattr(
model_runner_module,
"get_offload_backend",
lambda od_config, device, host_weight_plan: None,
"enable_offload_backend",
lambda od_config, pipeline, device: (pipeline, None),
)
monkeypatch.setattr(
model_runner_module, "get_cache_backend", lambda cache_backend, cache_config: dummy_cache_backend
@@ -883,8 +880,6 @@ def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch):
assert dummy_cache_backend.enabled is False
@pytest.mark.core_model
@pytest.mark.cpu
def test_set_forward_context_enters_vllm_config_contexts(monkeypatch):
"""Ensure `with set_forward_context(...):` enters vllm's context managers internally and calls desired vllm functions."""
import vllm.config.vllm as vllm_config_module
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for step-level diffusion execution across runner / worker / executor / engine."""
import contextlib
@@ -879,7 +879,11 @@ class TestRunner:
monkeypatch.setattr(model_runner_module, "DiffusersPipelineLoader", _FakeLoader)
monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _FakeProfiler)
monkeypatch.setattr(model_runner_module, "get_offload_backend", lambda *args, **kwargs: None)
monkeypatch.setattr(
model_runner_module,
"enable_offload_backend",
lambda _config, pipeline, **_kwargs: (pipeline, None),
)
monkeypatch.setattr(model_runner_module, "get_cache_backend", lambda *args, **kwargs: None)
with pytest.raises(ValueError, match="RequestOnlyPipeline"):
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Integration tests for diffusion pipeline streaming output mode.
Covers:
@@ -544,7 +544,11 @@ class TestSupportedPipelines:
monkeypatch.setattr(model_runner_module, "DiffusersPipelineLoader", _FakeDiffusersPipelineLoader)
monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _FakeDeviceMemoryProfiler)
monkeypatch.setattr(model_runner_module, "get_offload_backend", lambda *args, **kwargs: None)
monkeypatch.setattr(
model_runner_module,
"enable_offload_backend",
lambda _config, pipeline, **_kwargs: (pipeline, None),
)
monkeypatch.setattr(model_runner_module, "get_cache_backend", lambda *args, **kwargs: None)
with pytest.raises(ValueError, match="NoStepPipeline"):