[feature] Add HSDP pre-sharded option (#7948)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
1 parent
b742f86136
commit
97d6c693a6
17 files changed
+1370
-87
No files matched your search
@@ -0,0 +1,200 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Compare full and pre-sharded Cosmos loading with real two-rank FSDP."""
|
||||
|
||||
from contextlib import ExitStack
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.core_model, pytest.mark.gpu]
|
||||
|
||||
|
||||
class _SDPAAttention(nn.Module):
|
||||
"""Use deterministic native attention while testing loading and FSDP."""
|
||||
|
||||
def __init__(self, *, causal, softmax_scale, **kwargs):
|
||||
super().__init__()
|
||||
self.causal = causal
|
||||
self.scale = softmax_scale
|
||||
|
||||
def forward(self, query, key, value):
|
||||
return F.scaled_dot_product_attention(
|
||||
query.transpose(1, 2),
|
||||
key.transpose(1, 2),
|
||||
value.transpose(1, 2),
|
||||
is_causal=self.causal,
|
||||
scale=self.scale,
|
||||
).transpose(1, 2)
|
||||
|
||||
|
||||
def _load_cosmos_worker(rank, rendezvous, checkpoint_dir, edge):
|
||||
from tests.model_executor.helpers import bootstrap_vllm_layer_custom_op_modules
|
||||
|
||||
bootstrap_vllm_layer_custom_op_modules()
|
||||
|
||||
from safetensors.torch import save_file
|
||||
from vllm.config import DeviceConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.layers import linear
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
from vllm_omni.diffusion.distributed import hsdp
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3, transformer_cosmos3_edge
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
device = torch.device("cuda", rank)
|
||||
current_omni_platform.set_device(device)
|
||||
dist.init_process_group("nccl", init_method=rendezvous, rank=rank, world_size=2, timeout=timedelta(seconds=120))
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cuda"))))
|
||||
stack.enter_context(set_default_torch_dtype(torch.bfloat16))
|
||||
# HSDP spans both workers, while every vLLM linear uses TP=1.
|
||||
for module in (linear, parameter):
|
||||
stack.enter_context(patch.object(module, "get_tensor_model_parallel_rank", return_value=0))
|
||||
stack.enter_context(patch.object(module, "get_tensor_model_parallel_world_size", return_value=1))
|
||||
for module in (transformer_cosmos3, transformer_cosmos3_edge):
|
||||
stack.enter_context(patch.object(module, "get_tensor_model_parallel_world_size", return_value=1))
|
||||
stack.enter_context(patch.object(module, "FrameworkAttention", _SDPAAttention))
|
||||
stack.enter_context(patch.object(transformer_cosmos3, "_get_ulysses_state", return_value=(1, 0, None)))
|
||||
stack.enter_context(
|
||||
patch.object(hsdp, "get_world_group", return_value=SimpleNamespace(world_size=2, rank_in_group=rank))
|
||||
)
|
||||
|
||||
model_cls = (
|
||||
transformer_cosmos3_edge.Cosmos3EdgeVFMTransformer
|
||||
if edge
|
||||
else transformer_cosmos3.Cosmos3VFMTransformer
|
||||
)
|
||||
tf_config = dict(
|
||||
hidden_size=16,
|
||||
num_hidden_layers=1,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
head_dim=8,
|
||||
intermediate_size=32,
|
||||
vocab_size=32,
|
||||
latent_patch_size=2,
|
||||
latent_channel=48,
|
||||
temporal_compression_factor=4,
|
||||
rope_scaling={"mrope_section": [2, 1, 1]},
|
||||
)
|
||||
if edge:
|
||||
tf_config.update(
|
||||
backbone_type=transformer_cosmos3_edge.COSMOS3_EDGE_BACKBONE_TYPE,
|
||||
qk_norm_for_text=False,
|
||||
use_und_k_norm_for_gen=True,
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
tf_model_config=tf_config,
|
||||
dtype=torch.bfloat16,
|
||||
quantization_config=None,
|
||||
lora_path=None,
|
||||
num_weight_load_threads=1,
|
||||
hsdp_weight_load_strategy="full",
|
||||
parallel_config=SimpleNamespace(
|
||||
use_hsdp=True, hsdp_replicate_size=1, hsdp_shard_size=2, tensor_parallel_size=1
|
||||
),
|
||||
)
|
||||
|
||||
class Pipeline(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.transformer = model_cls(config)
|
||||
self.transformer.register_buffer("scratch", torch.tensor([7.0]), persistent=False)
|
||||
self.vae = nn.Linear(2, 2)
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(checkpoint_dir, None, None, "transformer.", False)
|
||||
]
|
||||
|
||||
def load_weights(self, weights):
|
||||
state = dict(weights)
|
||||
self.transformer.load_state_dict(
|
||||
{name.removeprefix("transformer."): t for name, t in state.items()}
|
||||
)
|
||||
self.transformer.post_load_weights()
|
||||
self.transformer.eval()
|
||||
self.transformer.validate_loaded_weights(set(state))
|
||||
return set(state)
|
||||
|
||||
full = Pipeline()
|
||||
if rank == 0:
|
||||
torch.manual_seed(42)
|
||||
with torch.no_grad():
|
||||
for name, tensor in full.transformer.named_parameters():
|
||||
tensor.copy_(torch.randn_like(tensor) * 0.02)
|
||||
if "norm" in name:
|
||||
tensor.fill_(1)
|
||||
save_file(full.transformer.state_dict(), str(Path(checkpoint_dir) / "model.safetensors"))
|
||||
dist.barrier()
|
||||
|
||||
def load(model):
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), config)
|
||||
with patch.object(loader, "_init_from_load_format", return_value=model):
|
||||
return loader._load_model_with_hsdp(device)
|
||||
|
||||
full = load(full)
|
||||
config.hsdp_weight_load_strategy = "pre_sharded"
|
||||
pre_sharded = load(Pipeline())
|
||||
full_parameters = dict(full.transformer.named_parameters())
|
||||
from torch.distributed.tensor import DTensor
|
||||
|
||||
for name, actual in pre_sharded.transformer.named_parameters():
|
||||
expected = full_parameters[name]
|
||||
if name.startswith("time_embedder."):
|
||||
assert not isinstance(actual, DTensor)
|
||||
assert actual.dtype == torch.float32
|
||||
else:
|
||||
assert isinstance(actual, DTensor)
|
||||
actual = actual.full_tensor()
|
||||
expected = expected.full_tensor()
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
assert pre_sharded.vae.weight.device == device
|
||||
torch.testing.assert_close(pre_sharded.transformer.scratch, full.transformer.scratch)
|
||||
assert all(t.device.type != "meta" for t in pre_sharded.transformer.buffers())
|
||||
|
||||
inputs = dict(
|
||||
hidden_states=torch.linspace(-1, 1, 48 * 4, device=device).reshape(1, 48, 1, 2, 2),
|
||||
timestep=torch.tensor([1.0], device=device),
|
||||
text_ids=torch.tensor([[1, 2]], device=device),
|
||||
text_mask=torch.ones(1, 2, dtype=torch.long, device=device),
|
||||
video_shape=(1, 2, 2),
|
||||
fps=24.0,
|
||||
)
|
||||
with torch.no_grad():
|
||||
# Also exercise the cached UND path on the second denoising step.
|
||||
for step in (1.0, 0.5):
|
||||
inputs["timestep"].fill_(step)
|
||||
expected = full.transformer(**inputs)
|
||||
actual = pre_sharded.transformer(**inputs)
|
||||
assert torch.isfinite(actual).all()
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("edge", [False, True], ids=["cosmos3", "cosmos3_edge"])
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.accelerator.device_count() < 2 or not dist.is_nccl_available(),
|
||||
reason="requires two CUDA GPUs and NCCL",
|
||||
)
|
||||
def test_cosmos3_pre_sharded_matches_full_hsdp(tmp_path, edge):
|
||||
torch.multiprocessing.spawn(
|
||||
_load_cosmos_worker,
|
||||
args=(f"file://{tmp_path / 'rendezvous'}", str(tmp_path), edge),
|
||||
nprocs=2,
|
||||
join=True,
|
||||
)
|
||||
@@ -3,11 +3,13 @@
|
||||
"""Unit tests for HSDP (Hybrid Sharded Data Parallel) configuration and utilities."""
|
||||
|
||||
import gc
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
from torch.distributed.fsdp import fully_shard
|
||||
from torch.distributed.tensor import DeviceMesh, DTensor
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
|
||||
@@ -18,6 +20,7 @@ from vllm_omni.diffusion.distributed.hsdp import (
|
||||
_unshardable_parameters,
|
||||
shard_model,
|
||||
)
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
|
||||
pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.cpu, pytest.mark.core_model]
|
||||
|
||||
@@ -338,3 +341,92 @@ class TestHSDPShardConditions:
|
||||
matched.append(name)
|
||||
assert "blocks.0" in matched
|
||||
assert "blocks.1" in matched
|
||||
|
||||
|
||||
def test_pre_sharded_to_empty_materializes_fsdp_internal_storage(cpu_process_group):
|
||||
with torch.device("meta"):
|
||||
model = nn.Module()
|
||||
model.transformer = nn.Sequential(nn.Linear(2, 2, bias=False))
|
||||
model.transformer.register_buffer("scratch", torch.tensor([3.0]), persistent=False)
|
||||
shard_model(
|
||||
model.transformer,
|
||||
mesh=DeviceMesh("cpu", [0]),
|
||||
hsdp_shard_conditions=[lambda name, _module: name == "0"],
|
||||
)
|
||||
|
||||
load_plan = SimpleNamespace(
|
||||
roots=[SimpleNamespace(name="transformer", module=model.transformer)],
|
||||
groups=[SimpleNamespace(module=model.transformer[0])],
|
||||
binding_names=frozenset({"transformer.0.weight"}),
|
||||
)
|
||||
sharded_weight = model.transformer[0].weight
|
||||
|
||||
DiffusersPipelineLoader._materialize_pre_sharded_hsdp_state(model, load_plan, torch.device("cpu"))
|
||||
|
||||
fsdp_param = fully_shard.state(model.transformer[0])._fsdp_param_group.fsdp_params[0]
|
||||
assert model.transformer[0].weight is sharded_weight
|
||||
assert model.transformer[0].weight.to_local().device.type == "cpu"
|
||||
assert fsdp_param._sharded_param_data.device.type == "cpu"
|
||||
assert (
|
||||
fsdp_param._sharded_param_data.untyped_storage().data_ptr()
|
||||
== model.transformer[0].weight.to_local().untyped_storage().data_ptr()
|
||||
)
|
||||
assert torch.equal(model.transformer.scratch, torch.tensor([3.0]))
|
||||
|
||||
|
||||
def test_pre_sharded_load_restores_checkpoint_values(cpu_process_group, tmp_path, monkeypatch):
|
||||
from safetensors.torch import save_file
|
||||
from vllm.config.load import LoadConfig
|
||||
|
||||
from vllm_omni.diffusion.distributed import hsdp as hsdp_module
|
||||
from vllm_omni.diffusion.models.utils import release_module_parameters_to_meta
|
||||
|
||||
class Transformer(nn.Module):
|
||||
_hsdp_shard_conditions = [lambda name, module: name == "block"]
|
||||
_hsdp_ignored_modules = ["time_embedder"]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.block = nn.Linear(3, 4, bias=False)
|
||||
self.time_embedder = nn.Linear(2, 2, bias=False, dtype=torch.bfloat16)
|
||||
self.root_weight = nn.Parameter(torch.zeros(2))
|
||||
self.register_buffer("scratch", torch.tensor([7.0]), persistent=False)
|
||||
release_module_parameters_to_meta(self.block)
|
||||
|
||||
def post_load_weights(self):
|
||||
self.time_embedder.to(torch.float32)
|
||||
|
||||
model = nn.Module()
|
||||
model.transformer = Transformer()
|
||||
model.vae = nn.Linear(2, 2)
|
||||
tensors = {
|
||||
"block.weight": torch.arange(12, dtype=torch.float32).reshape(4, 3),
|
||||
"root_weight": torch.tensor([2.0, 3.0]),
|
||||
"time_embedder.weight": torch.arange(4, dtype=torch.bfloat16).reshape(2, 2),
|
||||
}
|
||||
save_file(tensors, str(tmp_path / "model.safetensors"))
|
||||
model.weights_sources = [DiffusersPipelineLoader.ComponentSource(str(tmp_path), None, None, "transformer.", False)]
|
||||
config = SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
quantization_config=None,
|
||||
lora_path=None,
|
||||
hsdp_weight_load_strategy="pre_sharded",
|
||||
num_weight_load_threads=1,
|
||||
parallel_config=SimpleNamespace(use_hsdp=True, hsdp_replicate_size=1, hsdp_shard_size=1),
|
||||
)
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), config)
|
||||
monkeypatch.setattr(loader, "_init_from_load_format", lambda *args, **kwargs: model)
|
||||
monkeypatch.setattr(hsdp_module, "get_world_group", lambda: SimpleNamespace(world_size=1, rank_in_group=0))
|
||||
monkeypatch.setattr(hsdp_module, "current_omni_platform", SimpleNamespace(device_type="cpu"))
|
||||
loaded = loader._load_model_with_hsdp(torch.device("cpu"))
|
||||
assert loaded is model
|
||||
for name, expected in tensors.items():
|
||||
actual = dict(model.transformer.named_parameters())[name]
|
||||
if name.startswith("time_embedder."):
|
||||
assert not isinstance(actual, DTensor)
|
||||
assert actual.dtype == torch.float32
|
||||
torch.testing.assert_close(actual, expected.float())
|
||||
else:
|
||||
assert isinstance(actual, DTensor)
|
||||
torch.testing.assert_close(actual.to_local(), expected)
|
||||
torch.testing.assert_close(model.transformer.scratch, torch.tensor([7.0]))
|
||||
@@ -27,6 +27,7 @@ from vllm_omni.diffusion.model_loader.host_weight_plan import (
|
||||
HostWeightPlan,
|
||||
HostWeightPlanResult,
|
||||
TensorBinding,
|
||||
build_checkpoint_binding_plan,
|
||||
)
|
||||
from vllm_omni.diffusion.model_loader.host_weights import source_identity as source_identity_module
|
||||
from vllm_omni.diffusion.models.helios import HeliosPipeline
|
||||
@@ -2089,3 +2090,206 @@ def test_hsdp_broadcast_weight_load_online_quant_multiprocess():
|
||||
)
|
||||
assert os.path.exists(os.path.join(temp_dir, "rank_0_success.flag"))
|
||||
assert os.path.exists(os.path.join(temp_dir, "rank_1_success.flag"))
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_rejects_quantization_before_loading():
|
||||
od_config = SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
hsdp_weight_load_strategy="pre_sharded",
|
||||
lora_path=None,
|
||||
quantization_config=object(),
|
||||
parallel_config=SimpleNamespace(
|
||||
use_hsdp=True,
|
||||
hsdp_replicate_size=1,
|
||||
hsdp_shard_size=2,
|
||||
),
|
||||
)
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
|
||||
with pytest.raises(ValueError, match="does not support quantization"):
|
||||
loader._load_model_with_hsdp(torch.device("cpu"))
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_strategy_dispatches_without_using_full_loader(mocker):
|
||||
import vllm_omni.diffusion.model_loader.diffusers_loader as loader_mod
|
||||
from vllm_omni.diffusion.offloader.module_collector import PipelineModules
|
||||
|
||||
od_config = SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
hsdp_weight_load_strategy="pre_sharded",
|
||||
lora_path=None,
|
||||
quantization_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
use_hsdp=True,
|
||||
hsdp_replicate_size=1,
|
||||
hsdp_shard_size=2,
|
||||
),
|
||||
)
|
||||
loader = DiffusersPipelineLoader(LoadConfig(), od_config)
|
||||
model = nn.Module()
|
||||
model.transformer = nn.Linear(2, 2, bias=False)
|
||||
discovered = PipelineModules(
|
||||
dits=[model.transformer],
|
||||
dit_names=["transformer"],
|
||||
vaes=[],
|
||||
encoders=[],
|
||||
encoder_names=[],
|
||||
resident_modules=[],
|
||||
resident_names=[],
|
||||
)
|
||||
loader._init_from_load_format = mocker.Mock(return_value=model) # type: ignore[method-assign]
|
||||
loader.load_weights = mocker.Mock(side_effect=AssertionError("full loader must not run")) # type: ignore[method-assign]
|
||||
mocker.patch.object(loader_mod.ModuleDiscovery, "discover", return_value=discovered)
|
||||
pre_sharded_load = mocker.patch.object(
|
||||
loader,
|
||||
"_load_model_with_pre_sharded_hsdp",
|
||||
return_value=model,
|
||||
)
|
||||
|
||||
assert loader._load_model_with_hsdp(torch.device("cpu")) is model
|
||||
pre_sharded_load.assert_called_once()
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_rejects_tensor_transform():
|
||||
def transform(tensor):
|
||||
return tensor.transpose(-2, -1)
|
||||
|
||||
load_plan = SimpleNamespace(
|
||||
bindings={
|
||||
"transformer.weight": TensorBinding(
|
||||
checkpoint_key="weight",
|
||||
file_path="model.safetensors",
|
||||
transform=transform,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="tensor transforms are unsupported"):
|
||||
DiffusersPipelineLoader._validate_pre_sharded_hsdp_bindings(load_plan)
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_accepts_direct_safetensors_binding():
|
||||
load_plan = SimpleNamespace(
|
||||
bindings={
|
||||
"transformer.weight": TensorBinding(
|
||||
checkpoint_key="weight",
|
||||
file_path="model.safetensors",
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
DiffusersPipelineLoader._validate_pre_sharded_hsdp_bindings(load_plan)
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_uses_fsdp_destination_directly():
|
||||
target = torch.empty(8, 4, 3)
|
||||
load_plan = SimpleNamespace(
|
||||
bindings={
|
||||
"transformer.weight": TensorBinding(
|
||||
checkpoint_key="source_weight",
|
||||
file_path="transformer/model.safetensors",
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
checkpoint_states, local_bytes = DiffusersPipelineLoader._prepare_pre_sharded_checkpoint_states(
|
||||
load_plan,
|
||||
{"transformer.weight": target},
|
||||
)
|
||||
checkpoint_target = checkpoint_states[Path("transformer")]["source_weight"]
|
||||
|
||||
assert checkpoint_target is target
|
||||
assert checkpoint_target.shape == (8, 4, 3)
|
||||
assert checkpoint_target.device.type == "cpu"
|
||||
assert checkpoint_target.untyped_storage().data_ptr() == target.untyped_storage().data_ptr()
|
||||
assert local_bytes == target.numel() * target.element_size()
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_checkpoint_writes_runtime_storage_directly():
|
||||
runtime_target = torch.empty(2, 3)
|
||||
load_plan = SimpleNamespace(
|
||||
bindings={
|
||||
"transformer.weight": TensorBinding(
|
||||
checkpoint_key="source_weight",
|
||||
file_path="transformer/model.safetensors",
|
||||
)
|
||||
}
|
||||
)
|
||||
checkpoint_tensor = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
checkpoint_states, _ = DiffusersPipelineLoader._prepare_pre_sharded_checkpoint_states(
|
||||
load_plan,
|
||||
{"transformer.weight": runtime_target},
|
||||
)
|
||||
checkpoint_states[Path("transformer")]["source_weight"].copy_(checkpoint_tensor)
|
||||
|
||||
assert torch.equal(runtime_target, checkpoint_tensor)
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_rejects_duplicate_checkpoint_destinations():
|
||||
binding = TensorBinding(checkpoint_key="weight", file_path="transformer/model.safetensors")
|
||||
load_plan = SimpleNamespace(
|
||||
bindings={
|
||||
"transformer.first": binding,
|
||||
"transformer.second": binding,
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="multiple runtime tensors"):
|
||||
DiffusersPipelineLoader._prepare_pre_sharded_checkpoint_states(
|
||||
load_plan,
|
||||
{"transformer.first": torch.empty(2, 3), "transformer.second": torch.empty(2, 3)},
|
||||
)
|
||||
|
||||
|
||||
def test_pre_sharded_hsdp_managed_module_traversal_is_child_first_and_skips_nested_roots():
|
||||
root = nn.Module()
|
||||
root.left = nn.Sequential(nn.Linear(2, 2), nn.ReLU())
|
||||
root.right = nn.Sequential(nn.Linear(2, 2), nn.ReLU())
|
||||
|
||||
names = {module: name or "root" for name, module in root.named_modules()}
|
||||
modules = DiffusersPipelineLoader._hsdp_managed_modules_post_order(
|
||||
root,
|
||||
nested_hsdp_roots={root.left},
|
||||
)
|
||||
|
||||
assert [names[module] for module in modules] == ["right.0", "right.1", "right", "root"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("checkpoint_key", ["renamed", "unexpected"])
|
||||
def test_hsdp_checkpoint_plan_honors_remap_and_rejects_missing(tmp_path, checkpoint_key, monkeypatch):
|
||||
import vllm_omni.diffusion.model_loader.host_weight_plan as plan_mod
|
||||
|
||||
monkeypatch.setattr(plan_mod, "get_direct_mmap_adapter", lambda _model: None)
|
||||
|
||||
class Pipeline(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.transformer = nn.Linear(2, 2, bias=False)
|
||||
|
||||
@staticmethod
|
||||
def remap_checkpoint_key(name):
|
||||
return "transformer.weight" if name == "transformer.renamed" else name
|
||||
|
||||
checkpoint = tmp_path / "model.safetensors"
|
||||
save_file({checkpoint_key: torch.ones(2, 2)}, str(checkpoint))
|
||||
source = SimpleNamespace(
|
||||
model_or_path=str(tmp_path),
|
||||
subfolder=None,
|
||||
revision=None,
|
||||
prefix="transformer.",
|
||||
)
|
||||
model = Pipeline()
|
||||
result = build_checkpoint_binding_plan(
|
||||
model,
|
||||
dit_modules=(("transformer", model.transformer),),
|
||||
sources=(source,),
|
||||
model_path=None,
|
||||
tensor_parallel_size=1,
|
||||
online_quantization=False,
|
||||
)
|
||||
|
||||
if checkpoint_key == "renamed":
|
||||
assert result.plan is not None
|
||||
assert result.plan.bindings["transformer.weight"].checkpoint_key == "renamed"
|
||||
else:
|
||||
assert result.plan is None
|
||||
assert "no checkpoint binding" in result.fallback_reason
|
||||
@@ -868,6 +868,93 @@ def test_qwen_gen_mlp_remains_gated() -> None:
|
||||
assert hasattr(model.gen_layers[0].mlp, "gate_proj")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("edge", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"use_hsdp,strategy", [(False, "full"), (False, "pre_sharded"), (True, "full"), (True, "pre_sharded")]
|
||||
)
|
||||
def test_cosmos3_releases_blocks_only_for_pre_sharded_hsdp(edge, use_hsdp, strategy, monkeypatch):
|
||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3, transformer_cosmos3_edge
|
||||
from vllm_omni.diffusion.models.utils import release_module_parameters_to_meta
|
||||
|
||||
released = []
|
||||
|
||||
def release_and_check(block):
|
||||
parameters = dict(block.named_parameters())
|
||||
metadata = {name: dict(parameter.__dict__) for name, parameter in parameters.items()}
|
||||
release_module_parameters_to_meta(block)
|
||||
for name, parameter in block.named_parameters():
|
||||
assert parameter is parameters[name]
|
||||
assert parameter.device.type == "meta"
|
||||
assert parameter.__dict__ == metadata[name]
|
||||
released.append(block)
|
||||
|
||||
monkeypatch.setattr(transformer_cosmos3, "release_module_parameters_to_meta", release_and_check)
|
||||
monkeypatch.setattr(transformer_cosmos3_edge, "release_module_parameters_to_meta", release_and_check)
|
||||
model_cls = (
|
||||
transformer_cosmos3_edge.Cosmos3EdgeVFMTransformer if edge else transformer_cosmos3.Cosmos3VFMTransformer
|
||||
)
|
||||
config = _tiny_cosmos3_edge_config if edge else _tiny_cosmos3_config
|
||||
model = model_cls(
|
||||
SimpleNamespace(
|
||||
tf_model_config=config(num_hidden_layers=1),
|
||||
dtype=torch.float32,
|
||||
parallel_config=SimpleNamespace(use_hsdp=use_hsdp),
|
||||
hsdp_weight_load_strategy=strategy,
|
||||
)
|
||||
)
|
||||
blocks = [*model.language_model.layers, *model.gen_layers]
|
||||
expected_meta = use_hsdp and strategy == "pre_sharded"
|
||||
assert len(released) == (len(blocks) if expected_meta else 0)
|
||||
assert all((p.device.type == "meta") == expected_meta for block in blocks for p in block.parameters())
|
||||
assert model.time_embedder.linear_1.weight.device.type != "meta"
|
||||
assert all(buffer.device.type != "meta" for buffer in model.buffers())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("edge", [False, True])
|
||||
def test_cosmos3_pre_sharded_meta_processing_accepts_real_unquantized_blocks(edge, monkeypatch):
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.cosmos3 import Cosmos3EdgeVFMTransformer, Cosmos3VFMTransformer
|
||||
|
||||
model_cls = Cosmos3EdgeVFMTransformer if edge else Cosmos3VFMTransformer
|
||||
config = _tiny_cosmos3_edge_config if edge else _tiny_cosmos3_config
|
||||
model = model_cls(
|
||||
SimpleNamespace(
|
||||
tf_model_config=config(num_hidden_layers=1),
|
||||
dtype=torch.float32,
|
||||
parallel_config=SimpleNamespace(use_hsdp=True),
|
||||
hsdp_weight_load_strategy="pre_sharded",
|
||||
)
|
||||
)
|
||||
groups = [SimpleNamespace(module=block) for block in (*model.language_model.layers, *model.gen_layers)]
|
||||
plan = SimpleNamespace(roots=[SimpleNamespace(module=model)], groups=groups)
|
||||
processed = []
|
||||
# Exercise model selection with real vLLM linears, without invoking the
|
||||
# CPU-specific weight packing hook on meta tensors in CPU-only CI.
|
||||
monkeypatch.setattr(
|
||||
UnquantizedLinearMethod, "process_weights_after_loading", lambda self, layer: processed.append(layer)
|
||||
)
|
||||
DiffusersPipelineLoader._process_pre_sharded_hsdp_groups_on_meta(plan)
|
||||
expected = [module for group in groups for module in group.module.modules() if hasattr(module, "quant_method")]
|
||||
assert expected
|
||||
assert processed == expected
|
||||
|
||||
model._hsdp_pre_sharded_meta_post_load = False
|
||||
with pytest.raises(ValueError, match="has not declared"):
|
||||
DiffusersPipelineLoader._process_pre_sharded_hsdp_groups_on_meta(plan)
|
||||
model._hsdp_pre_sharded_meta_post_load = True
|
||||
|
||||
class UnsupportedMethod(UnquantizedLinearMethod):
|
||||
pass
|
||||
|
||||
expected[-1].quant_method = UnsupportedMethod()
|
||||
processed.clear()
|
||||
with pytest.raises(ValueError, match="UnsupportedMethod"):
|
||||
DiffusersPipelineLoader._process_pre_sharded_hsdp_groups_on_meta(plan)
|
||||
assert not processed
|
||||
|
||||
|
||||
def test_model_cpu_offload_swaps_back_to_generator(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from vllm_omni.diffusion.models.cosmos3 import transformer_cosmos3
|
||||
|
||||
|
||||
Reference in new issue
Block a user