Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d989aa6fe2 | ||
|
|
039808e0d9 |
No files matched your search
@@ -67,6 +67,26 @@ views and mapped ranges. A separate transport still decides whether to use
|
|||||||
registered mmap, private pinned staging, synchronous copies, or asynchronous
|
registered mmap, private pinned staging, synchronous copies, or asynchronous
|
||||||
H2D transfer.
|
H2D transfer.
|
||||||
|
|
||||||
|
## Payload validation at startup
|
||||||
|
|
||||||
|
Eligible diffusion loaders accept `--host-weight-runtime-validation` (or the
|
||||||
|
`host_weight_runtime_validation` offline/stage configuration field):
|
||||||
|
|
||||||
|
- `manifest_and_metadata` is the existing default. It validates identity,
|
||||||
|
metadata, tensor structure, and sizes, but does not detect payload-only
|
||||||
|
corruption.
|
||||||
|
- `full_checksum` reads and hashes every payload on warm acquisition before
|
||||||
|
restoring weights. This adds startup I/O/CPU work proportional to artifact
|
||||||
|
bytes; it is not a per-inference check.
|
||||||
|
|
||||||
|
For example, add `--host-weight-runtime-validation full_checksum` to an enabled
|
||||||
|
HWR deployment. Preferred mode falls back to canonical loading and can publish
|
||||||
|
a replacement after detecting corruption; required mode fails startup. Readers
|
||||||
|
can use different validation levels in the same domain without changing artifact
|
||||||
|
identity. Neither level authenticates an untrusted writer or prevents mutation
|
||||||
|
after validation. Checksumming does not imply automatic recovery of a running
|
||||||
|
service.
|
||||||
|
|
||||||
## Resolution behavior
|
## Resolution behavior
|
||||||
|
|
||||||
The loader resolves the immutable canonical source and computes the exact
|
The loader resolves the immutable canonical source and computes the exact
|
||||||
|
|||||||
@@ -193,6 +193,37 @@ def test_resolve_execution_mode_rejects_unknown_execution_type():
|
|||||||
omni_config_module._resolve_execution_mode("unknown_execution_type")
|
omni_config_module._resolve_execution_mode("unknown_execution_type")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("validation", ["manifest_and_metadata", "full_checksum"])
|
||||||
|
def test_hwr_validation_survives_config_projection(validation: str):
|
||||||
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
|
|
||||||
|
override = StageDeployConfig(stage_id=0, host_weight_runtime_validation=validation)
|
||||||
|
config = _from_pipeline_key(
|
||||||
|
"dreamzero",
|
||||||
|
deploy_config_path="dreamzero_tp1_cfg2",
|
||||||
|
cli_overrides={"host_weight_runtime_validation": validation},
|
||||||
|
)
|
||||||
|
stage = config.stage_by_id(0)
|
||||||
|
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||||
|
assert stage.diffusion_config.host_weight_runtime_validation == validation
|
||||||
|
projection = omni_config_module._DiffusionConfigProjection.from_kwargs(
|
||||||
|
host_weight_runtime_validation=override.host_weight_runtime_validation
|
||||||
|
)
|
||||||
|
assert projection.host_weight_runtime_validation == validation
|
||||||
|
assert OmniDiffusionConfig(host_weight_runtime_validation=validation).host_weight_runtime_validation == validation
|
||||||
|
assert omni_config_module._DiffusionConfigProjection().host_weight_runtime_validation == "manifest_and_metadata"
|
||||||
|
assert OmniDiffusionConfig().host_weight_runtime_validation == "manifest_and_metadata"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("validation", ["fs_verity", "unknown", None])
|
||||||
|
def test_hwr_validation_rejects_unsupported_values(validation):
|
||||||
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
|
|
||||||
|
for config_class in (OmniDiffusionConfig, omni_config_module._DiffusionConfigProjection):
|
||||||
|
with pytest.raises(ValueError, match="host_weight_runtime_validation"):
|
||||||
|
config_class(host_weight_runtime_validation=validation)
|
||||||
|
|
||||||
|
|
||||||
def test_from_pipeline_config_preserves_current_pipeline_config_object():
|
def test_from_pipeline_config_preserves_current_pipeline_config_object():
|
||||||
omni_config = _from_pipeline_key("minicpmo_4_5")
|
omni_config = _from_pipeline_key("minicpmo_4_5")
|
||||||
pipeline = _resolve_pipeline_or_skip("minicpmo_4_5")
|
pipeline = _resolve_pipeline_or_skip("minicpmo_4_5")
|
||||||
|
|||||||
@@ -27,6 +27,202 @@ from vllm_omni.platforms import current_omni_platform
|
|||||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_uaa_equal_rank_contract_skips_length_gather(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
from vllm_omni.diffusion.attention.parallel import ulysses
|
||||||
|
|
||||||
|
class FakeGroup:
|
||||||
|
ulysses_group = object()
|
||||||
|
ulysses_world_size = 2
|
||||||
|
ulysses_rank = 0
|
||||||
|
ring_world_size = 1
|
||||||
|
|
||||||
|
observed = []
|
||||||
|
|
||||||
|
def fake_all_to_all(pg, tensor, **kwargs):
|
||||||
|
observed.append((kwargs["seq_lens"], kwargs["padded_head_cnt"]))
|
||||||
|
return tensor, tensor.shape[2]
|
||||||
|
|
||||||
|
monkeypatch.setattr(ulysses, "_ulysses_all_to_all_any_qkv", fake_all_to_all)
|
||||||
|
monkeypatch.setattr(ulysses, "get_ulysses_mode", lambda **kwargs: "advanced_uaa")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ulysses,
|
||||||
|
"_all_gather_int",
|
||||||
|
lambda *args, **kwargs: pytest.fail("length gather must be skipped"),
|
||||||
|
)
|
||||||
|
strategy = ulysses.UlyssesParallelAttention(
|
||||||
|
FakeGroup(),
|
||||||
|
scatter_idx=2,
|
||||||
|
gather_idx=1,
|
||||||
|
use_sync=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with set_forward_context():
|
||||||
|
get_forward_context()._sp_equal_pad_stack.append(True)
|
||||||
|
strategy.pre_attention(
|
||||||
|
torch.zeros(1, 3, 28, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert observed == [([3, 3], 32), ([3, 3], 8), ([3, 3], 8)]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_uaa_hybrid_equal_rank_contract_skips_ulysses_gather_keeps_ring_check(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
"""Contract + ring: the fast path skips only the Ulysses gather.
|
||||||
|
|
||||||
|
The ring equality check is a separate collective on ring_group and must
|
||||||
|
still run, receiving s_global derived from the contract.
|
||||||
|
"""
|
||||||
|
from vllm_omni.diffusion.attention.parallel import ulysses
|
||||||
|
|
||||||
|
class FakeGroup:
|
||||||
|
ulysses_group = object()
|
||||||
|
ring_group = object()
|
||||||
|
ulysses_world_size = 2
|
||||||
|
ulysses_rank = 0
|
||||||
|
ring_world_size = 2
|
||||||
|
|
||||||
|
sp_group = FakeGroup()
|
||||||
|
observed = []
|
||||||
|
ring_checks = []
|
||||||
|
|
||||||
|
def fake_all_to_all(pg, tensor, **kwargs):
|
||||||
|
observed.append((kwargs["seq_lens"], kwargs["padded_head_cnt"]))
|
||||||
|
return tensor, tensor.shape[2]
|
||||||
|
|
||||||
|
def fake_gather_int(pg, value, *, device):
|
||||||
|
if pg is sp_group.ulysses_group:
|
||||||
|
pytest.fail("Ulysses length gather must be skipped under the contract")
|
||||||
|
assert pg is sp_group.ring_group
|
||||||
|
ring_checks.append(int(value))
|
||||||
|
return [int(value), int(value)]
|
||||||
|
|
||||||
|
monkeypatch.setattr(ulysses, "_ulysses_all_to_all_any_qkv", fake_all_to_all)
|
||||||
|
monkeypatch.setattr(ulysses, "_all_gather_int", fake_gather_int)
|
||||||
|
monkeypatch.setattr(ulysses, "get_ulysses_mode", lambda **kwargs: "advanced_uaa")
|
||||||
|
strategy = ulysses.UlyssesParallelAttention(
|
||||||
|
sp_group,
|
||||||
|
scatter_idx=2,
|
||||||
|
gather_idx=1,
|
||||||
|
use_sync=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with set_forward_context():
|
||||||
|
get_forward_context()._sp_equal_pad_stack.append(True)
|
||||||
|
strategy.pre_attention(
|
||||||
|
torch.zeros(1, 3, 28, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Ring check ran: s_global = ulysses_world_size * local = 2 * 3.
|
||||||
|
assert ring_checks == [6]
|
||||||
|
assert observed == [([3, 3], 32), ([3, 3], 8), ([3, 3], 8)]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_uaa_without_contract_keeps_dynamic_length_gather(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
"""No auto_pad boundary means shards may differ, so the gather must stay."""
|
||||||
|
from vllm_omni.diffusion.attention.parallel import ulysses
|
||||||
|
|
||||||
|
class FakeGroup:
|
||||||
|
ulysses_group = object()
|
||||||
|
ulysses_world_size = 2
|
||||||
|
ulysses_rank = 0
|
||||||
|
ring_world_size = 1
|
||||||
|
|
||||||
|
observed = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ulysses,
|
||||||
|
"_all_gather_int",
|
||||||
|
lambda *args, **kwargs: [3, 4],
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(ulysses, "get_ulysses_mode", lambda **kwargs: "advanced_uaa")
|
||||||
|
|
||||||
|
def fake_all_to_all(pg, tensor, **kwargs):
|
||||||
|
observed.append(kwargs["seq_lens"])
|
||||||
|
return tensor, tensor.shape[2]
|
||||||
|
|
||||||
|
monkeypatch.setattr(ulysses, "_ulysses_all_to_all_any_qkv", fake_all_to_all)
|
||||||
|
strategy = ulysses.UlyssesParallelAttention(
|
||||||
|
FakeGroup(),
|
||||||
|
scatter_idx=2,
|
||||||
|
gather_idx=1,
|
||||||
|
use_sync=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with set_forward_context():
|
||||||
|
strategy.pre_attention(
|
||||||
|
torch.zeros(1, 3, 28, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert observed == [[3, 4], [3, 4], [3, 4]]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.core_model
|
||||||
|
@pytest.mark.cpu
|
||||||
|
def test_uaa_mixed_boundaries_keep_the_dynamic_length_gather(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
"""One manual boundary inside an auto_pad region voids the contract."""
|
||||||
|
from vllm_omni.diffusion.attention.parallel import ulysses
|
||||||
|
|
||||||
|
class FakeGroup:
|
||||||
|
ulysses_group = object()
|
||||||
|
ulysses_world_size = 2
|
||||||
|
ulysses_rank = 0
|
||||||
|
ring_world_size = 1
|
||||||
|
|
||||||
|
monkeypatch.setattr(ulysses, "get_ulysses_mode", lambda **kwargs: "advanced_uaa")
|
||||||
|
gathered = []
|
||||||
|
|
||||||
|
def fake_gather(*args, **kwargs):
|
||||||
|
gathered.append(args)
|
||||||
|
return [3, 4]
|
||||||
|
|
||||||
|
monkeypatch.setattr(ulysses, "_all_gather_int", fake_gather)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ulysses,
|
||||||
|
"_ulysses_all_to_all_any_qkv",
|
||||||
|
lambda pg, tensor, **kwargs: (tensor, tensor.shape[2]),
|
||||||
|
)
|
||||||
|
strategy = ulysses.UlyssesParallelAttention(
|
||||||
|
FakeGroup(),
|
||||||
|
scatter_idx=2,
|
||||||
|
gather_idx=1,
|
||||||
|
use_sync=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with set_forward_context():
|
||||||
|
ctx = get_forward_context()
|
||||||
|
ctx._sp_equal_pad_stack.extend([True, False])
|
||||||
|
assert not ctx.sp_rank_local_seq_lens_equal
|
||||||
|
strategy.pre_attention(
|
||||||
|
torch.zeros(1, 3, 28, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
torch.zeros(1, 3, 7, 4),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(gathered) == 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.core_model
|
@pytest.mark.core_model
|
||||||
@pytest.mark.cpu
|
@pytest.mark.cpu
|
||||||
def test_uaa_gqa_head_padding_preserves_the_query_to_kv_ratio(
|
def test_uaa_gqa_head_padding_preserves_the_query_to_kv_ratio(
|
||||||
@@ -634,6 +830,187 @@ def test_ulysses_uaa_matches_baseline(
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _run_contract_skip_case(
|
||||||
|
local_rank: int,
|
||||||
|
world_size: int,
|
||||||
|
init_method: str,
|
||||||
|
input_file: str,
|
||||||
|
num_heads: int,
|
||||||
|
head_size: int,
|
||||||
|
num_kv_heads: int | None,
|
||||||
|
) -> None:
|
||||||
|
"""Per-rank worker: slow path vs contract fast path on real rank pairs.
|
||||||
|
|
||||||
|
Phase 1 leaves the equal-pad stack empty so the Ulysses length all-gather
|
||||||
|
really runs (counted); phase 2 pushes the split hook's auto_pad marker and
|
||||||
|
replaces _all_gather_int with a raising guard. With ring_degree=1 any call
|
||||||
|
to it is the Ulysses length gather, so the guard proves the skip, and the
|
||||||
|
per-rank output comparison proves the fast path is numerically equivalent
|
||||||
|
to the slow path it replaces.
|
||||||
|
"""
|
||||||
|
from vllm_omni.diffusion.attention.parallel import ulysses as ulysses_mod
|
||||||
|
|
||||||
|
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
|
||||||
|
current_omni_platform.set_device(device)
|
||||||
|
|
||||||
|
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
|
||||||
|
|
||||||
|
init_distributed_environment(world_size=world_size, rank=local_rank, distributed_init_method=init_method)
|
||||||
|
initialize_model_parallel(
|
||||||
|
data_parallel_size=1,
|
||||||
|
cfg_parallel_size=1,
|
||||||
|
sequence_parallel_size=world_size,
|
||||||
|
ulysses_degree=world_size,
|
||||||
|
ring_degree=1,
|
||||||
|
tensor_parallel_size=1,
|
||||||
|
pipeline_parallel_size=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
parallel_config = DiffusionParallelConfig(
|
||||||
|
pipeline_parallel_size=1,
|
||||||
|
data_parallel_size=1,
|
||||||
|
tensor_parallel_size=1,
|
||||||
|
sequence_parallel_size=world_size,
|
||||||
|
ulysses_degree=world_size,
|
||||||
|
ring_degree=1,
|
||||||
|
cfg_parallel_size=1,
|
||||||
|
ulysses_mode="advanced_uaa",
|
||||||
|
)
|
||||||
|
od_config = OmniDiffusionConfig(model="test", dtype=torch.float32, parallel_config=parallel_config)
|
||||||
|
|
||||||
|
with set_forward_context(omni_diffusion_config=od_config), set_current_diffusion_config(od_config):
|
||||||
|
attn = Attention(
|
||||||
|
num_heads=num_heads,
|
||||||
|
head_size=head_size,
|
||||||
|
causal=False,
|
||||||
|
softmax_scale=1.0 / (head_size**0.5),
|
||||||
|
num_kv_heads=num_kv_heads,
|
||||||
|
).to(device=device, dtype=torch.float32)
|
||||||
|
|
||||||
|
with np.load(input_file, allow_pickle=False) as payload:
|
||||||
|
q_full = torch.from_numpy(payload["q"]).to(device=device)
|
||||||
|
k_full = torch.from_numpy(payload["k"]).to(device=device)
|
||||||
|
v_full = torch.from_numpy(payload["v"]).to(device=device)
|
||||||
|
|
||||||
|
def make_shards() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||||
|
# Fresh contiguous shards for each phase; seq_len is divisible by
|
||||||
|
# world_size, so every rank gets the same local length.
|
||||||
|
return (
|
||||||
|
torch.tensor_split(q_full, world_size, dim=1)[local_rank].contiguous(),
|
||||||
|
torch.tensor_split(k_full, world_size, dim=1)[local_rank].contiguous(),
|
||||||
|
torch.tensor_split(v_full, world_size, dim=1)[local_rank].contiguous(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# The Attention layer only enables SP communication when
|
||||||
|
# ForwardContext.sp_active is True; in production the split hook sets
|
||||||
|
# this (and pushes the auto_pad equal-length marker onto
|
||||||
|
# _sp_equal_pad_stack). Here we drive it directly.
|
||||||
|
ctx = get_forward_context()
|
||||||
|
ctx._sp_shard_depth = 1
|
||||||
|
|
||||||
|
def make_sdp_ctx() -> object:
|
||||||
|
# Fresh instance per phase: the @contextmanager instance releases
|
||||||
|
# its state on first exit and cannot be re-entered.
|
||||||
|
return torch.backends.cuda.sdp_kernel(enable_flash=False, enable_mem_efficient=False, enable_math=True)
|
||||||
|
|
||||||
|
# Phase 1: contract absent -> the length all-gather must really run.
|
||||||
|
real_gather = ulysses_mod._all_gather_int
|
||||||
|
gather_calls: list[object] = []
|
||||||
|
|
||||||
|
def counting_gather(pg: object, value: int, *, device: torch.device) -> list[int]:
|
||||||
|
gather_calls.append(pg)
|
||||||
|
return real_gather(pg, value, device=device)
|
||||||
|
|
||||||
|
ulysses_mod._all_gather_int = counting_gather
|
||||||
|
try:
|
||||||
|
q, k, v = make_shards()
|
||||||
|
with make_sdp_ctx():
|
||||||
|
out_slow = attn(q, k, v, attn_metadata=None).contiguous()
|
||||||
|
finally:
|
||||||
|
ulysses_mod._all_gather_int = real_gather
|
||||||
|
assert len(gather_calls) >= 1, "slow path must call the Ulysses length all-gather"
|
||||||
|
|
||||||
|
# Phase 2: replicate the split hook's auto_pad marker so the fast
|
||||||
|
# path fires; any _all_gather_int call in this phase is the Ulysses
|
||||||
|
# length gather and must not happen.
|
||||||
|
ctx._sp_equal_pad_stack.append(True)
|
||||||
|
|
||||||
|
def fail_on_gather(pg: object, value: int, *, device: torch.device) -> list[int]:
|
||||||
|
raise AssertionError("the Ulysses length all-gather must be skipped under the equal-pad contract")
|
||||||
|
|
||||||
|
ulysses_mod._all_gather_int = fail_on_gather
|
||||||
|
try:
|
||||||
|
q, k, v = make_shards()
|
||||||
|
with make_sdp_ctx():
|
||||||
|
out_fast = attn(q, k, v, attn_metadata=None).contiguous()
|
||||||
|
finally:
|
||||||
|
ulysses_mod._all_gather_int = real_gather
|
||||||
|
ctx._sp_equal_pad_stack.pop()
|
||||||
|
|
||||||
|
torch.testing.assert_close(out_fast, out_slow, atol=1e-5, rtol=1e-5)
|
||||||
|
|
||||||
|
destroy_distributed_env()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"num_heads,num_kv_heads",
|
||||||
|
[
|
||||||
|
(3, None), # MHA: heads padded 3 -> 4 by ulysses_degree=2
|
||||||
|
(28, 7), # GQA: derived-Q padding 28/7 -> 32/8, the main case for the skip
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_ulysses_uaa_contract_skips_length_gather_matches_slow_path(num_heads: int, num_kv_heads: int | None) -> None:
|
||||||
|
"""Real 2-rank parity: contract fast path == slow path, gather skipped.
|
||||||
|
|
||||||
|
The CPU-only contract tests replace the all-to-all with a fake; this test
|
||||||
|
runs the real strategy on two ranks. Each rank first computes the slow
|
||||||
|
path (empty equal-pad stack, real length all-gather), then the fast path
|
||||||
|
with the split hook's auto_pad marker pushed and _all_gather_int replaced
|
||||||
|
by a raising guard -- which also proves the collective was truly skipped,
|
||||||
|
not just silently replaced.
|
||||||
|
"""
|
||||||
|
sp_world_size = 2
|
||||||
|
if current_omni_platform.get_device_count() < sp_world_size:
|
||||||
|
pytest.skip(f"Test requires {sp_world_size} GPUs")
|
||||||
|
|
||||||
|
batch_size = 2
|
||||||
|
head_size = 8
|
||||||
|
seq_len = 6 # divisible by 2 so both ranks hold equal-length shards
|
||||||
|
|
||||||
|
init_method = get_distributed_init_method()
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
|
||||||
|
input_file = f_in.name
|
||||||
|
|
||||||
|
try:
|
||||||
|
torch.manual_seed(0)
|
||||||
|
kv_heads = num_kv_heads if num_kv_heads is not None else num_heads
|
||||||
|
np.savez(
|
||||||
|
input_file,
|
||||||
|
q=torch.randn(batch_size, seq_len, num_heads, head_size, dtype=torch.float32).numpy(),
|
||||||
|
k=torch.randn(batch_size, seq_len, kv_heads, head_size, dtype=torch.float32).numpy(),
|
||||||
|
v=torch.randn(batch_size, seq_len, kv_heads, head_size, dtype=torch.float32).numpy(),
|
||||||
|
)
|
||||||
|
|
||||||
|
torch.multiprocessing.spawn(
|
||||||
|
_run_contract_skip_case,
|
||||||
|
args=(
|
||||||
|
sp_world_size,
|
||||||
|
init_method,
|
||||||
|
input_file,
|
||||||
|
num_heads,
|
||||||
|
head_size,
|
||||||
|
num_kv_heads,
|
||||||
|
),
|
||||||
|
nprocs=sp_world_size,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
os.remove(input_file)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"num_heads,num_kv_heads,joint_len",
|
"num_heads,num_kv_heads,joint_len",
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -1113,6 +1113,120 @@ class TestStrictModeSplitValidation:
|
|||||||
hook._prepare_sp_input(x, metadata["x"], (), {})
|
hook._prepare_sp_input(x, metadata["x"], (), {})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.cpu
|
||||||
|
class TestEqualPadContract:
|
||||||
|
"""The stack of per-boundary auto_pad flags that lets attention skip a gather.
|
||||||
|
|
||||||
|
A boundary is recorded as equal-length only when the framework padded it,
|
||||||
|
because only then is every rank's shard guaranteed the same size.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _split_hook(self, *, auto_pad: bool, monkeypatch):
|
||||||
|
from vllm_omni.diffusion.attention import selector
|
||||||
|
from vllm_omni.diffusion.distributed import parallel_state, sp_sharding
|
||||||
|
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig
|
||||||
|
from vllm_omni.diffusion.hooks.sequence_parallel import (
|
||||||
|
SequenceParallelSplitHook,
|
||||||
|
)
|
||||||
|
|
||||||
|
class MaskBackend:
|
||||||
|
supports_attention_mask = True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_name():
|
||||||
|
return "test"
|
||||||
|
|
||||||
|
monkeypatch.setattr(parallel_state, "get_sequence_parallel_world_size", lambda: 2)
|
||||||
|
monkeypatch.setattr(parallel_state, "get_sequence_parallel_rank", lambda: 1)
|
||||||
|
monkeypatch.setattr(parallel_state, "get_ring_parallel_world_size", lambda: 1)
|
||||||
|
monkeypatch.setattr(selector, "get_attn_backend_for_role", lambda **_: (MaskBackend(), None))
|
||||||
|
# sp_shard, used by the manual path, binds these at import time.
|
||||||
|
monkeypatch.setattr(sp_sharding, "get_sequence_parallel_world_size", lambda: 2)
|
||||||
|
monkeypatch.setattr(sp_sharding, "get_sequence_parallel_rank", lambda: 1)
|
||||||
|
|
||||||
|
hook = SequenceParallelSplitHook(
|
||||||
|
{
|
||||||
|
0: SequenceParallelInput(
|
||||||
|
split_dim=1,
|
||||||
|
expected_dims=3,
|
||||||
|
split_output=True,
|
||||||
|
auto_pad=auto_pad,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
SequenceParallelConfig(ulysses_degree=2),
|
||||||
|
)
|
||||||
|
hook.initialize_hook(nn.Identity())
|
||||||
|
return hook
|
||||||
|
|
||||||
|
def test_auto_pad_split_claims_equal_rank_lengths(self, monkeypatch):
|
||||||
|
from vllm_omni.diffusion.forward_context import (
|
||||||
|
get_forward_context,
|
||||||
|
set_forward_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
hook = self._split_hook(auto_pad=True, monkeypatch=monkeypatch)
|
||||||
|
with set_forward_context():
|
||||||
|
hook.pre_forward(nn.Identity())
|
||||||
|
shard = hook.post_forward(nn.Identity(), torch.arange(5).reshape(1, 5, 1))
|
||||||
|
ctx = get_forward_context()
|
||||||
|
|
||||||
|
# 5 tokens padded to 6, so both ranks hold 3.
|
||||||
|
assert shard.shape[1] == 3
|
||||||
|
assert ctx._sp_equal_pad_stack == [True]
|
||||||
|
assert ctx.sp_rank_local_seq_lens_equal
|
||||||
|
|
||||||
|
def test_manual_split_does_not_claim_equal_rank_lengths(self, monkeypatch):
|
||||||
|
from vllm_omni.diffusion.forward_context import (
|
||||||
|
get_forward_context,
|
||||||
|
set_forward_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
hook = self._split_hook(auto_pad=False, monkeypatch=monkeypatch)
|
||||||
|
with set_forward_context():
|
||||||
|
hook.pre_forward(nn.Identity())
|
||||||
|
hook.post_forward(nn.Identity(), torch.arange(6).reshape(1, 6, 1))
|
||||||
|
ctx = get_forward_context()
|
||||||
|
|
||||||
|
assert ctx._sp_equal_pad_stack == [False]
|
||||||
|
assert not ctx.sp_rank_local_seq_lens_equal
|
||||||
|
|
||||||
|
def test_gather_pops_only_its_own_boundary(self, monkeypatch):
|
||||||
|
"""A stack, not a counter: the gather must not resurrect the contract.
|
||||||
|
|
||||||
|
With one auto_pad and one manual boundary open, a counter could not
|
||||||
|
tell which one a gather closes and would wrongly report equal lengths.
|
||||||
|
"""
|
||||||
|
from vllm_omni.diffusion.distributed.sp_plan import SequenceParallelConfig
|
||||||
|
from vllm_omni.diffusion.forward_context import (
|
||||||
|
get_forward_context,
|
||||||
|
set_forward_context,
|
||||||
|
)
|
||||||
|
from vllm_omni.diffusion.hooks import sequence_parallel
|
||||||
|
from vllm_omni.diffusion.hooks.sequence_parallel import (
|
||||||
|
SequenceParallelGatherHook,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sequence_parallel,
|
||||||
|
"sp_gather",
|
||||||
|
lambda tensor, dim, validate: torch.cat([tensor, tensor], dim=dim),
|
||||||
|
)
|
||||||
|
hook = SequenceParallelGatherHook(
|
||||||
|
SequenceParallelOutput(gather_dim=1, expected_dims=3),
|
||||||
|
SequenceParallelConfig(ulysses_degree=2),
|
||||||
|
)
|
||||||
|
|
||||||
|
with set_forward_context():
|
||||||
|
ctx = get_forward_context()
|
||||||
|
ctx._sp_shard_depth = 2
|
||||||
|
ctx._sp_equal_pad_stack.extend([False, True])
|
||||||
|
hook.post_forward(nn.Identity(), torch.zeros(1, 3, 4))
|
||||||
|
|
||||||
|
assert ctx._sp_shard_depth == 1
|
||||||
|
assert ctx._sp_equal_pad_stack == [False]
|
||||||
|
assert not ctx.sp_rank_local_seq_lens_equal
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.cpu
|
@pytest.mark.cpu
|
||||||
class TestSequenceParallelConfig:
|
class TestSequenceParallelConfig:
|
||||||
"""Test SequenceParallelConfig dataclass."""
|
"""Test SequenceParallelConfig dataclass."""
|
||||||
|
|||||||
@@ -260,6 +260,78 @@ def test_hwr_cold_publication_and_warm_restore_skip_ordinary_dit_loading(
|
|||||||
assert len(tuple((store_root / "source-digests-v1" / "entries").glob("*.json"))) == 1
|
assert len(tuple((store_root / "source-digests-v1" / "entries").glob("*.json"))) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("validation", ["manifest_and_metadata", "full_checksum"])
|
||||||
|
@pytest.mark.parametrize("mode", ["preferred", "required"])
|
||||||
|
def test_hwr_loader_payload_integrity_policy(tmp_path: Path, monkeypatch, validation: str, mode: str):
|
||||||
|
from vllm_omni.diffusion.offloader.startup import take_offload_startup_state
|
||||||
|
|
||||||
|
canonical = tmp_path / "canonical"
|
||||||
|
canonical.mkdir()
|
||||||
|
expected = torch.arange(4, dtype=torch.float32).to(torch.bfloat16).reshape(2, 2)
|
||||||
|
save_file({"weight": expected}, str(canonical / "model.safetensors"))
|
||||||
|
root = tmp_path / "store"
|
||||||
|
|
||||||
|
def make_loader(runtime_mode: str):
|
||||||
|
config = _hwr_config(canonical, root, mode=runtime_mode)
|
||||||
|
config.host_weight_runtime_validation = validation
|
||||||
|
loader = DiffusersPipelineLoader(LoadConfig(), config)
|
||||||
|
pipeline = _HWRPipeline(canonical)
|
||||||
|
monkeypatch.setattr(loader, "_init_from_load_format", lambda *args, **kwargs: pipeline)
|
||||||
|
return loader, pipeline
|
||||||
|
|
||||||
|
def close_plan(pipeline):
|
||||||
|
state = take_offload_startup_state(pipeline)
|
||||||
|
if state is not None and state.host_weight_plan is not None:
|
||||||
|
carrier = state.host_weight_plan.lease_carrier
|
||||||
|
if carrier is not None:
|
||||||
|
carrier.close()
|
||||||
|
|
||||||
|
cold_loader, cold = make_loader("preferred")
|
||||||
|
cold_loader.load_model(load_device="cpu", device=torch.device("cpu"))
|
||||||
|
assert cold.load_count == 1
|
||||||
|
close_plan(cold)
|
||||||
|
payloads = list((root / "artifacts").glob("*/*.safetensors"))
|
||||||
|
assert len(payloads) == 1
|
||||||
|
payload = payloads[0]
|
||||||
|
data = bytearray(payload.read_bytes())
|
||||||
|
data[-1] ^= 1 # Keep the header and file length valid; change only a BF16 payload byte.
|
||||||
|
original_mode = payload.stat().st_mode
|
||||||
|
payload.chmod(0o600)
|
||||||
|
payload.write_bytes(data)
|
||||||
|
payload.chmod(original_mode)
|
||||||
|
|
||||||
|
loader, pipeline = make_loader(mode)
|
||||||
|
try:
|
||||||
|
if validation == "full_checksum" and mode == "required":
|
||||||
|
with pytest.raises(RuntimeError, match="Host Weight Runtime resolution failed"):
|
||||||
|
loader.load_model(load_device="cpu", device=torch.device("cpu"))
|
||||||
|
assert pipeline.load_count == 0
|
||||||
|
assert loader.take_host_weight_plan() is None
|
||||||
|
else:
|
||||||
|
loader.load_model(load_device="cpu", device=torch.device("cpu"))
|
||||||
|
if validation == "manifest_and_metadata":
|
||||||
|
assert pipeline.load_count == 0
|
||||||
|
assert not torch.equal(pipeline.transformer.weight, expected)
|
||||||
|
else:
|
||||||
|
assert pipeline.load_count == 1
|
||||||
|
assert torch.equal(pipeline.transformer.weight, expected)
|
||||||
|
finally:
|
||||||
|
close_plan(pipeline)
|
||||||
|
|
||||||
|
if validation == "full_checksum":
|
||||||
|
if mode == "required":
|
||||||
|
assert list((root / "deny").iterdir())
|
||||||
|
else:
|
||||||
|
assert list((root / "quarantine").iterdir())
|
||||||
|
warm_loader, warm = make_loader(mode)
|
||||||
|
try:
|
||||||
|
warm_loader.load_model(load_device="cpu", device=torch.device("cpu"))
|
||||||
|
assert warm.load_count == 0
|
||||||
|
assert torch.equal(warm.transformer.weight, expected)
|
||||||
|
finally:
|
||||||
|
close_plan(warm)
|
||||||
|
|
||||||
|
|
||||||
def test_maybe_fuse_distilled_lora_skips_when_hwr_warm_snapshot_present():
|
def test_maybe_fuse_distilled_lora_skips_when_hwr_warm_snapshot_present():
|
||||||
cfg = SimpleNamespace(
|
cfg = SimpleNamespace(
|
||||||
lora_backend="distill",
|
lora_backend="distill",
|
||||||
|
|||||||
@@ -503,6 +503,8 @@ def test_serve_cli_forwards_hwr_policy_for_no_allgather_dlo():
|
|||||||
"--omni",
|
"--omni",
|
||||||
"--enable-distributed-layerwise-offload",
|
"--enable-distributed-layerwise-offload",
|
||||||
"--dlo-no-use-allgather",
|
"--dlo-no-use-allgather",
|
||||||
|
"--host-weight-runtime-validation",
|
||||||
|
"full_checksum",
|
||||||
"--host-weight-runtime-mode",
|
"--host-weight-runtime-mode",
|
||||||
"preferred",
|
"preferred",
|
||||||
"--host-weight-runtime-root",
|
"--host-weight-runtime-root",
|
||||||
@@ -519,6 +521,8 @@ def test_serve_cli_forwards_hwr_policy_for_no_allgather_dlo():
|
|||||||
assert explicit_kwargs["host_weight_runtime_mode"] == "preferred"
|
assert explicit_kwargs["host_weight_runtime_mode"] == "preferred"
|
||||||
assert explicit_kwargs["host_weight_runtime_root"] == "/var/cache/vllm-omni/hwr"
|
assert explicit_kwargs["host_weight_runtime_root"] == "/var/cache/vllm-omni/hwr"
|
||||||
assert explicit_kwargs["dlo_host_registration_limit_gib"] == 80
|
assert explicit_kwargs["dlo_host_registration_limit_gib"] == 80
|
||||||
|
assert explicit_kwargs["host_weight_runtime_validation"] == "full_checksum"
|
||||||
|
assert engine_args["host_weight_runtime_validation"] == "full_checksum"
|
||||||
assert engine_args["host_weight_runtime_mode"] == "preferred"
|
assert engine_args["host_weight_runtime_mode"] == "preferred"
|
||||||
assert engine_args["host_weight_runtime_root"] == "/var/cache/vllm-omni/hwr"
|
assert engine_args["host_weight_runtime_root"] == "/var/cache/vllm-omni/hwr"
|
||||||
assert engine_args["dlo_host_registration_limit_gib"] == 80
|
assert engine_args["dlo_host_registration_limit_gib"] == 80
|
||||||
|
|||||||
@@ -739,6 +739,7 @@ class _DiffusionConfigProjection:
|
|||||||
dlo_resident_layers: int = Field(default=0, ge=0)
|
dlo_resident_layers: int = Field(default=0, ge=0)
|
||||||
host_weight_runtime_mode: Literal["disabled", "preferred", "required"] = "disabled"
|
host_weight_runtime_mode: Literal["disabled", "preferred", "required"] = "disabled"
|
||||||
host_weight_runtime_root: str | None = None
|
host_weight_runtime_root: str | None = None
|
||||||
|
host_weight_runtime_validation: str = "manifest_and_metadata"
|
||||||
dlo_host_registration_limit_gib: float = Field(default=0.0, ge=0)
|
dlo_host_registration_limit_gib: float = Field(default=0.0, ge=0)
|
||||||
pin_cpu_memory: bool = True
|
pin_cpu_memory: bool = True
|
||||||
diffusion_compile_granularity: Literal["regional", "full"] = "regional"
|
diffusion_compile_granularity: Literal["regional", "full"] = "regional"
|
||||||
@@ -892,6 +893,7 @@ class _DiffusionConfigProjection:
|
|||||||
validate_host_weight_runtime_options(
|
validate_host_weight_runtime_options(
|
||||||
mode=self.host_weight_runtime_mode,
|
mode=self.host_weight_runtime_mode,
|
||||||
root=self.host_weight_runtime_root,
|
root=self.host_weight_runtime_root,
|
||||||
|
validation=self.host_weight_runtime_validation,
|
||||||
)
|
)
|
||||||
self.dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
|
self.dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
|
||||||
limit_gib=self.dlo_host_registration_limit_gib,
|
limit_gib=self.dlo_host_registration_limit_gib,
|
||||||
|
|||||||
@@ -483,6 +483,7 @@ class StageDeployConfig:
|
|||||||
dlo_resident_layers: int | None = None
|
dlo_resident_layers: int | None = None
|
||||||
host_weight_runtime_mode: str | None = None
|
host_weight_runtime_mode: str | None = None
|
||||||
host_weight_runtime_root: str | None = None
|
host_weight_runtime_root: str | None = None
|
||||||
|
host_weight_runtime_validation: str | None = None
|
||||||
dlo_host_registration_limit_gib: float | None = None
|
dlo_host_registration_limit_gib: float | None = None
|
||||||
# Diffusion-specific debug and observability knobs.
|
# Diffusion-specific debug and observability knobs.
|
||||||
enable_diffusion_pipeline_profiler: bool | None = None
|
enable_diffusion_pipeline_profiler: bool | None = None
|
||||||
|
|||||||
@@ -14,7 +14,11 @@ from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
|
|||||||
from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext
|
from vllm_omni.diffusion.attention.parallel.base import ParallelAttentionContext
|
||||||
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
from vllm_omni.diffusion.distributed.comm import SeqAllToAll4D
|
||||||
from vllm_omni.diffusion.distributed.group_coordinator import SequenceParallelGroupCoordinator
|
from vllm_omni.diffusion.distributed.group_coordinator import SequenceParallelGroupCoordinator
|
||||||
from vllm_omni.diffusion.forward_context import get_ulysses_mode
|
from vllm_omni.diffusion.forward_context import (
|
||||||
|
get_forward_context,
|
||||||
|
get_ulysses_mode,
|
||||||
|
is_forward_context_available,
|
||||||
|
)
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -142,7 +146,7 @@ def _ulysses_all_to_all_any_qkv(
|
|||||||
|
|
||||||
input_split_sizes = [s_local] * world_size
|
input_split_sizes = [s_local] * world_size
|
||||||
output_split_sizes = seq_lens
|
output_split_sizes = seq_lens
|
||||||
s_global = int(sum(output_split_sizes))
|
s_global = sum(output_split_sizes)
|
||||||
|
|
||||||
out = torch.empty((s_global, bsz, head_cnt_local, head_dim), device=x.device, dtype=x.dtype)
|
out = torch.empty((s_global, bsz, head_cnt_local, head_dim), device=x.device, dtype=x.dtype)
|
||||||
dist.all_to_all_single(
|
dist.all_to_all_single(
|
||||||
@@ -177,7 +181,7 @@ def _ulysses_all_to_all_any_o(
|
|||||||
return x
|
return x
|
||||||
|
|
||||||
bsz, s_global, head_cnt_local, head_dim = x.shape
|
bsz, s_global, head_cnt_local, head_dim = x.shape
|
||||||
s_local = int(local_seq_len)
|
s_local = local_seq_len
|
||||||
|
|
||||||
# (B, S_global, H_local, D) -> (S_global, B, H_local, D)
|
# (B, S_global, H_local, D) -> (S_global, B, H_local, D)
|
||||||
x_t = x.permute(1, 0, 2, 3).contiguous()
|
x_t = x.permute(1, 0, 2, 3).contiguous()
|
||||||
@@ -411,9 +415,22 @@ class UlyssesParallelAttention:
|
|||||||
f"(got scatter_idx={self._scatter_idx}, gather_idx={self._gather_idx})."
|
f"(got scatter_idx={self._scatter_idx}, gather_idx={self._gather_idx})."
|
||||||
)
|
)
|
||||||
|
|
||||||
local_seq_len = int(query.shape[1])
|
if is_forward_context_available() and get_forward_context().sp_rank_local_seq_lens_equal:
|
||||||
seq_lens = _all_gather_int(self._ulysses_pg, local_seq_len, device=query.device)
|
# auto_pad already made every rank's shard the same length, so
|
||||||
s_global = int(sum(seq_lens))
|
# the lengths are known without a collective. Keep local_seq_len
|
||||||
|
# as a SymInt (no int()) so torch.compile(dynamic=True) does not
|
||||||
|
# specialize on it, and skip the host sync + graph break that
|
||||||
|
# _all_gather_int would introduce in every attention call.
|
||||||
|
local_seq_len = query.shape[1]
|
||||||
|
seq_lens = [local_seq_len] * ulysses_world_size
|
||||||
|
else:
|
||||||
|
local_seq_len = int(query.shape[1])
|
||||||
|
seq_lens = _all_gather_int(
|
||||||
|
self._ulysses_pg,
|
||||||
|
local_seq_len,
|
||||||
|
device=query.device,
|
||||||
|
)
|
||||||
|
s_global = sum(seq_lens)
|
||||||
|
|
||||||
# In hybrid Ulysses+Ring, Ring attention uses P2P send/recv with fixed-shape
|
# In hybrid Ulysses+Ring, Ring attention uses P2P send/recv with fixed-shape
|
||||||
# buffers. This requires all ring ranks to have the same seq_len after the
|
# buffers. This requires all ring ranks to have the same seq_len after the
|
||||||
@@ -571,8 +588,8 @@ class UlyssesParallelAttention:
|
|||||||
joint_len=joint_len,
|
joint_len=joint_len,
|
||||||
joint_strategy=joint_strategy,
|
joint_strategy=joint_strategy,
|
||||||
use_uaa=(mode == "advanced_uaa"),
|
use_uaa=(mode == "advanced_uaa"),
|
||||||
uaa_seq_lens=tuple(int(x) for x in seq_lens) if mode == "advanced_uaa" else (),
|
uaa_seq_lens=tuple(seq_lens) if mode == "advanced_uaa" else (),
|
||||||
uaa_local_seq_len=int(local_seq_len) if mode == "advanced_uaa" else 0,
|
uaa_local_seq_len=local_seq_len if mode == "advanced_uaa" else 0,
|
||||||
orig_head_cnt=int(orig_head_cnt) if mode == "advanced_uaa" else 0,
|
orig_head_cnt=int(orig_head_cnt) if mode == "advanced_uaa" else 0,
|
||||||
joint_orig_head_cnt=int(joint_orig_head_cnt) if mode == "advanced_uaa" else 0,
|
joint_orig_head_cnt=int(joint_orig_head_cnt) if mode == "advanced_uaa" else 0,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -100,7 +100,9 @@ def normalize_omni_diffusion_kwargs(kwargs: Mapping[str, Any]) -> dict[str, Any]
|
|||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def validate_host_weight_runtime_options(*, mode: object, root: object) -> None:
|
def validate_host_weight_runtime_options(
|
||||||
|
*, mode: object, root: object, validation: object = "manifest_and_metadata"
|
||||||
|
) -> None:
|
||||||
"""Validate HWR policy without touching the configured storage domain.
|
"""Validate HWR policy without touching the configured storage domain.
|
||||||
|
|
||||||
Filesystem locality and store construction belong to the eligible loader
|
Filesystem locality and store construction belong to the eligible loader
|
||||||
@@ -109,6 +111,8 @@ def validate_host_weight_runtime_options(*, mode: object, root: object) -> None:
|
|||||||
"""
|
"""
|
||||||
if mode not in {"disabled", "preferred", "required"}:
|
if mode not in {"disabled", "preferred", "required"}:
|
||||||
raise ValueError("host_weight_runtime_mode must be disabled, preferred, or required")
|
raise ValueError("host_weight_runtime_mode must be disabled, preferred, or required")
|
||||||
|
if validation not in ("manifest_and_metadata", "full_checksum"):
|
||||||
|
raise ValueError("host_weight_runtime_validation must be manifest_and_metadata or full_checksum")
|
||||||
if mode != "disabled" and (not isinstance(root, str) or not root.strip()):
|
if mode != "disabled" and (not isinstance(root, str) or not root.strip()):
|
||||||
raise ValueError("enabled Host Weight Runtime requires host_weight_runtime_root")
|
raise ValueError("enabled Host Weight Runtime requires host_weight_runtime_root")
|
||||||
|
|
||||||
@@ -831,6 +835,7 @@ class OmniDiffusionConfig:
|
|||||||
# existing loader/storage path.
|
# existing loader/storage path.
|
||||||
host_weight_runtime_mode: str = "disabled"
|
host_weight_runtime_mode: str = "disabled"
|
||||||
host_weight_runtime_root: str | None = None
|
host_weight_runtime_root: str | None = None
|
||||||
|
host_weight_runtime_validation: str = "manifest_and_metadata"
|
||||||
# Optional per-worker ceiling for registering final-layout HWR mappings.
|
# Optional per-worker ceiling for registering final-layout HWR mappings.
|
||||||
# Zero adds no ceiling; pin_cpu_memory controls whether registration is tried.
|
# Zero adds no ceiling; pin_cpu_memory controls whether registration is tried.
|
||||||
dlo_host_registration_limit_gib: float = 0.0
|
dlo_host_registration_limit_gib: float = 0.0
|
||||||
@@ -1241,6 +1246,7 @@ class OmniDiffusionConfig:
|
|||||||
validate_host_weight_runtime_options(
|
validate_host_weight_runtime_options(
|
||||||
mode=self.host_weight_runtime_mode,
|
mode=self.host_weight_runtime_mode,
|
||||||
root=self.host_weight_runtime_root,
|
root=self.host_weight_runtime_root,
|
||||||
|
validation=self.host_weight_runtime_validation,
|
||||||
)
|
)
|
||||||
self.dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
|
self.dlo_host_registration_limit_gib = validate_dlo_host_registration_options(
|
||||||
limit_gib=self.dlo_host_registration_limit_gib,
|
limit_gib=self.dlo_host_registration_limit_gib,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -66,6 +66,10 @@ class ForwardContext:
|
|||||||
# Tracks the depth of SP sharding - incremented on shard, decremented on gather
|
# Tracks the depth of SP sharding - incremented on shard, decremented on gather
|
||||||
# Used by attention layers to determine if SP communication should be enabled
|
# Used by attention layers to determine if SP communication should be enabled
|
||||||
_sp_shard_depth: int = 0
|
_sp_shard_depth: int = 0
|
||||||
|
# One entry per active SP split boundary: whether it auto-pads, which makes
|
||||||
|
# every rank's shard the same length and lets attention collectives skip a
|
||||||
|
# runtime length all-gather. Pushed on split, popped on gather.
|
||||||
|
_sp_equal_pad_stack: list[bool] = field(default_factory=list)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def sp_active(self) -> bool:
|
def sp_active(self) -> bool:
|
||||||
@@ -88,6 +92,15 @@ class ForwardContext:
|
|||||||
sp_size = self.omni_diffusion_config.parallel_config.sequence_parallel_size
|
sp_size = self.omni_diffusion_config.parallel_config.sequence_parallel_size
|
||||||
return sp_size is not None and sp_size > 1
|
return sp_size is not None and sp_size > 1
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sp_rank_local_seq_lens_equal(self) -> bool:
|
||||||
|
"""Whether every active SP boundary guarantees equal local shard sizes.
|
||||||
|
|
||||||
|
Only framework-managed auto_pad boundaries provide this contract; a
|
||||||
|
region that also shards manually keeps the dynamic length exchange.
|
||||||
|
"""
|
||||||
|
return bool(self._sp_equal_pad_stack) and all(self._sp_equal_pad_stack)
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
@@ -264,6 +264,7 @@ class SequenceParallelSplitHook(ModelHook):
|
|||||||
|
|
||||||
output_list = [output] if is_tensor else list(output)
|
output_list = [output] if is_tensor else list(output)
|
||||||
actually_sharded = False
|
actually_sharded = False
|
||||||
|
equal_rank_seq_lens = True
|
||||||
|
|
||||||
for index, spm in self.metadata.items():
|
for index, spm in self.metadata.items():
|
||||||
if not isinstance(index, int):
|
if not isinstance(index, int):
|
||||||
@@ -277,10 +278,13 @@ class SequenceParallelSplitHook(ModelHook):
|
|||||||
output_list[index] = self._prepare_sp_input(original, spm, self._last_args, self._last_kwargs)
|
output_list[index] = self._prepare_sp_input(original, spm, self._last_args, self._last_kwargs)
|
||||||
if output_list[index] is not original:
|
if output_list[index] is not original:
|
||||||
actually_sharded = True
|
actually_sharded = True
|
||||||
|
equal_rank_seq_lens &= isinstance(spm, SequenceParallelInput) and spm.auto_pad
|
||||||
|
|
||||||
# Mark SP as active only if at least one tensor was actually sharded
|
# Mark SP as active only if at least one tensor was actually sharded
|
||||||
if actually_sharded and is_forward_context_available():
|
if actually_sharded and is_forward_context_available():
|
||||||
get_forward_context()._sp_shard_depth += 1
|
ctx = get_forward_context()
|
||||||
|
ctx._sp_shard_depth += 1
|
||||||
|
ctx._sp_equal_pad_stack.append(equal_rank_seq_lens)
|
||||||
|
|
||||||
return output_list[0] if is_tensor else type(output)(output_list)
|
return output_list[0] if is_tensor else type(output)(output_list)
|
||||||
|
|
||||||
@@ -553,6 +557,8 @@ class SequenceParallelGatherHook(ModelHook):
|
|||||||
if actually_gathered and is_forward_context_available():
|
if actually_gathered and is_forward_context_available():
|
||||||
ctx = get_forward_context()
|
ctx = get_forward_context()
|
||||||
ctx._sp_shard_depth = max(0, ctx._sp_shard_depth - 1)
|
ctx._sp_shard_depth = max(0, ctx._sp_shard_depth - 1)
|
||||||
|
if ctx._sp_equal_pad_stack:
|
||||||
|
ctx._sp_equal_pad_stack.pop()
|
||||||
|
|
||||||
return output[0] if is_tensor else type(output)(output)
|
return output[0] if is_tensor else type(output)(output)
|
||||||
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from vllm_omni.diffusion.model_loader.host_weight_plan import HostWeightPlan, Te
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
from vllm_omni.diffusion.model_loader.host_weights.identity_adapter import FinalLayoutIdentityContext
|
from vllm_omni.diffusion.model_loader.host_weights.identity_adapter import FinalLayoutIdentityContext
|
||||||
from vllm_omni.diffusion.model_loader.host_weights.source_identity import (
|
from vllm_omni.diffusion.model_loader.host_weights.source_identity import (
|
||||||
NodeSourceDigestCache,
|
NodeSourceDigestCache,
|
||||||
@@ -40,6 +41,8 @@ class _HWRCommitError(RuntimeError):
|
|||||||
class HWRLoaderMixin:
|
class HWRLoaderMixin:
|
||||||
"""Optional final-layout HWR behavior shared by diffusion loaders."""
|
"""Optional final-layout HWR behavior shared by diffusion loaders."""
|
||||||
|
|
||||||
|
od_config: OmniDiffusionConfig
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _identity_value(value: object) -> object:
|
def _identity_value(value: object) -> object:
|
||||||
"""Convert config objects into deterministic identity metadata."""
|
"""Convert config objects into deterministic identity metadata."""
|
||||||
@@ -319,10 +322,12 @@ class HWRLoaderMixin:
|
|||||||
HostWeightLeaseCarrier,
|
HostWeightLeaseCarrier,
|
||||||
HostWeightRuntime,
|
HostWeightRuntime,
|
||||||
HostWeightRuntimeConfig,
|
HostWeightRuntimeConfig,
|
||||||
|
IntegrityPolicy,
|
||||||
ProductionPolicy,
|
ProductionPolicy,
|
||||||
ResolutionOutcome,
|
ResolutionOutcome,
|
||||||
RuntimeMode,
|
RuntimeMode,
|
||||||
StorageDomainPolicy,
|
StorageDomainPolicy,
|
||||||
|
ValidationLevel,
|
||||||
)
|
)
|
||||||
from vllm_omni.host_weight_runtime.filesystem import detect_storage_class
|
from vllm_omni.host_weight_runtime.filesystem import detect_storage_class
|
||||||
|
|
||||||
@@ -363,6 +368,11 @@ class HWRLoaderMixin:
|
|||||||
HostWeightRuntimeConfig(
|
HostWeightRuntimeConfig(
|
||||||
mode=mode,
|
mode=mode,
|
||||||
domain=StorageDomainPolicy(root=root, storage_class=detect_storage_class(root)),
|
domain=StorageDomainPolicy(root=root, storage_class=detect_storage_class(root)),
|
||||||
|
integrity=IntegrityPolicy(
|
||||||
|
local_lookup=ValidationLevel(
|
||||||
|
getattr(self.od_config, "host_weight_runtime_validation", "manifest_and_metadata")
|
||||||
|
),
|
||||||
|
),
|
||||||
production=ProductionPolicy(
|
production=ProductionPolicy(
|
||||||
allow_local_build=False,
|
allow_local_build=False,
|
||||||
allow_post_load_publish=True,
|
allow_post_load_publish=True,
|
||||||
|
|||||||
@@ -582,6 +582,7 @@ class OrchestratorArgs:
|
|||||||
dlo_resident_layers: int = 0
|
dlo_resident_layers: int = 0
|
||||||
host_weight_runtime_mode: str = "disabled"
|
host_weight_runtime_mode: str = "disabled"
|
||||||
host_weight_runtime_root: str | None = None
|
host_weight_runtime_root: str | None = None
|
||||||
|
host_weight_runtime_validation: str = "manifest_and_metadata"
|
||||||
dlo_host_registration_limit_gib: float = 0.0
|
dlo_host_registration_limit_gib: float = 0.0
|
||||||
boundary_ratio: float | None = None
|
boundary_ratio: float | None = None
|
||||||
flow_shift: float | None = None
|
flow_shift: float | None = None
|
||||||
|
|||||||
@@ -1145,6 +1145,7 @@ class AsyncOmniEngine:
|
|||||||
"dlo_resident_layers": kwargs.get("dlo_resident_layers", 0),
|
"dlo_resident_layers": kwargs.get("dlo_resident_layers", 0),
|
||||||
"host_weight_runtime_mode": kwargs.get("host_weight_runtime_mode", "disabled"),
|
"host_weight_runtime_mode": kwargs.get("host_weight_runtime_mode", "disabled"),
|
||||||
"host_weight_runtime_root": kwargs.get("host_weight_runtime_root"),
|
"host_weight_runtime_root": kwargs.get("host_weight_runtime_root"),
|
||||||
|
"host_weight_runtime_validation": kwargs.get("host_weight_runtime_validation", "manifest_and_metadata"),
|
||||||
"dlo_host_registration_limit_gib": kwargs.get("dlo_host_registration_limit_gib", 0.0),
|
"dlo_host_registration_limit_gib": kwargs.get("dlo_host_registration_limit_gib", 0.0),
|
||||||
"enforce_eager": False if kwargs.get("enforce_eager") is None else kwargs.get("enforce_eager"),
|
"enforce_eager": False if kwargs.get("enforce_eager") is None else kwargs.get("enforce_eager"),
|
||||||
"diffusion_compile_granularity": (
|
"diffusion_compile_granularity": (
|
||||||
|
|||||||
@@ -770,6 +770,15 @@ class OmniServeCommand(CLISubcommand):
|
|||||||
"preferred first."
|
"preferred first."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
omni_config_group.add_argument(
|
||||||
|
"--host-weight-runtime-validation",
|
||||||
|
choices=("manifest_and_metadata", "full_checksum"),
|
||||||
|
default="manifest_and_metadata",
|
||||||
|
help=(
|
||||||
|
"HWR warm-acquisition validation. The default checks metadata, not payload corruption; "
|
||||||
|
"full_checksum reads and hashes every payload before restoring weights."
|
||||||
|
),
|
||||||
|
)
|
||||||
omni_config_group.add_argument(
|
omni_config_group.add_argument(
|
||||||
"--host-weight-runtime-root",
|
"--host-weight-runtime-root",
|
||||||
type=str,
|
type=str,
|
||||||
|
|||||||
Reference in new issue
Block a user