[Diffusion] Add native KV cache initialization and Scheduler-managed block allocation (#6094)

Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
This commit is contained in:
Yukim1andzwhzzz0821 authored and GitHub committed 2026-08-18 07:00:24 +00:00
1 parent b283215c28
commit 0a1846aa45
33 files changed
+2609 -265

No files matched your search

@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.v1.kv_cache_interface import FullAttentionSpec
import vllm_omni.diffusion.diffusion_kv.initialization as initialization
import vllm_omni.diffusion.vllm_config as diffusion_vllm_config
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler
from vllm_omni.quantization import build_quant_config
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _Executor:
def __init__(self) -> None:
self.specs = [{"layer0": object()}, {"layer0": object()}]
self.memory = [1024, 1024]
self.configs = None
self.profile_requests = None
def get_kv_cache_specs(self):
return self.specs
def determine_available_kv_memory(self, profile_requests):
self.profile_requests = profile_requests
return self.memory
def set_kv_cache_configs(self, configs) -> None:
self.configs = configs
def _od_config(**overrides):
values = dict(
model="test-diffusion",
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
dtype=torch.bfloat16,
max_model_len=64,
quantization_config=None,
tf_model_config=None,
enforce_eager=True,
is_moe=False,
additional_config={},
profiler_config=None,
parallel_config=SimpleNamespace(
tensor_parallel_size=1,
pipeline_parallel_size=1,
data_parallel_size=1,
enable_expert_parallel=False,
),
gpu_memory_utilization=0.9,
kv_cache_memory_bytes=None,
max_num_batched_tokens=64,
num_gpus=1,
)
values.update(overrides)
return SimpleNamespace(**values)
def test_control_plane_builds_worker_and_scheduler_configs(monkeypatch) -> None:
executor = _Executor()
od_config = _od_config(num_gpus=2)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=4096),
max_in_flight_tokens=128,
)
worker_configs = [object(), object()]
scheduler_kv_cache_config = object()
profile_requests = [object()]
monkeypatch.setattr(initialization, "create_diffusion_vllm_config", lambda *args, **kwargs: vllm_config)
build = Mock(return_value=(worker_configs, scheduler_kv_cache_config))
monkeypatch.setattr(
initialization,
"build_native_kv_cache_configs",
build,
)
monkeypatch.setattr(initialization, "resolve_kv_cache_block_sizes", lambda *args: (16, 16))
result = initialization.initialize_diffusion_kv_control_plane(
executor,
od_config,
profile_requests=profile_requests,
device=torch.device("cpu"),
)
build.assert_called_once_with(vllm_config, executor.specs, executor.memory)
assert result == (scheduler_kv_cache_config, 16, 16, vllm_config)
assert executor.configs == worker_configs
assert executor.profile_requests is profile_requests
def test_dense_mode_skips_worker_kv_initialization(monkeypatch) -> None:
executor = _Executor()
od_config = _od_config(diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY)
get_device = Mock(side_effect=AssertionError("dense mode must not resolve a KV device"))
monkeypatch.setattr(initialization.current_omni_platform, "get_torch_device", get_device)
result = initialization.initialize_diffusion_kv_control_plane(executor, od_config)
assert result is None
assert executor.configs is None
get_device.assert_not_called()
def test_control_plane_rejects_rank_count_mismatch(monkeypatch) -> None:
executor = _Executor()
executor.memory = [1024]
od_config = _od_config(num_gpus=2)
monkeypatch.setattr(initialization, "create_diffusion_vllm_config", lambda *args, **kwargs: object())
with pytest.raises(ValueError, match="rank count mismatch"):
initialization.initialize_diffusion_kv_control_plane(
executor,
od_config,
profile_requests=[object()],
device=torch.device("cpu"),
)
def test_paged_control_plane_requires_prepared_profile_request() -> None:
with pytest.raises(ValueError, match="prepared memory profile requests"):
initialization.initialize_diffusion_kv_control_plane(
_Executor(),
_od_config(),
device=torch.device("cpu"),
)
def test_native_config_builder_produces_matching_worker_and_scheduler_pools() -> None:
od_config = _od_config()
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(torch.device("cpu"), od_config)
spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
non_causal=True,
)
num_blocks = 32
worker_configs, scheduler_kv_cache_config = initialization.build_native_kv_cache_configs(
vllm_config,
[{"layer0": spec}],
[spec.page_size_bytes * num_blocks],
)
assert len(worker_configs) == 1
assert worker_configs[0].num_blocks == num_blocks
assert scheduler_kv_cache_config.num_blocks == num_blocks
assert worker_configs[0].kv_cache_groups == scheduler_kv_cache_config.kv_cache_groups
def test_native_control_plane_config_initializes_scheduler() -> None:
od_config = _od_config()
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(torch.device("cpu"), od_config)
spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
non_causal=True,
)
_, scheduler_kv_cache_config = initialization.build_native_kv_cache_configs(
vllm_config,
[{"layer0": spec}],
[spec.page_size_bytes * 32],
)
scheduler = RequestScheduler()
scheduler.initialize(
od_config,
kv_cache_config=scheduler_kv_cache_config,
scheduler_block_size=spec.block_size,
hash_block_size=spec.block_size,
kv_vllm_config=vllm_config,
)
assert scheduler._diffusion_kv_manager is not None
scheduler.close()
def test_dense_config_keeps_new_paged_sizing_inputs_inactive() -> None:
od_config = _od_config(
diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY,
parallel_config=SimpleNamespace(
tensor_parallel_size=1,
pipeline_parallel_size=4,
data_parallel_size=1,
enable_expert_parallel=False,
),
gpu_memory_utilization=0.5,
kv_cache_memory_bytes=4096,
max_num_batched_tokens=64,
)
vllm_config = diffusion_vllm_config.create_base_diffusion_vllm_config(torch.device("cpu"), od_config)
original_pp_size = vllm_config.parallel_config.pipeline_parallel_size
original_memory_utilization = vllm_config.cache_config.gpu_memory_utilization
original_memory_bytes = vllm_config.cache_config.kv_cache_memory_bytes
original_token_budget = vllm_config.scheduler_config.max_num_batched_tokens
diffusion_vllm_config.configure_diffusion_vllm_config(vllm_config, od_config)
assert vllm_config.parallel_config.pipeline_parallel_size == original_pp_size
assert vllm_config.cache_config.gpu_memory_utilization == original_memory_utilization
assert vllm_config.cache_config.kv_cache_memory_bytes == original_memory_bytes
assert vllm_config.scheduler_config.max_num_batched_tokens == original_token_budget
def test_paged_config_forwards_gpu_memory_utilization_to_native_cache_config() -> None:
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(
torch.device("cpu"),
_od_config(gpu_memory_utilization=0.42),
)
assert vllm_config.cache_config.gpu_memory_utilization == 0.42
def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods() -> None:
quantization_config = build_quant_config(
{
"quant_method": "modelopt",
"quant_algo": "FP8",
"ignore": [],
}
)
model_config = diffusion_vllm_config._make_diffusion_vllm_model_config(
_od_config(
diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY,
quantization_config=quantization_config,
)
)
assert model_config.dtype is torch.bfloat16
assert model_config.quantization == "modelopt"
assert model_config.quantization_config is quantization_config
assert model_config.is_quantized
assert model_config.is_moe is False
assert model_config.is_nvfp4_quantized() is False
assert model_config.original_max_model_len == model_config.max_model_len
unquantized_model_config = diffusion_vllm_config._make_diffusion_vllm_model_config(
_od_config(
diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY,
quantization_config=None,
)
)
assert not unquantized_model_config.is_quantized
def test_minus_one_max_model_len_uses_model_limit() -> None:
od_config = SimpleNamespace(
max_model_len=-1,
tf_model_config=SimpleNamespace(max_position_embeddings=4096),
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
)
assert diffusion_vllm_config.resolve_diffusion_max_model_len(od_config) == 4096
model_config = diffusion_vllm_config._make_diffusion_vllm_model_config(
_od_config(
max_model_len=-1,
tf_model_config=SimpleNamespace(max_position_embeddings=4096),
)
)
assert model_config.max_model_len == 4096
assert model_config.original_max_model_len == -1
def test_minus_one_max_model_len_is_auto_fitted_by_native_cache_sizing() -> None:
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(
torch.device("cpu"),
_od_config(
max_model_len=-1,
tf_model_config=SimpleNamespace(max_position_embeddings=4096),
),
)
spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
non_causal=True,
)
initialization.build_native_kv_cache_configs(
vllm_config,
[{"layer0": spec}],
[spec.page_size_bytes * 16],
)
assert vllm_config.model_config.original_max_model_len == -1
assert vllm_config.model_config.max_model_len == 256
@@ -0,0 +1,211 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVAdmissionError, DiffusionKVCacheManager
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVContext, DiffusionKVRequest
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
BLOCK_SIZE = 4
def _config(num_blocks: int) -> KVCacheConfig:
native_kv_managers.register_all_kvcache_specs(None)
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
)
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=["layer0"])],
kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)],
)
def _request(public_id: str, sequence_id: int, *, seq_len: int = 8, kv_contexts=()) -> DiffusionKVRequest:
return DiffusionKVRequest(
f"{public_id}/diffusion-kv/{sequence_id}",
sequence_id=sequence_id,
prefix_len=4,
target_len=4,
seq_len=seq_len,
kv_contexts=kv_contexts,
)
def _manager(num_blocks: int, *, max_model_len: int = 64) -> DiffusionKVCacheManager:
return DiffusionKVCacheManager(
_config(num_blocks),
max_model_len=max_model_len,
scheduler_block_size=BLOCK_SIZE,
hash_block_size=BLOCK_SIZE,
)
def test_reserve_and_free_multi_cfg_request() -> None:
manager = _manager(8)
free_before = manager.native_manager.block_pool.get_num_free_blocks()
requests = (_request("public", 0), _request("public", 1))
metadata = manager.reserve_request("public", requests)
assert metadata is not None
assert metadata.request_id == "public"
assert metadata.allocation_generation == 1
assert [sequence.sequence_id for sequence in metadata.sequences] == [0, 1]
assert [sequence.prefix_len for sequence in metadata.sequences] == [4, 4]
assert [sequence.target_len for sequence in metadata.sequences] == [4, 4]
assert [sequence.seq_len for sequence in metadata.sequences] == [8, 8]
assert [len(sequence.block_ids[0]) for sequence in metadata.sequences] == [2, 2]
assert all(sequence.context_ids == () for sequence in metadata.sequences)
assert metadata.contexts == ()
assert metadata == manager.get_metadata("public")
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before - 4
manager.free_request("public")
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
def test_impossible_cfg_allocation_rolls_back_and_fails_fast() -> None:
manager = _manager(3)
free_before = manager.native_manager.block_pool.get_num_free_blocks()
with pytest.raises(DiffusionKVAdmissionError, match="cannot fit even when the block pool is empty"):
manager.reserve_request(
"public",
(_request("public", 0), _request("public", 1)),
)
assert manager.has_request("public") is False
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
def test_impossible_cfg_allocation_fails_fast_while_pool_is_busy() -> None:
manager = _manager(5)
assert manager.reserve_request("running", (_request("running", 0),)) is not None
free_before = manager.native_manager.block_pool.get_num_free_blocks()
with pytest.raises(DiffusionKVAdmissionError, match="required_blocks=6, available_blocks=4"):
manager.reserve_request(
"impossible",
tuple(_request("impossible", sequence_id) for sequence_id in range(3)),
)
assert manager.has_request("impossible") is False
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
def test_temporary_capacity_pressure_returns_none() -> None:
manager = _manager(3)
assert manager.reserve_request("running", (_request("running", 0),)) is not None
allocation = manager.reserve_request("waiting", (_request("waiting", 0),))
assert allocation is None
assert manager.has_request("waiting") is False
def test_partial_cfg_rollback_can_retry_after_capacity_is_released() -> None:
manager = _manager(5)
assert manager.reserve_request("running", (_request("running", 0),)) is not None
waiting = (_request("waiting", 0), _request("waiting", 1))
assert manager.reserve_request("waiting", waiting) is None
assert manager.has_request("waiting") is False
manager.free_request("running")
metadata = manager.reserve_request("waiting", waiting)
assert metadata is not None
assert [len(sequence.block_ids[0]) for sequence in metadata.sequences] == [2, 2]
def test_reallocated_request_gets_a_new_allocation_generation() -> None:
manager = _manager(3)
request = (_request("public", 0),)
first = manager.reserve_request("public", request)
assert first is not None
manager.free_request("public")
second = manager.reserve_request("public", request)
assert second is not None
assert second.allocation_generation == first.allocation_generation + 1
def test_cfg_allocation_exception_rolls_back(monkeypatch) -> None:
manager = _manager(8)
free_before = manager.native_manager.block_pool.get_num_free_blocks()
native_allocate = manager.native_manager.allocate_slots
num_calls = 0
def fail_second_allocation(*args, **kwargs):
nonlocal num_calls
num_calls += 1
if num_calls == 2:
raise RuntimeError("injected allocation failure")
return native_allocate(*args, **kwargs)
monkeypatch.setattr(manager.native_manager, "allocate_slots", fail_second_allocation)
with pytest.raises(RuntimeError, match="injected allocation failure"):
manager.reserve_request(
"public",
(_request("public", 0), _request("public", 1)),
)
assert manager.has_request("public") is False
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
def test_rejects_independent_context_until_role_routing_is_implemented() -> None:
manager = _manager(8)
context = DiffusionKVContext(context_id="text", cache_role="cross.text", num_tokens=8)
with pytest.raises(DiffusionKVAdmissionError, match="DiffusionKVContext"):
manager.reserve_request(
"public",
(_request("public", 0, kv_contexts=(context,)),),
)
def test_rejects_sequence_longer_than_admission_bound() -> None:
manager = _manager(8, max_model_len=8)
with pytest.raises(DiffusionKVAdmissionError, match="exceeds max_model_len"):
manager.reserve_request("public", (_request("public", 0, seq_len=9),))
@pytest.mark.parametrize(
"requests",
[
(_request("public", 1),),
(_request("public", 1), _request("public", 0)),
(_request("public", 0), _request("public", 2)),
],
)
def test_rejects_ambiguous_sequence_order(requests) -> None:
manager = _manager(8)
with pytest.raises(ValueError, match="ordered by contiguous sequence_id"):
manager.reserve_request("public", requests)
def test_close_releases_every_public_request() -> None:
manager = _manager(8)
free_before = manager.native_manager.block_pool.get_num_free_blocks()
assert manager.reserve_request("first", (_request("first", 0),)) is not None
assert manager.reserve_request("second", (_request("second", 0),)) is not None
manager.close()
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
assert manager.has_request("first") is False
assert manager.has_request("second") is False
+2 -5
View File
@@ -107,11 +107,8 @@ def test_request_rejects_non_context_values() -> None:
def _manager(*, enable_caching: bool) -> KVCacheManager:
# vLLM 0.26 lazily registers its built-in KV specs. Another test may have
# already registered an out-of-tree spec, making the registry non-empty
# before FullAttentionSpec is registered. Initialize the native mappings
# explicitly; vLLM 0.25 has no registration hook and needs no setup.
register_all = getattr(native_kv_managers, "register_all_kvcache_specs", None)
if register_all is not None:
register_all(None)
# before FullAttentionSpec is registered. Initialize the native mappings.
native_kv_managers.register_all_kvcache_specs(None)
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from torch import nn
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
import vllm_omni.diffusion.worker.diffusion_worker as diffusion_worker_module
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _Backend:
@classmethod
def indexes_kv_by_block_stride(cls) -> bool:
return True
def _attention(*, enabled: bool) -> Attention:
attention = Attention.__new__(Attention)
nn.Module.__init__(attention)
attention.paged_kv_cache_role = "primary" if enabled else None
attention.paged_kv_cache_dtype = torch.bfloat16
attention.num_kv_heads = 2
attention.head_size = 8
attention.causal = False
attention.attn_backend = _Backend
return attention
def _runner(attention: Attention) -> DiffusionModelRunner:
runner = object.__new__(DiffusionModelRunner)
runner.od_config = SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER)
runner.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=4),
model_config=SimpleNamespace(dtype=torch.float16),
)
pipeline = nn.Module()
pipeline.image_attention = attention
runner.pipeline = pipeline
return runner
def test_runner_discovers_native_spec_from_loaded_attention() -> None:
specs = _runner(_attention(enabled=True)).get_kv_cache_spec()
assert set(specs) == {"image_attention"}
spec = specs["image_attention"]
assert isinstance(spec, FullAttentionSpec)
assert spec.block_size == 4
assert spec.num_kv_heads == 2
assert spec.head_size == 8
assert spec.dtype is torch.bfloat16
assert spec.indexes_kv_by_block_stride is True
assert spec.non_causal is True
def test_runner_rejects_paged_mode_without_cache_enabled_attention() -> None:
runner = _runner(_attention(enabled=False))
with pytest.raises(RuntimeError, match="no cache-enabled Attention"):
runner.get_kv_cache_spec()
def test_runner_retains_matching_rank_local_config() -> None:
runner = _runner(_attention(enabled=True))
spec = runner.get_kv_cache_spec()["image_attention"]
config = KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * 8, shared_by=["image_attention"])],
kv_cache_groups=[KVCacheGroupSpec(layer_names=["image_attention"], kv_cache_spec=spec)],
)
runner.set_kv_cache_config(config)
assert runner.kv_cache_config is config
def test_runner_rejects_rank_local_config_for_different_layers() -> None:
runner = _runner(_attention(enabled=True))
spec = runner.get_kv_cache_spec()["image_attention"]
config = KVCacheConfig(
num_blocks=8,
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * 8, shared_by=["other_attention"])],
kv_cache_groups=[KVCacheGroupSpec(layer_names=["other_attention"], kv_cache_spec=spec)],
)
with pytest.raises(ValueError, match="layer mismatch"):
runner.set_kv_cache_config(config)
def test_worker_selects_its_rank_local_config() -> None:
worker = object.__new__(DiffusionWorker)
worker.rank = 1
worker.od_config = SimpleNamespace(num_gpus=2)
worker.model_runner = SimpleNamespace(set_kv_cache_config=lambda config: setattr(worker, "installed", config))
configs = [object(), object()]
worker.set_kv_cache_configs(configs)
assert worker.installed is configs[1]
with pytest.raises(ValueError, match="rank count mismatch"):
worker.set_kv_cache_configs(configs[:1])
def test_worker_honors_explicit_kv_memory_budget(monkeypatch) -> None:
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.vllm_config = SimpleNamespace(cache_config=SimpleNamespace(kv_cache_memory_bytes=4096))
worker.init_snapshot = SimpleNamespace(free_memory=16384)
worker.requested_memory = 8192
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=1024)
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
profile_request = object()
assert worker.determine_available_kv_memory(profile_request) == [4096]
worker.model_runner.profile_run.assert_called_once_with(profile_request)
def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch) -> None:
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.init_snapshot = SimpleNamespace(free_memory=900)
worker.requested_memory = 750
worker.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
kv_cache_memory_bytes=0,
gpu_memory_utilization=0.75,
)
)
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=100)
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
profile_result = SimpleNamespace(
non_kv_cache_memory=400,
after_profile=SimpleNamespace(free_memory=500),
)
monkeypatch.setattr(
diffusion_worker_module,
"memory_profiling",
lambda snapshot, weights_memory: nullcontext(profile_result),
)
profile_request = object()
assert worker.determine_available_kv_memory(profile_request) == [350]
worker.model_runner.profile_run.assert_called_once_with(profile_request)
@pytest.mark.parametrize("non_kv_cache_memory", [750, 800])
def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cache_memory: int) -> None:
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.init_snapshot = SimpleNamespace(free_memory=900)
worker.requested_memory = 750
worker.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
kv_cache_memory_bytes=None,
gpu_memory_utilization=0.75,
)
)
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=100)
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
profile_result = SimpleNamespace(
non_kv_cache_memory=non_kv_cache_memory,
after_profile=SimpleNamespace(free_memory=100),
)
monkeypatch.setattr(
diffusion_worker_module,
"memory_profiling",
lambda snapshot, weights_memory: nullcontext(profile_result),
)
profile_request = object()
with pytest.raises(
RuntimeError,
match=(
"rank 0: RuntimeError: No memory remains for Diffusion KV cache after profiling: "
f"requested_memory=750 bytes, non_kv_cache_memory={non_kv_cache_memory} bytes"
),
):
worker.determine_available_kv_memory(profile_request)
worker.model_runner.profile_run.assert_called_once_with(profile_request)
def test_worker_profiles_activation_headroom_instead_of_current_residency(monkeypatch) -> None:
worker = object.__new__(DiffusionWorker)
worker.rank = 0
worker.init_snapshot = SimpleNamespace(free_memory=1200)
worker.requested_memory = 1000
worker.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
kv_cache_memory_bytes=None,
gpu_memory_utilization=0.8,
)
)
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=250)
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
profile_result = SimpleNamespace(
non_kv_cache_memory=650,
after_profile=SimpleNamespace(free_memory=550),
)
profiling_calls = []
def profile(snapshot, *, weights_memory):
profiling_calls.append((snapshot, weights_memory))
return nullcontext(profile_result)
monkeypatch.setattr(
diffusion_worker_module,
"memory_profiling",
profile,
)
profile_request = object()
assert worker.determine_available_kv_memory(profile_request) == [350]
assert profiling_calls == [(worker.init_snapshot, 250)]
worker.model_runner.profile_run.assert_called_once_with(profile_request)
def test_worker_rejects_profile_without_preload_snapshot(monkeypatch) -> None:
worker = object.__new__(DiffusionWorker)
worker.init_snapshot = None
worker.requested_memory = None
worker.vllm_config = SimpleNamespace(cache_config=SimpleNamespace(kv_cache_memory_bytes=None))
worker.model_runner = SimpleNamespace()
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
with pytest.raises(RuntimeError, match="snapshot was not captured before model loading"):
worker.determine_available_kv_memory(object())
def test_rank_probe_gathers_local_failure_before_raising(monkeypatch) -> None:
gathered = []
def gather(local_result):
gathered.append(local_result)
return [local_result, (True, "peer-result")]
def fail_local_probe():
raise ValueError("local failure")
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", gather)
with pytest.raises(RuntimeError, match="rank 0: ValueError: local failure"):
diffusion_worker_module._run_and_gather_rank_values(
"test probe",
fail_local_probe,
)
assert gathered == [(False, "ValueError: local failure")]
@@ -85,19 +85,15 @@ def test_new_request_data_carries_scheduler_allocation_atomically() -> None:
assert new_req.diffusion_kv_metadata is metadata
def test_paged_request_without_metadata_reaches_request_forward() -> None:
def test_paged_request_without_metadata_fails_before_request_forward() -> None:
runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER)
expected_output = DiffusionOutput(output=None)
runner._execute_request_list = Mock(
return_value=SimpleNamespace(
runner_outputs=[SimpleNamespace(result=expected_output)],
)
)
runner._execute_request_list = Mock()
req = SimpleNamespace(request_id="req-0")
assert runner.execute_model(req) is expected_output
with pytest.raises(ValueError, match="requires Diffusion KV metadata"):
runner.execute_model(req)
runner._execute_request_list.assert_called_once()
runner._execute_request_list.assert_not_called()
def test_dense_request_rejects_metadata_before_request_forward() -> None:
@@ -141,18 +137,18 @@ def test_request_rpc_rejects_envelope_request_identity_mismatch() -> None:
assert calls == []
def test_batch_path_allows_missing_metadata_before_forward() -> None:
def test_batch_path_rejects_missing_metadata_before_forward() -> None:
runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER)
expected_output = object()
runner._execute_request_list = Mock(return_value=expected_output)
runner._execute_request_list = Mock()
new_req = NewRequestData(
request_id="req-0",
req=SimpleNamespace(request_id="req-0"),
)
assert runner.execute_model_batch(make_scheduler_output(new_req), runner.od_config) is expected_output
with pytest.raises(ValueError, match="requires Diffusion KV metadata"):
runner.execute_model_batch(make_scheduler_output(new_req), runner.od_config)
runner._execute_request_list.assert_called_once()
runner._execute_request_list.assert_not_called()
def test_batch_path_rejects_envelope_request_identity_mismatch() -> None:
@@ -170,7 +166,7 @@ def test_batch_path_rejects_envelope_request_identity_mismatch() -> None:
runner._execute_request_list.assert_not_called()
def test_step_path_allows_missing_metadata_before_step_support_check() -> None:
def test_step_path_rejects_missing_metadata_before_step_support_check() -> None:
runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER)
runner.pipeline = object()
runner._supports_step_mode = Mock(return_value=False)
@@ -179,10 +175,10 @@ def test_step_path_allows_missing_metadata_before_step_support_check() -> None:
req=SimpleNamespace(request_id="req-0"),
)
with pytest.raises(ValueError, match="does not support step execution"):
with pytest.raises(ValueError, match="requires Diffusion KV metadata"):
runner.execute_stepwise(make_scheduler_output(new_req))
runner._supports_step_mode.assert_called_once()
runner._supports_step_mode.assert_not_called()
def test_step_path_rejects_envelope_request_identity_mismatch() -> None:
@@ -288,3 +284,42 @@ def test_dense_worker_call_does_not_extend_model_runner_signature() -> None:
worker.execute_model(req, SimpleNamespace())
assert calls == [(req, None)]
def test_dlo_worker_selects_request_and_metadata_from_same_envelope(monkeypatch) -> None:
calls: list[tuple] = []
class ModelRunner:
def execute_model(self, req, **kwargs):
calls.append((req, kwargs))
return DiffusionOutput(output=None)
worker = object.__new__(DiffusionWorker)
worker.model_runner = ModelRunner()
worker.lora_manager = None
worker._get_profiler = lambda: None
monkeypatch.setattr(
"vllm_omni.diffusion.distributed.parallel_state.get_data_parallel_rank",
lambda: 1,
)
req_0 = SimpleNamespace(request_id="req-0")
req_1 = SimpleNamespace(request_id="req-1")
metadata_0 = make_metadata("req-0")
metadata_1 = make_metadata("req-1")
envelopes = [
NewRequestData(request_id="req-0", req=req_0, diffusion_kv_metadata=metadata_0),
NewRequestData(request_id="req-1", req=req_1, diffusion_kv_metadata=metadata_1),
]
result = worker.execute_model(envelopes, SimpleNamespace())
assert calls == [
(
req_1,
{
"kv_prefetch_job": None,
"diffusion_kv_metadata": metadata_1,
},
)
]
assert result["dp_rank"] == 1
@@ -118,6 +118,14 @@ def _call_mgr(
)
def test_cache_manager_registers_attention_without_adding_dense_state() -> None:
mgr = _make_cache_mgr()
assert isinstance(mgr, nn.Module)
assert dict(mgr.named_modules())["attn"] is mgr.attn
assert mgr.state_dict() == {}
# ============================================================
# Test 1: No AR KV — basic cache → reuse
# ============================================================
@@ -130,6 +130,21 @@ class TestCreateDefaultDiffusion:
assert od.diffusion_kv_mode is DiffusionKVCacheMode.PAGED_SCHEDULER
def test_diffusion_kv_sizing_fields_roundtrip(self, monkeypatch):
monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda self: 29500)
od = _roundtrip_diffusion_config(
model="x",
kv_cache_memory_bytes=4096,
gpu_memory_utilization=0.75,
max_num_batched_tokens=2048,
max_model_len=4096,
)
assert od.kv_cache_memory_bytes == 4096
assert od.gpu_memory_utilization == 0.75
assert od.max_num_batched_tokens == 2048
assert od.max_model_len == 4096
def test_qwen_image_edit_plus_sets_generic_multimodal_limit():
od_config = OmniDiffusionConfig(model="Qwen/Qwen-Image-Edit-2511", model_class_name="QwenImageEditPlusPipeline")
@@ -161,9 +161,83 @@ def test_scheduler_initialization_failure_closes_scheduler_and_executor(monkeypa
assert exc_info.value is initialization_error
custom_scheduler.initialize.assert_called_once_with(od_config)
custom_scheduler.close.assert_called_once_with()
def test_init_shuts_down_executor_when_kv_control_plane_initialization_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
fake_executor = SimpleNamespace(shutdown=Mock())
monkeypatch.setattr(DiffusionEngine, "_init_process_hooks", lambda self, config: None)
monkeypatch.setattr(
DiffusionEngine,
"_resolve_execution_mode",
lambda self, config: DiffusionExecutionMode.REQUEST_BATCH,
)
monkeypatch.setattr(
DiffusionEngine, "_init_executor", lambda self, config: setattr(self, "executor", fake_executor)
)
monkeypatch.setattr(
"vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane",
Mock(side_effect=RuntimeError("KV initialization failed")),
)
with pytest.raises(RuntimeError, match="KV initialization failed"):
DiffusionEngine(SimpleNamespace())
fake_executor.shutdown.assert_called_once_with()
def test_paged_init_profiles_before_scheduler_initialization(monkeypatch: pytest.MonkeyPatch) -> None:
events = []
expected_profile_requests = [object()]
fake_executor = SimpleNamespace(shutdown=Mock())
kv_cache_config = object()
kv_vllm_config = object()
od_config = SimpleNamespace(diffusion_kv_mode="paged_scheduler")
monkeypatch.setattr(DiffusionEngine, "_init_process_hooks", lambda self, config: None)
monkeypatch.setattr(
DiffusionEngine,
"_resolve_execution_mode",
lambda self, config: DiffusionExecutionMode.REQUEST_BATCH,
)
monkeypatch.setattr(
DiffusionEngine,
"_init_executor",
lambda self, config: setattr(self, "executor", fake_executor),
)
monkeypatch.setattr(
DiffusionEngine,
"_prepare_diffusion_kv_profile_requests",
lambda self: events.append("prepare-profile") or expected_profile_requests,
)
def initialize(executor, config, *, profile_requests):
events.append("profile-workers")
assert executor is fake_executor
assert config is od_config
assert profile_requests is expected_profile_requests
return kv_cache_config, 16, 16, kv_vllm_config
monkeypatch.setattr(
"vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane",
initialize,
)
def init_scheduler(self, config, scheduler, *args, **kwargs):
events.append("initialize-scheduler")
self.scheduler = SimpleNamespace(close=Mock())
monkeypatch.setattr(DiffusionEngine, "_init_scheduler", init_scheduler)
monkeypatch.setattr(DiffusionEngine, "_init_runtime_state", lambda self: None)
monkeypatch.setattr(DiffusionEngine, "_init_execute_fn", lambda self: None)
monkeypatch.setattr(DiffusionEngine, "_log_execution_mode", lambda self, config: None)
DiffusionEngine(od_config)
assert events == ["prepare-profile", "profile-workers", "initialize-scheduler"]
@pytest.mark.asyncio
async def test_step_compatibility_wrapper_returns_final_batch() -> None:
engine = _make_engine()
@@ -1,9 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from vllm_omni.diffusion import io_support
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@@ -55,3 +63,151 @@ def test_dummy_run_num_frames_uses_audio_input_fallback(monkeypatch: pytest.Monk
)
assert io_support.get_dummy_run_num_frames("unknown", supports_audio_input=True) == 2
def test_dummy_run_image_count_resolves_hunyuan_architecture_alias() -> None:
assert io_support.get_dummy_run_num_image_inputs("HunyuanImage3ForCausalMM") == 3
assert io_support.get_dummy_run_num_image_inputs("unknown") == 1
def test_dense_mode_does_not_build_kv_profile_request() -> None:
engine = object.__new__(DiffusionEngine)
engine.od_config = SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY)
engine._make_dummy_request = Mock(side_effect=AssertionError("dense mode must not profile"))
assert engine._prepare_diffusion_kv_profile_requests() is None
engine._make_dummy_request.assert_not_called()
@pytest.mark.parametrize(
("execution_mode", "uses_dlo_dp", "max_num_seqs", "expected_profile_requests"),
[
(DiffusionExecutionMode.STEP_BATCH, False, 2, 2),
(DiffusionExecutionMode.REQUEST_BATCH, True, 4, 1),
(DiffusionExecutionMode.STEP_BATCH, True, 3, 3),
],
)
def test_paged_kv_profile_requests_match_per_rank_batch(
execution_mode: DiffusionExecutionMode,
uses_dlo_dp: bool,
max_num_seqs: int,
expected_profile_requests: int,
) -> None:
engine = object.__new__(DiffusionEngine)
engine.execution_mode = execution_mode
engine.od_config = SimpleNamespace(
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
model_class_name="HunyuanImage3ForCausalMM",
max_num_seqs=max_num_seqs,
parallel_config=SimpleNamespace(data_parallel_size=max_num_seqs if uses_dlo_dp else 1),
enable_distributed_layerwise_offload=uses_dlo_dp,
dlo_use_allgather=True,
)
request = OmniDiffusionRequest(
prompt="profile",
request_id="profile-request",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
)
engine._make_dummy_request = Mock(return_value=request)
prepared_layout = object()
scheduler_kv_state = (
DiffusionKVRequest(
"profile-request/diffusion-kv/0",
sequence_id=0,
prefix_len=3,
target_len=8,
seq_len=12,
),
DiffusionKVRequest(
"profile-request/diffusion-kv/1",
sequence_id=1,
prefix_len=4,
target_len=8,
seq_len=13,
),
)
def preprocess(req: OmniDiffusionRequest) -> OmniDiffusionRequest:
req.prepared_layout = prepared_layout
req.diffusion_kv_requests = scheduler_kv_state
return req
engine._prepare_request_for_admission = Mock(side_effect=preprocess)
result = engine._prepare_diffusion_kv_profile_requests()
assert result is not None
assert len(result) == expected_profile_requests
assert len({profile_request.request_id for profile_request in result}) == expected_profile_requests
assert all(profile_request.is_dummy_run() for profile_request in result)
assert all(profile_request.prepared_layout is prepared_layout for profile_request in result)
assert all(profile_request.diffusion_kv_requests is None for profile_request in result)
assert len({id(profile_request.sampling_params) for profile_request in result}) == expected_profile_requests
assert engine._diffusion_kv_profile_limits == (2, 13, 8)
assert len(result) * engine._diffusion_kv_profile_limits[0] == expected_profile_requests * 2
engine._make_dummy_request.assert_called_once_with(
height=1024,
width=1024,
guidance_scale=5.0,
num_image_inputs=3,
)
engine._prepare_request_for_admission.assert_called_once_with(request)
@pytest.mark.parametrize(
("num_sequences", "seq_len", "target_len"),
[
(3, 16, 8),
(1, 17, 8),
(1, 16, 9),
],
)
def test_paged_kv_admission_rejects_shape_beyond_profile_envelope(
num_sequences: int,
seq_len: int,
target_len: int,
) -> None:
engine = object.__new__(DiffusionEngine)
engine._diffusion_kv_profile_limits = (2, 16, 8)
engine.pre_process_func = lambda request: request
request = OmniDiffusionRequest(
prompt="too large",
request_id="too-large",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
diffusion_kv_requests=tuple(
DiffusionKVRequest(
f"too-large/diffusion-kv/{sequence_id}",
sequence_id=sequence_id,
prefix_len=0,
target_len=target_len,
seq_len=seq_len,
)
for sequence_id in range(num_sequences)
),
)
with pytest.raises(ValueError, match="exceeds the startup memory-profile envelope"):
engine._prepare_request_for_admission(request)
def test_paged_kv_admission_accepts_shape_at_profile_envelope() -> None:
engine = object.__new__(DiffusionEngine)
engine._diffusion_kv_profile_limits = (2, 16, 8)
engine.pre_process_func = lambda request: request
request = OmniDiffusionRequest(
prompt="fits",
request_id="fits",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
diffusion_kv_requests=tuple(
DiffusionKVRequest(
f"fits/diffusion-kv/{sequence_id}",
sequence_id=sequence_id,
prefix_len=8,
target_len=8,
seq_len=16,
)
for sequence_id in range(2)
),
)
assert engine._prepare_request_for_admission(request) is request
@@ -473,6 +473,38 @@ async def test_cancelled_future_does_not_kill_busy_loop():
_stop_engine(engine)
@pytest.mark.asyncio
async def test_native_kv_reservation_error_wakes_stream_without_killing_busy_loop():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
def fail_reservation(*args, **kwargs):
raise RuntimeError("native allocation bug")
engine.scheduler._diffusion_kv_manager = SimpleNamespace(
has_request=lambda request_id: False,
reserve_request=fail_reservation,
free_request=lambda request_id: None,
)
request = _make_request("kv-error")
request.diffusion_kv_requests = (object(),)
try:
response_stream = engine.async_add_req_and_stream_response(request)
output_queue = engine._out_streams[request.request_id]
for _ in range(300):
if not output_queue.empty():
break
await asyncio.sleep(0.01)
assert not output_queue.empty(), "terminal KV allocation error was not delivered to the request stream"
output = await anext(response_stream)
assert output.error == "native allocation bug"
assert output.finished
assert engine.worker_thread.is_alive()
finally:
_stop_engine(engine)
# ─────────────────────────── _RpcTask basics ───────────────────────────────
@@ -3,6 +3,7 @@
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
@@ -512,6 +513,84 @@ def test_execute_model_accepts_bare_diffusion_output_from_single_request_pipelin
assert runner.pipeline.last_req.num_reqs == 1
@pytest.mark.core_model
@pytest.mark.cpu
def test_profile_run_executes_forward_without_scheduler_kv_validation(monkeypatch):
runner = _make_runner(cache_backend=None, cache_backend_name="none")
request = _make_request()
runner._validate_diffusion_kv_metadata = Mock(side_effect=AssertionError("profile must bypass admission"))
record_names = []
original_execute = runner._execute_request_list
def execute(*args, **kwargs):
record_names.append(kwargs["record_name"])
return original_execute(*args, **kwargs)
runner._execute_request_list = execute
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
reset_peak_memory_stats = Mock()
monkeypatch.setattr(
model_runner_module.current_omni_platform,
"reset_peak_memory_stats",
reset_peak_memory_stats,
)
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
synchronize = Mock()
monkeypatch.setattr(model_runner_module.current_omni_platform, "synchronize", synchronize)
runner.profile_run([request])
assert runner.pipeline.forward_calls == 1
assert record_names == ["pipeline_memory_profile"]
runner._validate_diffusion_kv_metadata.assert_not_called()
reset_peak_memory_stats.assert_not_called()
synchronize.assert_called_once_with()
@pytest.mark.core_model
@pytest.mark.cpu
def test_profile_run_executes_maximum_step_batch_without_resetting_peak(monkeypatch):
runner = _make_runner(cache_backend=None, cache_backend_name=None)
runner.pipeline = _FinalOnlyStepPipeline()
runner.od_config.step_execution = True
requests = [_make_request(), _make_request()]
requests[0].request_id = "profile-0"
requests[1].request_id = "profile-1"
for request in requests:
request.sampling_params.num_inference_steps = 1
observed_batch_rows = []
original_denoise_step = runner.pipeline.denoise_step
def denoise_step(input_batch, states):
observed_batch_rows.append((len(states), int(input_batch.latents.shape[0])))
return original_denoise_step(input_batch, states)
runner.pipeline.denoise_step = denoise_step
runner._validate_diffusion_kv_metadata = Mock(side_effect=AssertionError("profile must bypass admission"))
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
reset_peak_memory_stats = Mock()
monkeypatch.setattr(
model_runner_module.current_omni_platform,
"reset_peak_memory_stats",
reset_peak_memory_stats,
)
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
synchronize = Mock()
monkeypatch.setattr(model_runner_module.current_omni_platform, "synchronize", synchronize)
runner.profile_run(requests)
assert observed_batch_rows == [(2, 2)]
assert runner.state_cache == {}
assert runner.input_batch is None
runner._validate_diffusion_kv_metadata.assert_not_called()
reset_peak_memory_stats.assert_not_called()
synchronize.assert_called_once_with()
class _BatchPipeline:
"""Pipeline returning a configurable list of outputs from forward()."""
@@ -52,6 +52,7 @@ def test_dummy_run_request_is_identified_by_reserved_request_id():
assert req.is_dummy_run()
assert OmniDiffusionRequest.is_dummy_run_request_id(DUMMY_DIFFUSION_REQUEST_ID)
assert OmniDiffusionRequest.is_dummy_run_request_id(f"{DUMMY_DIFFUSION_REQUEST_ID}/kv-profile-1")
def test_non_dummy_request_is_not_identified_as_dummy_run():
+274 -7
View File
@@ -8,12 +8,14 @@ from types import SimpleNamespace
import pytest
import torch
import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers
from pytest_mock import MockerFixture
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
from vllm_omni.diffusion.data import DiffusionOutput, DiffusionRequestAbortedError
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVContext, DiffusionKVRequest
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.sched import (
BaseScheduler,
@@ -87,6 +89,52 @@ def _cached_ids(sched_output) -> list[str]:
return list(sched_output.scheduled_cached_reqs.request_ids)
def _initialize_paged_scheduler(
scheduler: BaseScheduler,
*,
num_blocks: int = 64,
max_num_seqs: int = 1,
) -> None:
native_kv_managers.register_all_kvcache_specs(None)
spec = FullAttentionSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
)
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=["layer0"])],
kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)],
)
scheduler.initialize(
SimpleNamespace(
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
max_model_len=64,
max_num_seqs=max_num_seqs,
),
kv_cache_config=config,
scheduler_block_size=4,
hash_block_size=4,
kv_vllm_config=SimpleNamespace(
model_config=SimpleNamespace(max_model_len=64),
max_in_flight_tokens=64,
),
)
def _attach_diffusion_kv(request: OmniDiffusionRequest, *, seq_len: int = 8) -> None:
request.diffusion_kv_requests = (
DiffusionKVRequest(
f"{request.request_id}/diffusion-kv/0",
sequence_id=0,
prefix_len=4,
target_len=4,
seq_len=seq_len,
),
)
class _StubScheduler:
def __init__(self, request: OmniDiffusionRequest, output) -> None:
self._request = request
@@ -400,18 +448,16 @@ class TestRequestScheduler:
assert self.scheduler.has_requests() is False
def test_diffusion_kv_request_moves_to_scheduler_state(self) -> None:
self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER))
_initialize_paged_scheduler(self.scheduler)
request = _make_request("diffusion-kv")
prepared_layout = object()
request.prepared_layout = prepared_layout
text_context = DiffusionKVContext(context_id="text", cache_role="cross.text", num_tokens=8)
kv_request = DiffusionKVRequest(
"diffusion-kv/diffusion-kv/0",
sequence_id=0,
prefix_len=4,
target_len=8,
seq_len=16,
kv_contexts=(text_context,),
)
request.diffusion_kv_requests = (kv_request,)
@@ -421,12 +467,179 @@ class TestRequestScheduler:
assert state is not None
assert state.diffusion_kv_requests == (kv_request,)
assert state.diffusion_kv_requests[0].kv_contexts == (text_context,)
assert request.diffusion_kv_requests is None
assert scheduler_output.scheduled_new_reqs[0].req.prepared_layout is prepared_layout
metadata = scheduler_output.scheduled_new_reqs[0].diffusion_kv_metadata
assert metadata is not None
assert metadata.request_id == request.request_id
assert len(metadata.sequences[0].block_ids[0]) == 4
def test_diffusion_kv_capacity_backpressures_fifo_until_blocks_are_freed(self) -> None:
_initialize_paged_scheduler(self.scheduler, num_blocks=3, max_num_seqs=2)
first_request = _make_request("first")
second_request = _make_request("second")
_attach_diffusion_kv(first_request)
_attach_diffusion_kv(second_request)
self.scheduler.add_request(first_request)
self.scheduler.add_request(second_request)
first_output = self.scheduler.schedule()
assert _new_ids(first_output) == ["first"]
assert first_output.num_waiting_reqs == 1
self.scheduler.update_from_output(first_output, _make_request_output("first"))
second_output = self.scheduler.schedule()
assert _new_ids(second_output) == ["second"]
assert second_output.num_waiting_reqs == 0
# The same output carries both the release and the newly admitted block
# table; the future Worker data plane must process finished ids first.
assert second_output.finished_req_ids == {"first"}
assert second_output.scheduled_new_reqs[0].diffusion_kv_metadata is not None
def test_impossible_diffusion_kv_capacity_finishes_only_that_request(self) -> None:
_initialize_paged_scheduler(self.scheduler, num_blocks=2, max_num_seqs=2)
impossible = _make_request("impossible")
schedulable = _make_request("schedulable")
_attach_diffusion_kv(impossible, seq_len=8)
schedulable.diffusion_kv_requests = (
DiffusionKVRequest(
"schedulable/diffusion-kv/0",
sequence_id=0,
prefix_len=0,
target_len=4,
seq_len=4,
),
)
self.scheduler.add_request(impossible)
self.scheduler.add_request(schedulable)
sched_output = self.scheduler.schedule()
failed_state = self.scheduler.get_request_state("impossible")
assert failed_state is not None
assert failed_state.status == DiffusionRequestStatus.FINISHED_ERROR
assert failed_state.error is not None
assert "cannot fit even when the block pool is empty" in failed_state.error
assert sched_output.finished_req_ids == {"impossible"}
assert _new_ids(sched_output) == ["schedulable"]
finished = self.scheduler.update_from_output(
sched_output,
_make_request_output("schedulable"),
)
assert finished == {"impossible", "schedulable"}
def test_impossible_diffusion_kv_capacity_does_not_block_waiters_under_load(self) -> None:
_initialize_paged_scheduler(self.scheduler, num_blocks=4, max_num_seqs=2)
running = _make_request("running")
_attach_diffusion_kv(running)
self.scheduler.add_request(running)
assert _new_ids(self.scheduler.schedule()) == ["running"]
impossible = _make_request("impossible")
impossible.diffusion_kv_requests = tuple(
DiffusionKVRequest(
f"impossible/diffusion-kv/{sequence_id}",
sequence_id=sequence_id,
prefix_len=0,
target_len=4,
seq_len=4,
)
for sequence_id in range(4)
)
schedulable = _make_request("schedulable")
schedulable.diffusion_kv_requests = (
DiffusionKVRequest(
"schedulable/diffusion-kv/0",
sequence_id=0,
prefix_len=0,
target_len=4,
seq_len=4,
),
)
self.scheduler.add_request(impossible)
self.scheduler.add_request(schedulable)
sched_output = self.scheduler.schedule()
failed_state = self.scheduler.get_request_state("impossible")
assert failed_state is not None
assert failed_state.status == DiffusionRequestStatus.FINISHED_ERROR
assert failed_state.error is not None
assert "required_blocks=4, available_blocks=3" in failed_state.error
assert sched_output.finished_req_ids == {"impossible"}
assert _new_ids(sched_output) == ["schedulable"]
def test_diffusion_kv_internal_allocation_error_finishes_request(self, monkeypatch) -> None:
_initialize_paged_scheduler(self.scheduler)
request = _make_request("native-error")
_attach_diffusion_kv(request)
self.scheduler.add_request(request)
manager = self.scheduler._diffusion_kv_manager
assert manager is not None
def raise_native_error(*args, **kwargs):
raise ValueError("native allocation bug")
monkeypatch.setattr(
manager,
"reserve_request",
raise_native_error,
)
sched_output = self.scheduler.schedule()
state = self.scheduler.get_request_state("native-error")
assert state is not None
assert state.status == DiffusionRequestStatus.FINISHED_ERROR
assert state.error == "native allocation bug"
assert sched_output.finished_req_ids == {"native-error"}
assert sched_output.scheduled_request_ids == []
def test_diffusion_kv_preemption_retains_allocation(self) -> None:
_initialize_paged_scheduler(self.scheduler, num_blocks=3)
request = _make_request("preempted")
_attach_diffusion_kv(request)
self.scheduler.add_request(request)
self.scheduler.schedule()
manager = self.scheduler._diffusion_kv_manager
assert manager is not None
metadata = manager.get_metadata("preempted")
assert self.scheduler.preempt_request("preempted") is True
resumed_output = self.scheduler.schedule()
assert resumed_output.scheduled_new_reqs == []
assert _cached_ids(resumed_output) == ["preempted"]
assert manager.get_metadata("preempted") == metadata
@pytest.mark.parametrize(
"status",
[
DiffusionRequestStatus.FINISHED_COMPLETED,
DiffusionRequestStatus.FINISHED_ABORTED,
DiffusionRequestStatus.FINISHED_ERROR,
],
)
def test_diffusion_kv_terminal_status_releases_allocation(self, status: DiffusionRequestStatus) -> None:
_initialize_paged_scheduler(self.scheduler, num_blocks=3)
request = _make_request("terminal")
_attach_diffusion_kv(request)
self.scheduler.add_request(request)
self.scheduler.schedule()
manager = self.scheduler._diffusion_kv_manager
assert manager is not None
assert manager.has_request("terminal") is True
self.scheduler.finish_requests("terminal", status)
assert manager.has_request("terminal") is False
def test_paged_scheduler_rejects_missing_kv_request(self) -> None:
self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER))
_initialize_paged_scheduler(self.scheduler)
request = _make_request("diffusion-kv")
with pytest.raises(ValueError, match="did not produce DiffusionKVRequest"):
@@ -443,7 +656,7 @@ class TestRequestScheduler:
],
)
def test_diffusion_kv_request_rejects_legacy_dense_kv(self, owner_name: str, field_name: str) -> None:
self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER))
_initialize_paged_scheduler(self.scheduler)
request = _make_request("diffusion-kv")
owner = request if owner_name == "request" else request.sampling_params
setattr(owner, field_name, object())
@@ -921,6 +1134,45 @@ class TestDiffusionEngine:
assert engine.scheduler.max_num_running_reqs == 1
fake_executor_cls.assert_called_once_with(od_config)
def test_initializes_paged_scheduler_from_native_control_plane(
self,
monkeypatch: pytest.MonkeyPatch,
mocker: MockerFixture,
) -> None:
od_config = SimpleNamespace(model_class_name="mock_model", streaming_output=False)
fake_executor_cls = mocker.Mock(return_value=mocker.Mock())
scheduler = mocker.Mock()
kv_cache_config = object()
kv_vllm_config = object()
monkeypatch.setattr(
"vllm_omni.diffusion.diffusion_engine.get_diffusion_post_process_func",
lambda *args, **kwargs: None,
)
monkeypatch.setattr(
"vllm_omni.diffusion.diffusion_engine.get_diffusion_pre_process_func",
lambda *args, **kwargs: None,
)
monkeypatch.setattr(
"vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class",
lambda *args, **kwargs: fake_executor_cls,
)
monkeypatch.setattr(
"vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane",
lambda *args, **kwargs: (kv_cache_config, 16, 16, kv_vllm_config),
)
engine = DiffusionEngine(od_config, scheduler=scheduler)
assert engine.scheduler is scheduler
scheduler.initialize.assert_called_once_with(
od_config,
kv_cache_config=kv_cache_config,
scheduler_block_size=16,
hash_block_size=16,
kv_vllm_config=kv_vllm_config,
)
def test_scheduler_alias_keeps_default_request_scheduler(self) -> None:
scheduler = Scheduler()
scheduler.initialize(SimpleNamespace())
@@ -987,6 +1239,21 @@ class TestDiffusionEngine:
assert output.aborted is True
assert output.abort_message == "Request req-finalize aborted."
def test_finalize_finished_request_returns_scheduler_admission_error(self) -> None:
engine = DiffusionEngine.__new__(DiffusionEngine)
engine.scheduler = StepScheduler()
engine.scheduler.initialize(SimpleNamespace())
req_id = engine.scheduler.add_request(_make_request("req-error"))
engine.scheduler._finish_requests(
{req_id: DiffusionRequestStatus.FINISHED_ERROR},
{req_id: "KV request cannot fit"},
)
output = engine._finalize_finished_request(req_id)
assert output.error == "KV request cannot fit"
@pytest.mark.asyncio
async def test_streaming_runner_output_notifies_each_chunk(self) -> None:
engine = DiffusionEngine.__new__(DiffusionEngine)
+1 -79
View File
@@ -13,11 +13,7 @@ import pytest
import torch
from pytest_mock import MockerFixture
from vllm_omni.diffusion.worker.diffusion_worker import (
DiffusionWorker,
_create_diffusion_worker_vllm_config,
_make_diffusion_vllm_model_config,
)
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.gpu]
@@ -58,34 +54,6 @@ def mock_gpu_worker(mocker: MockerFixture, mock_od_config):
return worker
def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods():
from types import SimpleNamespace
from vllm_omni.quantization import build_quant_config
od_config = SimpleNamespace(
model="dummy",
dtype=torch.bfloat16,
quantization_config=build_quant_config(
{
"quant_method": "modelopt",
"quant_algo": "FP8",
"ignore": [],
}
),
tf_model_config=SimpleNamespace(),
enforce_eager=True,
is_moe=False,
)
model_config = _make_diffusion_vllm_model_config(od_config)
assert model_config.dtype is torch.bfloat16
assert model_config.quantization == "modelopt"
assert model_config.quantization_config is od_config.quantization_config
assert model_config.is_quantized()
class TestDiffusionWorkerSleep:
"""Test DiffusionWorker.sleep method."""
@@ -329,49 +297,3 @@ class TestDiffusionWorkerWakeUp:
mock_buffer2.data.copy_.assert_not_called()
assert result is True
class TestWorkerVllmConfigAdditionalConfig:
"""Test worker-side VllmConfig construction with additional_config."""
def test_create_diffusion_worker_vllm_config_passes_additional_config(self, mocker: MockerFixture):
mock_vllm_config = mocker.Mock()
mock_vllm_cls = mocker.patch(
"vllm_omni.diffusion.worker.diffusion_worker.VllmConfig",
return_value=mock_vllm_config,
)
od_config = mocker.Mock(additional_config={"torchair_graph_config": {"enabled": True}})
result = _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config)
assert result is mock_vllm_config
assert mock_vllm_cls.call_args.kwargs["additional_config"] == od_config.additional_config
def test_create_diffusion_worker_vllm_config_falls_back_when_constructor_rejects_additional_config(
self, mocker: MockerFixture
):
mock_vllm_config = mocker.Mock()
mock_vllm_cls = mocker.patch(
"vllm_omni.diffusion.worker.diffusion_worker.VllmConfig",
side_effect=[
TypeError("VllmConfig.__init__() got an unexpected keyword argument 'additional_config'"),
mock_vllm_config,
],
)
od_config = mocker.Mock(additional_config={"ascend_scheduler_config": {"foo": "bar"}})
result = _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config)
assert result is mock_vllm_config
assert mock_vllm_cls.call_count == 2
assert getattr(mock_vllm_config, "additional_config") == od_config.additional_config
def test_create_diffusion_worker_vllm_config_reraises_other_type_errors(self, mocker: MockerFixture):
mocker.patch(
"vllm_omni.diffusion.worker.diffusion_worker.VllmConfig",
side_effect=TypeError("additional_config values must be hashable"),
)
od_config = mocker.Mock(additional_config={"ascend_scheduler_config": {"foo": "bar"}})
with pytest.raises(TypeError, match="hashable"):
_create_diffusion_worker_vllm_config(torch.device("cpu"), od_config)
@@ -413,6 +413,26 @@ class TestRequestModeDispatch:
assert [output.result.error for output in result.runner_outputs] == ["A", "B"]
executor.collective_rpc.assert_called_once()
def test_dlo_dp_forwards_request_metadata_envelopes_as_one_wave(self):
executor, _, _ = _make_executor(num_gpus=2)
executor.od_config = SimpleNamespace(
step_execution=False,
parallel_config=SimpleNamespace(data_parallel_size=2),
enable_distributed_layerwise_offload=True,
dlo_use_allgather=True,
)
executor.collective_rpc = Mock(return_value=[_tagged_output("A"), _tagged_output("B")])
scheduler_output = _make_sched_output("A", "B")
for new_req in scheduler_output.scheduled_new_reqs:
new_req.diffusion_kv_metadata = SimpleNamespace(request_id=new_req.request_id)
result = executor.execute_request(scheduler_output)
assert [output.result.error for output in result.runner_outputs] == ["A", "B"]
forwarded_envelopes = executor.collective_rpc.call_args.kwargs["args"][0]
assert forwarded_envelopes is scheduler_output.scheduled_new_reqs
assert [envelope.diffusion_kv_metadata.request_id for envelope in forwarded_envelopes] == ["A", "B"]
@pytest.mark.parametrize(
("field", "value"),
[