[Diffusion] Add native KV cache initialization and Scheduler-managed block allocation (#6094)
Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com> Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
This commit is contained in:
1 parent
b283215c28
commit
0a1846aa45
33 files changed
+2609
-265
No files matched your search
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
import vllm_omni.diffusion.diffusion_kv.initialization as initialization
|
||||
import vllm_omni.diffusion.vllm_config as diffusion_vllm_config
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler
|
||||
from vllm_omni.quantization import build_quant_config
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _Executor:
|
||||
def __init__(self) -> None:
|
||||
self.specs = [{"layer0": object()}, {"layer0": object()}]
|
||||
self.memory = [1024, 1024]
|
||||
self.configs = None
|
||||
self.profile_requests = None
|
||||
|
||||
def get_kv_cache_specs(self):
|
||||
return self.specs
|
||||
|
||||
def determine_available_kv_memory(self, profile_requests):
|
||||
self.profile_requests = profile_requests
|
||||
return self.memory
|
||||
|
||||
def set_kv_cache_configs(self, configs) -> None:
|
||||
self.configs = configs
|
||||
|
||||
|
||||
def _od_config(**overrides):
|
||||
values = dict(
|
||||
model="test-diffusion",
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
|
||||
dtype=torch.bfloat16,
|
||||
max_model_len=64,
|
||||
quantization_config=None,
|
||||
tf_model_config=None,
|
||||
enforce_eager=True,
|
||||
is_moe=False,
|
||||
additional_config={},
|
||||
profiler_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=False,
|
||||
),
|
||||
gpu_memory_utilization=0.9,
|
||||
kv_cache_memory_bytes=None,
|
||||
max_num_batched_tokens=64,
|
||||
num_gpus=1,
|
||||
)
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_control_plane_builds_worker_and_scheduler_configs(monkeypatch) -> None:
|
||||
executor = _Executor()
|
||||
od_config = _od_config(num_gpus=2)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=4096),
|
||||
max_in_flight_tokens=128,
|
||||
)
|
||||
worker_configs = [object(), object()]
|
||||
scheduler_kv_cache_config = object()
|
||||
profile_requests = [object()]
|
||||
monkeypatch.setattr(initialization, "create_diffusion_vllm_config", lambda *args, **kwargs: vllm_config)
|
||||
build = Mock(return_value=(worker_configs, scheduler_kv_cache_config))
|
||||
monkeypatch.setattr(
|
||||
initialization,
|
||||
"build_native_kv_cache_configs",
|
||||
build,
|
||||
)
|
||||
monkeypatch.setattr(initialization, "resolve_kv_cache_block_sizes", lambda *args: (16, 16))
|
||||
|
||||
result = initialization.initialize_diffusion_kv_control_plane(
|
||||
executor,
|
||||
od_config,
|
||||
profile_requests=profile_requests,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
build.assert_called_once_with(vllm_config, executor.specs, executor.memory)
|
||||
assert result == (scheduler_kv_cache_config, 16, 16, vllm_config)
|
||||
assert executor.configs == worker_configs
|
||||
assert executor.profile_requests is profile_requests
|
||||
|
||||
|
||||
def test_dense_mode_skips_worker_kv_initialization(monkeypatch) -> None:
|
||||
executor = _Executor()
|
||||
od_config = _od_config(diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY)
|
||||
get_device = Mock(side_effect=AssertionError("dense mode must not resolve a KV device"))
|
||||
monkeypatch.setattr(initialization.current_omni_platform, "get_torch_device", get_device)
|
||||
|
||||
result = initialization.initialize_diffusion_kv_control_plane(executor, od_config)
|
||||
|
||||
assert result is None
|
||||
assert executor.configs is None
|
||||
get_device.assert_not_called()
|
||||
|
||||
|
||||
def test_control_plane_rejects_rank_count_mismatch(monkeypatch) -> None:
|
||||
executor = _Executor()
|
||||
executor.memory = [1024]
|
||||
od_config = _od_config(num_gpus=2)
|
||||
monkeypatch.setattr(initialization, "create_diffusion_vllm_config", lambda *args, **kwargs: object())
|
||||
|
||||
with pytest.raises(ValueError, match="rank count mismatch"):
|
||||
initialization.initialize_diffusion_kv_control_plane(
|
||||
executor,
|
||||
od_config,
|
||||
profile_requests=[object()],
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def test_paged_control_plane_requires_prepared_profile_request() -> None:
|
||||
with pytest.raises(ValueError, match="prepared memory profile requests"):
|
||||
initialization.initialize_diffusion_kv_control_plane(
|
||||
_Executor(),
|
||||
_od_config(),
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def test_native_config_builder_produces_matching_worker_and_scheduler_pools() -> None:
|
||||
od_config = _od_config()
|
||||
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(torch.device("cpu"), od_config)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
non_causal=True,
|
||||
)
|
||||
num_blocks = 32
|
||||
|
||||
worker_configs, scheduler_kv_cache_config = initialization.build_native_kv_cache_configs(
|
||||
vllm_config,
|
||||
[{"layer0": spec}],
|
||||
[spec.page_size_bytes * num_blocks],
|
||||
)
|
||||
|
||||
assert len(worker_configs) == 1
|
||||
assert worker_configs[0].num_blocks == num_blocks
|
||||
assert scheduler_kv_cache_config.num_blocks == num_blocks
|
||||
assert worker_configs[0].kv_cache_groups == scheduler_kv_cache_config.kv_cache_groups
|
||||
|
||||
|
||||
def test_native_control_plane_config_initializes_scheduler() -> None:
|
||||
od_config = _od_config()
|
||||
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(torch.device("cpu"), od_config)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
non_causal=True,
|
||||
)
|
||||
_, scheduler_kv_cache_config = initialization.build_native_kv_cache_configs(
|
||||
vllm_config,
|
||||
[{"layer0": spec}],
|
||||
[spec.page_size_bytes * 32],
|
||||
)
|
||||
|
||||
scheduler = RequestScheduler()
|
||||
scheduler.initialize(
|
||||
od_config,
|
||||
kv_cache_config=scheduler_kv_cache_config,
|
||||
scheduler_block_size=spec.block_size,
|
||||
hash_block_size=spec.block_size,
|
||||
kv_vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
assert scheduler._diffusion_kv_manager is not None
|
||||
scheduler.close()
|
||||
|
||||
|
||||
def test_dense_config_keeps_new_paged_sizing_inputs_inactive() -> None:
|
||||
od_config = _od_config(
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY,
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=4,
|
||||
data_parallel_size=1,
|
||||
enable_expert_parallel=False,
|
||||
),
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_cache_memory_bytes=4096,
|
||||
max_num_batched_tokens=64,
|
||||
)
|
||||
vllm_config = diffusion_vllm_config.create_base_diffusion_vllm_config(torch.device("cpu"), od_config)
|
||||
original_pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
original_memory_utilization = vllm_config.cache_config.gpu_memory_utilization
|
||||
original_memory_bytes = vllm_config.cache_config.kv_cache_memory_bytes
|
||||
original_token_budget = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
diffusion_vllm_config.configure_diffusion_vllm_config(vllm_config, od_config)
|
||||
|
||||
assert vllm_config.parallel_config.pipeline_parallel_size == original_pp_size
|
||||
assert vllm_config.cache_config.gpu_memory_utilization == original_memory_utilization
|
||||
assert vllm_config.cache_config.kv_cache_memory_bytes == original_memory_bytes
|
||||
assert vllm_config.scheduler_config.max_num_batched_tokens == original_token_budget
|
||||
|
||||
|
||||
def test_paged_config_forwards_gpu_memory_utilization_to_native_cache_config() -> None:
|
||||
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(
|
||||
torch.device("cpu"),
|
||||
_od_config(gpu_memory_utilization=0.42),
|
||||
)
|
||||
|
||||
assert vllm_config.cache_config.gpu_memory_utilization == 0.42
|
||||
|
||||
|
||||
def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods() -> None:
|
||||
quantization_config = build_quant_config(
|
||||
{
|
||||
"quant_method": "modelopt",
|
||||
"quant_algo": "FP8",
|
||||
"ignore": [],
|
||||
}
|
||||
)
|
||||
model_config = diffusion_vllm_config._make_diffusion_vllm_model_config(
|
||||
_od_config(
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY,
|
||||
quantization_config=quantization_config,
|
||||
)
|
||||
)
|
||||
|
||||
assert model_config.dtype is torch.bfloat16
|
||||
assert model_config.quantization == "modelopt"
|
||||
assert model_config.quantization_config is quantization_config
|
||||
assert model_config.is_quantized
|
||||
assert model_config.is_moe is False
|
||||
assert model_config.is_nvfp4_quantized() is False
|
||||
assert model_config.original_max_model_len == model_config.max_model_len
|
||||
|
||||
unquantized_model_config = diffusion_vllm_config._make_diffusion_vllm_model_config(
|
||||
_od_config(
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY,
|
||||
quantization_config=None,
|
||||
)
|
||||
)
|
||||
assert not unquantized_model_config.is_quantized
|
||||
|
||||
|
||||
def test_minus_one_max_model_len_uses_model_limit() -> None:
|
||||
od_config = SimpleNamespace(
|
||||
max_model_len=-1,
|
||||
tf_model_config=SimpleNamespace(max_position_embeddings=4096),
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
|
||||
)
|
||||
|
||||
assert diffusion_vllm_config.resolve_diffusion_max_model_len(od_config) == 4096
|
||||
|
||||
model_config = diffusion_vllm_config._make_diffusion_vllm_model_config(
|
||||
_od_config(
|
||||
max_model_len=-1,
|
||||
tf_model_config=SimpleNamespace(max_position_embeddings=4096),
|
||||
)
|
||||
)
|
||||
assert model_config.max_model_len == 4096
|
||||
assert model_config.original_max_model_len == -1
|
||||
|
||||
|
||||
def test_minus_one_max_model_len_is_auto_fitted_by_native_cache_sizing() -> None:
|
||||
vllm_config = diffusion_vllm_config.create_diffusion_vllm_config(
|
||||
torch.device("cpu"),
|
||||
_od_config(
|
||||
max_model_len=-1,
|
||||
tf_model_config=SimpleNamespace(max_position_embeddings=4096),
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
non_causal=True,
|
||||
)
|
||||
|
||||
initialization.build_native_kv_cache_configs(
|
||||
vllm_config,
|
||||
[{"layer0": spec}],
|
||||
[spec.page_size_bytes * 16],
|
||||
)
|
||||
|
||||
assert vllm_config.model_config.original_max_model_len == -1
|
||||
assert vllm_config.model_config.max_model_len == 256
|
||||
@@ -0,0 +1,211 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
|
||||
|
||||
from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVAdmissionError, DiffusionKVCacheManager
|
||||
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVContext, DiffusionKVRequest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
BLOCK_SIZE = 4
|
||||
|
||||
|
||||
def _config(num_blocks: int) -> KVCacheConfig:
|
||||
native_kv_managers.register_all_kvcache_specs(None)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=["layer0"])],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)],
|
||||
)
|
||||
|
||||
|
||||
def _request(public_id: str, sequence_id: int, *, seq_len: int = 8, kv_contexts=()) -> DiffusionKVRequest:
|
||||
return DiffusionKVRequest(
|
||||
f"{public_id}/diffusion-kv/{sequence_id}",
|
||||
sequence_id=sequence_id,
|
||||
prefix_len=4,
|
||||
target_len=4,
|
||||
seq_len=seq_len,
|
||||
kv_contexts=kv_contexts,
|
||||
)
|
||||
|
||||
|
||||
def _manager(num_blocks: int, *, max_model_len: int = 64) -> DiffusionKVCacheManager:
|
||||
return DiffusionKVCacheManager(
|
||||
_config(num_blocks),
|
||||
max_model_len=max_model_len,
|
||||
scheduler_block_size=BLOCK_SIZE,
|
||||
hash_block_size=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_and_free_multi_cfg_request() -> None:
|
||||
manager = _manager(8)
|
||||
free_before = manager.native_manager.block_pool.get_num_free_blocks()
|
||||
requests = (_request("public", 0), _request("public", 1))
|
||||
|
||||
metadata = manager.reserve_request("public", requests)
|
||||
|
||||
assert metadata is not None
|
||||
assert metadata.request_id == "public"
|
||||
assert metadata.allocation_generation == 1
|
||||
assert [sequence.sequence_id for sequence in metadata.sequences] == [0, 1]
|
||||
assert [sequence.prefix_len for sequence in metadata.sequences] == [4, 4]
|
||||
assert [sequence.target_len for sequence in metadata.sequences] == [4, 4]
|
||||
assert [sequence.seq_len for sequence in metadata.sequences] == [8, 8]
|
||||
assert [len(sequence.block_ids[0]) for sequence in metadata.sequences] == [2, 2]
|
||||
assert all(sequence.context_ids == () for sequence in metadata.sequences)
|
||||
assert metadata.contexts == ()
|
||||
assert metadata == manager.get_metadata("public")
|
||||
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before - 4
|
||||
|
||||
manager.free_request("public")
|
||||
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
|
||||
|
||||
|
||||
def test_impossible_cfg_allocation_rolls_back_and_fails_fast() -> None:
|
||||
manager = _manager(3)
|
||||
free_before = manager.native_manager.block_pool.get_num_free_blocks()
|
||||
|
||||
with pytest.raises(DiffusionKVAdmissionError, match="cannot fit even when the block pool is empty"):
|
||||
manager.reserve_request(
|
||||
"public",
|
||||
(_request("public", 0), _request("public", 1)),
|
||||
)
|
||||
|
||||
assert manager.has_request("public") is False
|
||||
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
|
||||
|
||||
|
||||
def test_impossible_cfg_allocation_fails_fast_while_pool_is_busy() -> None:
|
||||
manager = _manager(5)
|
||||
assert manager.reserve_request("running", (_request("running", 0),)) is not None
|
||||
free_before = manager.native_manager.block_pool.get_num_free_blocks()
|
||||
|
||||
with pytest.raises(DiffusionKVAdmissionError, match="required_blocks=6, available_blocks=4"):
|
||||
manager.reserve_request(
|
||||
"impossible",
|
||||
tuple(_request("impossible", sequence_id) for sequence_id in range(3)),
|
||||
)
|
||||
|
||||
assert manager.has_request("impossible") is False
|
||||
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
|
||||
|
||||
|
||||
def test_temporary_capacity_pressure_returns_none() -> None:
|
||||
manager = _manager(3)
|
||||
assert manager.reserve_request("running", (_request("running", 0),)) is not None
|
||||
|
||||
allocation = manager.reserve_request("waiting", (_request("waiting", 0),))
|
||||
|
||||
assert allocation is None
|
||||
assert manager.has_request("waiting") is False
|
||||
|
||||
|
||||
def test_partial_cfg_rollback_can_retry_after_capacity_is_released() -> None:
|
||||
manager = _manager(5)
|
||||
assert manager.reserve_request("running", (_request("running", 0),)) is not None
|
||||
waiting = (_request("waiting", 0), _request("waiting", 1))
|
||||
|
||||
assert manager.reserve_request("waiting", waiting) is None
|
||||
assert manager.has_request("waiting") is False
|
||||
|
||||
manager.free_request("running")
|
||||
metadata = manager.reserve_request("waiting", waiting)
|
||||
|
||||
assert metadata is not None
|
||||
assert [len(sequence.block_ids[0]) for sequence in metadata.sequences] == [2, 2]
|
||||
|
||||
|
||||
def test_reallocated_request_gets_a_new_allocation_generation() -> None:
|
||||
manager = _manager(3)
|
||||
request = (_request("public", 0),)
|
||||
|
||||
first = manager.reserve_request("public", request)
|
||||
assert first is not None
|
||||
manager.free_request("public")
|
||||
second = manager.reserve_request("public", request)
|
||||
|
||||
assert second is not None
|
||||
assert second.allocation_generation == first.allocation_generation + 1
|
||||
|
||||
|
||||
def test_cfg_allocation_exception_rolls_back(monkeypatch) -> None:
|
||||
manager = _manager(8)
|
||||
free_before = manager.native_manager.block_pool.get_num_free_blocks()
|
||||
native_allocate = manager.native_manager.allocate_slots
|
||||
num_calls = 0
|
||||
|
||||
def fail_second_allocation(*args, **kwargs):
|
||||
nonlocal num_calls
|
||||
num_calls += 1
|
||||
if num_calls == 2:
|
||||
raise RuntimeError("injected allocation failure")
|
||||
return native_allocate(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(manager.native_manager, "allocate_slots", fail_second_allocation)
|
||||
|
||||
with pytest.raises(RuntimeError, match="injected allocation failure"):
|
||||
manager.reserve_request(
|
||||
"public",
|
||||
(_request("public", 0), _request("public", 1)),
|
||||
)
|
||||
|
||||
assert manager.has_request("public") is False
|
||||
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
|
||||
|
||||
|
||||
def test_rejects_independent_context_until_role_routing_is_implemented() -> None:
|
||||
manager = _manager(8)
|
||||
context = DiffusionKVContext(context_id="text", cache_role="cross.text", num_tokens=8)
|
||||
|
||||
with pytest.raises(DiffusionKVAdmissionError, match="DiffusionKVContext"):
|
||||
manager.reserve_request(
|
||||
"public",
|
||||
(_request("public", 0, kv_contexts=(context,)),),
|
||||
)
|
||||
|
||||
|
||||
def test_rejects_sequence_longer_than_admission_bound() -> None:
|
||||
manager = _manager(8, max_model_len=8)
|
||||
|
||||
with pytest.raises(DiffusionKVAdmissionError, match="exceeds max_model_len"):
|
||||
manager.reserve_request("public", (_request("public", 0, seq_len=9),))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"requests",
|
||||
[
|
||||
(_request("public", 1),),
|
||||
(_request("public", 1), _request("public", 0)),
|
||||
(_request("public", 0), _request("public", 2)),
|
||||
],
|
||||
)
|
||||
def test_rejects_ambiguous_sequence_order(requests) -> None:
|
||||
manager = _manager(8)
|
||||
|
||||
with pytest.raises(ValueError, match="ordered by contiguous sequence_id"):
|
||||
manager.reserve_request("public", requests)
|
||||
|
||||
|
||||
def test_close_releases_every_public_request() -> None:
|
||||
manager = _manager(8)
|
||||
free_before = manager.native_manager.block_pool.get_num_free_blocks()
|
||||
assert manager.reserve_request("first", (_request("first", 0),)) is not None
|
||||
assert manager.reserve_request("second", (_request("second", 0),)) is not None
|
||||
|
||||
manager.close()
|
||||
|
||||
assert manager.native_manager.block_pool.get_num_free_blocks() == free_before
|
||||
assert manager.has_request("first") is False
|
||||
assert manager.has_request("second") is False
|
||||
@@ -107,11 +107,8 @@ def test_request_rejects_non_context_values() -> None:
|
||||
def _manager(*, enable_caching: bool) -> KVCacheManager:
|
||||
# vLLM 0.26 lazily registers its built-in KV specs. Another test may have
|
||||
# already registered an out-of-tree spec, making the registry non-empty
|
||||
# before FullAttentionSpec is registered. Initialize the native mappings
|
||||
# explicitly; vLLM 0.25 has no registration hook and needs no setup.
|
||||
register_all = getattr(native_kv_managers, "register_all_kvcache_specs", None)
|
||||
if register_all is not None:
|
||||
register_all(None)
|
||||
# before FullAttentionSpec is registered. Initialize the native mappings.
|
||||
native_kv_managers.register_all_kvcache_specs(None)
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
|
||||
|
||||
import vllm_omni.diffusion.worker.diffusion_worker as diffusion_worker_module
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _Backend:
|
||||
@classmethod
|
||||
def indexes_kv_by_block_stride(cls) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _attention(*, enabled: bool) -> Attention:
|
||||
attention = Attention.__new__(Attention)
|
||||
nn.Module.__init__(attention)
|
||||
attention.paged_kv_cache_role = "primary" if enabled else None
|
||||
attention.paged_kv_cache_dtype = torch.bfloat16
|
||||
attention.num_kv_heads = 2
|
||||
attention.head_size = 8
|
||||
attention.causal = False
|
||||
attention.attn_backend = _Backend
|
||||
return attention
|
||||
|
||||
|
||||
def _runner(attention: Attention) -> DiffusionModelRunner:
|
||||
runner = object.__new__(DiffusionModelRunner)
|
||||
runner.od_config = SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER)
|
||||
runner.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=4),
|
||||
model_config=SimpleNamespace(dtype=torch.float16),
|
||||
)
|
||||
pipeline = nn.Module()
|
||||
pipeline.image_attention = attention
|
||||
runner.pipeline = pipeline
|
||||
return runner
|
||||
|
||||
|
||||
def test_runner_discovers_native_spec_from_loaded_attention() -> None:
|
||||
specs = _runner(_attention(enabled=True)).get_kv_cache_spec()
|
||||
|
||||
assert set(specs) == {"image_attention"}
|
||||
spec = specs["image_attention"]
|
||||
assert isinstance(spec, FullAttentionSpec)
|
||||
assert spec.block_size == 4
|
||||
assert spec.num_kv_heads == 2
|
||||
assert spec.head_size == 8
|
||||
assert spec.dtype is torch.bfloat16
|
||||
assert spec.indexes_kv_by_block_stride is True
|
||||
assert spec.non_causal is True
|
||||
|
||||
|
||||
def test_runner_rejects_paged_mode_without_cache_enabled_attention() -> None:
|
||||
runner = _runner(_attention(enabled=False))
|
||||
|
||||
with pytest.raises(RuntimeError, match="no cache-enabled Attention"):
|
||||
runner.get_kv_cache_spec()
|
||||
|
||||
|
||||
def test_runner_retains_matching_rank_local_config() -> None:
|
||||
runner = _runner(_attention(enabled=True))
|
||||
spec = runner.get_kv_cache_spec()["image_attention"]
|
||||
config = KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * 8, shared_by=["image_attention"])],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layer_names=["image_attention"], kv_cache_spec=spec)],
|
||||
)
|
||||
|
||||
runner.set_kv_cache_config(config)
|
||||
|
||||
assert runner.kv_cache_config is config
|
||||
|
||||
|
||||
def test_runner_rejects_rank_local_config_for_different_layers() -> None:
|
||||
runner = _runner(_attention(enabled=True))
|
||||
spec = runner.get_kv_cache_spec()["image_attention"]
|
||||
config = KVCacheConfig(
|
||||
num_blocks=8,
|
||||
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * 8, shared_by=["other_attention"])],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layer_names=["other_attention"], kv_cache_spec=spec)],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="layer mismatch"):
|
||||
runner.set_kv_cache_config(config)
|
||||
|
||||
|
||||
def test_worker_selects_its_rank_local_config() -> None:
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 1
|
||||
worker.od_config = SimpleNamespace(num_gpus=2)
|
||||
worker.model_runner = SimpleNamespace(set_kv_cache_config=lambda config: setattr(worker, "installed", config))
|
||||
configs = [object(), object()]
|
||||
|
||||
worker.set_kv_cache_configs(configs)
|
||||
|
||||
assert worker.installed is configs[1]
|
||||
with pytest.raises(ValueError, match="rank count mismatch"):
|
||||
worker.set_kv_cache_configs(configs[:1])
|
||||
|
||||
|
||||
def test_worker_honors_explicit_kv_memory_budget(monkeypatch) -> None:
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.vllm_config = SimpleNamespace(cache_config=SimpleNamespace(kv_cache_memory_bytes=4096))
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=16384)
|
||||
worker.requested_memory = 8192
|
||||
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=1024)
|
||||
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
|
||||
profile_request = object()
|
||||
|
||||
assert worker.determine_available_kv_memory(profile_request) == [4096]
|
||||
worker.model_runner.profile_run.assert_called_once_with(profile_request)
|
||||
|
||||
|
||||
def test_worker_treats_zero_kv_memory_budget_as_profiled_auto_sizing(monkeypatch) -> None:
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=900)
|
||||
worker.requested_memory = 750
|
||||
worker.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
kv_cache_memory_bytes=0,
|
||||
gpu_memory_utilization=0.75,
|
||||
)
|
||||
)
|
||||
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=100)
|
||||
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
|
||||
profile_result = SimpleNamespace(
|
||||
non_kv_cache_memory=400,
|
||||
after_profile=SimpleNamespace(free_memory=500),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
diffusion_worker_module,
|
||||
"memory_profiling",
|
||||
lambda snapshot, weights_memory: nullcontext(profile_result),
|
||||
)
|
||||
profile_request = object()
|
||||
|
||||
assert worker.determine_available_kv_memory(profile_request) == [350]
|
||||
worker.model_runner.profile_run.assert_called_once_with(profile_request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("non_kv_cache_memory", [750, 800])
|
||||
def test_worker_rejects_non_positive_profiled_kv_memory(monkeypatch, non_kv_cache_memory: int) -> None:
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=900)
|
||||
worker.requested_memory = 750
|
||||
worker.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
kv_cache_memory_bytes=None,
|
||||
gpu_memory_utilization=0.75,
|
||||
)
|
||||
)
|
||||
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=100)
|
||||
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
|
||||
profile_result = SimpleNamespace(
|
||||
non_kv_cache_memory=non_kv_cache_memory,
|
||||
after_profile=SimpleNamespace(free_memory=100),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
diffusion_worker_module,
|
||||
"memory_profiling",
|
||||
lambda snapshot, weights_memory: nullcontext(profile_result),
|
||||
)
|
||||
profile_request = object()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match=(
|
||||
"rank 0: RuntimeError: No memory remains for Diffusion KV cache after profiling: "
|
||||
f"requested_memory=750 bytes, non_kv_cache_memory={non_kv_cache_memory} bytes"
|
||||
),
|
||||
):
|
||||
worker.determine_available_kv_memory(profile_request)
|
||||
|
||||
worker.model_runner.profile_run.assert_called_once_with(profile_request)
|
||||
|
||||
|
||||
def test_worker_profiles_activation_headroom_instead_of_current_residency(monkeypatch) -> None:
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.rank = 0
|
||||
worker.init_snapshot = SimpleNamespace(free_memory=1200)
|
||||
worker.requested_memory = 1000
|
||||
worker.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
kv_cache_memory_bytes=None,
|
||||
gpu_memory_utilization=0.8,
|
||||
)
|
||||
)
|
||||
worker.model_runner = SimpleNamespace(profile_run=Mock(), model_memory_usage=250)
|
||||
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
|
||||
profile_result = SimpleNamespace(
|
||||
non_kv_cache_memory=650,
|
||||
after_profile=SimpleNamespace(free_memory=550),
|
||||
)
|
||||
profiling_calls = []
|
||||
|
||||
def profile(snapshot, *, weights_memory):
|
||||
profiling_calls.append((snapshot, weights_memory))
|
||||
return nullcontext(profile_result)
|
||||
|
||||
monkeypatch.setattr(
|
||||
diffusion_worker_module,
|
||||
"memory_profiling",
|
||||
profile,
|
||||
)
|
||||
profile_request = object()
|
||||
|
||||
assert worker.determine_available_kv_memory(profile_request) == [350]
|
||||
assert profiling_calls == [(worker.init_snapshot, 250)]
|
||||
worker.model_runner.profile_run.assert_called_once_with(profile_request)
|
||||
|
||||
|
||||
def test_worker_rejects_profile_without_preload_snapshot(monkeypatch) -> None:
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.init_snapshot = None
|
||||
worker.requested_memory = None
|
||||
worker.vllm_config = SimpleNamespace(cache_config=SimpleNamespace(kv_cache_memory_bytes=None))
|
||||
worker.model_runner = SimpleNamespace()
|
||||
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", lambda value: [value])
|
||||
|
||||
with pytest.raises(RuntimeError, match="snapshot was not captured before model loading"):
|
||||
worker.determine_available_kv_memory(object())
|
||||
|
||||
|
||||
def test_rank_probe_gathers_local_failure_before_raising(monkeypatch) -> None:
|
||||
gathered = []
|
||||
|
||||
def gather(local_result):
|
||||
gathered.append(local_result)
|
||||
return [local_result, (True, "peer-result")]
|
||||
|
||||
def fail_local_probe():
|
||||
raise ValueError("local failure")
|
||||
|
||||
monkeypatch.setattr(diffusion_worker_module, "_all_gather_rank_values", gather)
|
||||
|
||||
with pytest.raises(RuntimeError, match="rank 0: ValueError: local failure"):
|
||||
diffusion_worker_module._run_and_gather_rank_values(
|
||||
"test probe",
|
||||
fail_local_probe,
|
||||
)
|
||||
|
||||
assert gathered == [(False, "ValueError: local failure")]
|
||||
@@ -85,19 +85,15 @@ def test_new_request_data_carries_scheduler_allocation_atomically() -> None:
|
||||
assert new_req.diffusion_kv_metadata is metadata
|
||||
|
||||
|
||||
def test_paged_request_without_metadata_reaches_request_forward() -> None:
|
||||
def test_paged_request_without_metadata_fails_before_request_forward() -> None:
|
||||
runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER)
|
||||
expected_output = DiffusionOutput(output=None)
|
||||
runner._execute_request_list = Mock(
|
||||
return_value=SimpleNamespace(
|
||||
runner_outputs=[SimpleNamespace(result=expected_output)],
|
||||
)
|
||||
)
|
||||
runner._execute_request_list = Mock()
|
||||
req = SimpleNamespace(request_id="req-0")
|
||||
|
||||
assert runner.execute_model(req) is expected_output
|
||||
with pytest.raises(ValueError, match="requires Diffusion KV metadata"):
|
||||
runner.execute_model(req)
|
||||
|
||||
runner._execute_request_list.assert_called_once()
|
||||
runner._execute_request_list.assert_not_called()
|
||||
|
||||
|
||||
def test_dense_request_rejects_metadata_before_request_forward() -> None:
|
||||
@@ -141,18 +137,18 @@ def test_request_rpc_rejects_envelope_request_identity_mismatch() -> None:
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_batch_path_allows_missing_metadata_before_forward() -> None:
|
||||
def test_batch_path_rejects_missing_metadata_before_forward() -> None:
|
||||
runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER)
|
||||
expected_output = object()
|
||||
runner._execute_request_list = Mock(return_value=expected_output)
|
||||
runner._execute_request_list = Mock()
|
||||
new_req = NewRequestData(
|
||||
request_id="req-0",
|
||||
req=SimpleNamespace(request_id="req-0"),
|
||||
)
|
||||
|
||||
assert runner.execute_model_batch(make_scheduler_output(new_req), runner.od_config) is expected_output
|
||||
with pytest.raises(ValueError, match="requires Diffusion KV metadata"):
|
||||
runner.execute_model_batch(make_scheduler_output(new_req), runner.od_config)
|
||||
|
||||
runner._execute_request_list.assert_called_once()
|
||||
runner._execute_request_list.assert_not_called()
|
||||
|
||||
|
||||
def test_batch_path_rejects_envelope_request_identity_mismatch() -> None:
|
||||
@@ -170,7 +166,7 @@ def test_batch_path_rejects_envelope_request_identity_mismatch() -> None:
|
||||
runner._execute_request_list.assert_not_called()
|
||||
|
||||
|
||||
def test_step_path_allows_missing_metadata_before_step_support_check() -> None:
|
||||
def test_step_path_rejects_missing_metadata_before_step_support_check() -> None:
|
||||
runner = make_runner(DiffusionKVCacheMode.PAGED_SCHEDULER)
|
||||
runner.pipeline = object()
|
||||
runner._supports_step_mode = Mock(return_value=False)
|
||||
@@ -179,10 +175,10 @@ def test_step_path_allows_missing_metadata_before_step_support_check() -> None:
|
||||
req=SimpleNamespace(request_id="req-0"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not support step execution"):
|
||||
with pytest.raises(ValueError, match="requires Diffusion KV metadata"):
|
||||
runner.execute_stepwise(make_scheduler_output(new_req))
|
||||
|
||||
runner._supports_step_mode.assert_called_once()
|
||||
runner._supports_step_mode.assert_not_called()
|
||||
|
||||
|
||||
def test_step_path_rejects_envelope_request_identity_mismatch() -> None:
|
||||
@@ -288,3 +284,42 @@ def test_dense_worker_call_does_not_extend_model_runner_signature() -> None:
|
||||
worker.execute_model(req, SimpleNamespace())
|
||||
|
||||
assert calls == [(req, None)]
|
||||
|
||||
|
||||
def test_dlo_worker_selects_request_and_metadata_from_same_envelope(monkeypatch) -> None:
|
||||
calls: list[tuple] = []
|
||||
|
||||
class ModelRunner:
|
||||
def execute_model(self, req, **kwargs):
|
||||
calls.append((req, kwargs))
|
||||
return DiffusionOutput(output=None)
|
||||
|
||||
worker = object.__new__(DiffusionWorker)
|
||||
worker.model_runner = ModelRunner()
|
||||
worker.lora_manager = None
|
||||
worker._get_profiler = lambda: None
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_data_parallel_rank",
|
||||
lambda: 1,
|
||||
)
|
||||
req_0 = SimpleNamespace(request_id="req-0")
|
||||
req_1 = SimpleNamespace(request_id="req-1")
|
||||
metadata_0 = make_metadata("req-0")
|
||||
metadata_1 = make_metadata("req-1")
|
||||
envelopes = [
|
||||
NewRequestData(request_id="req-0", req=req_0, diffusion_kv_metadata=metadata_0),
|
||||
NewRequestData(request_id="req-1", req=req_1, diffusion_kv_metadata=metadata_1),
|
||||
]
|
||||
|
||||
result = worker.execute_model(envelopes, SimpleNamespace())
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
req_1,
|
||||
{
|
||||
"kv_prefetch_job": None,
|
||||
"diffusion_kv_metadata": metadata_1,
|
||||
},
|
||||
)
|
||||
]
|
||||
assert result["dp_rank"] == 1
|
||||
@@ -118,6 +118,14 @@ def _call_mgr(
|
||||
)
|
||||
|
||||
|
||||
def test_cache_manager_registers_attention_without_adding_dense_state() -> None:
|
||||
mgr = _make_cache_mgr()
|
||||
|
||||
assert isinstance(mgr, nn.Module)
|
||||
assert dict(mgr.named_modules())["attn"] is mgr.attn
|
||||
assert mgr.state_dict() == {}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test 1: No AR KV — basic cache → reuse
|
||||
# ============================================================
|
||||
|
||||
@@ -130,6 +130,21 @@ class TestCreateDefaultDiffusion:
|
||||
|
||||
assert od.diffusion_kv_mode is DiffusionKVCacheMode.PAGED_SCHEDULER
|
||||
|
||||
def test_diffusion_kv_sizing_fields_roundtrip(self, monkeypatch):
|
||||
monkeypatch.setattr(OmniDiffusionConfig, "_resolve_master_port", lambda self: 29500)
|
||||
od = _roundtrip_diffusion_config(
|
||||
model="x",
|
||||
kv_cache_memory_bytes=4096,
|
||||
gpu_memory_utilization=0.75,
|
||||
max_num_batched_tokens=2048,
|
||||
max_model_len=4096,
|
||||
)
|
||||
|
||||
assert od.kv_cache_memory_bytes == 4096
|
||||
assert od.gpu_memory_utilization == 0.75
|
||||
assert od.max_num_batched_tokens == 2048
|
||||
assert od.max_model_len == 4096
|
||||
|
||||
|
||||
def test_qwen_image_edit_plus_sets_generic_multimodal_limit():
|
||||
od_config = OmniDiffusionConfig(model="Qwen/Qwen-Image-Edit-2511", model_class_name="QwenImageEditPlusPipeline")
|
||||
|
||||
@@ -161,9 +161,83 @@ def test_scheduler_initialization_failure_closes_scheduler_and_executor(monkeypa
|
||||
assert exc_info.value is initialization_error
|
||||
custom_scheduler.initialize.assert_called_once_with(od_config)
|
||||
custom_scheduler.close.assert_called_once_with()
|
||||
|
||||
|
||||
def test_init_shuts_down_executor_when_kv_control_plane_initialization_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
fake_executor = SimpleNamespace(shutdown=Mock())
|
||||
monkeypatch.setattr(DiffusionEngine, "_init_process_hooks", lambda self, config: None)
|
||||
monkeypatch.setattr(
|
||||
DiffusionEngine,
|
||||
"_resolve_execution_mode",
|
||||
lambda self, config: DiffusionExecutionMode.REQUEST_BATCH,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DiffusionEngine, "_init_executor", lambda self, config: setattr(self, "executor", fake_executor)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane",
|
||||
Mock(side_effect=RuntimeError("KV initialization failed")),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="KV initialization failed"):
|
||||
DiffusionEngine(SimpleNamespace())
|
||||
|
||||
fake_executor.shutdown.assert_called_once_with()
|
||||
|
||||
|
||||
def test_paged_init_profiles_before_scheduler_initialization(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
events = []
|
||||
expected_profile_requests = [object()]
|
||||
fake_executor = SimpleNamespace(shutdown=Mock())
|
||||
kv_cache_config = object()
|
||||
kv_vllm_config = object()
|
||||
od_config = SimpleNamespace(diffusion_kv_mode="paged_scheduler")
|
||||
|
||||
monkeypatch.setattr(DiffusionEngine, "_init_process_hooks", lambda self, config: None)
|
||||
monkeypatch.setattr(
|
||||
DiffusionEngine,
|
||||
"_resolve_execution_mode",
|
||||
lambda self, config: DiffusionExecutionMode.REQUEST_BATCH,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DiffusionEngine,
|
||||
"_init_executor",
|
||||
lambda self, config: setattr(self, "executor", fake_executor),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DiffusionEngine,
|
||||
"_prepare_diffusion_kv_profile_requests",
|
||||
lambda self: events.append("prepare-profile") or expected_profile_requests,
|
||||
)
|
||||
|
||||
def initialize(executor, config, *, profile_requests):
|
||||
events.append("profile-workers")
|
||||
assert executor is fake_executor
|
||||
assert config is od_config
|
||||
assert profile_requests is expected_profile_requests
|
||||
return kv_cache_config, 16, 16, kv_vllm_config
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane",
|
||||
initialize,
|
||||
)
|
||||
|
||||
def init_scheduler(self, config, scheduler, *args, **kwargs):
|
||||
events.append("initialize-scheduler")
|
||||
self.scheduler = SimpleNamespace(close=Mock())
|
||||
|
||||
monkeypatch.setattr(DiffusionEngine, "_init_scheduler", init_scheduler)
|
||||
monkeypatch.setattr(DiffusionEngine, "_init_runtime_state", lambda self: None)
|
||||
monkeypatch.setattr(DiffusionEngine, "_init_execute_fn", lambda self: None)
|
||||
monkeypatch.setattr(DiffusionEngine, "_log_execution_mode", lambda self, config: None)
|
||||
|
||||
DiffusionEngine(od_config)
|
||||
|
||||
assert events == ["prepare-profile", "profile-workers", "initialize-scheduler"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_step_compatibility_wrapper_returns_final_batch() -> None:
|
||||
engine = _make_engine()
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion import io_support
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
@@ -55,3 +63,151 @@ def test_dummy_run_num_frames_uses_audio_input_fallback(monkeypatch: pytest.Monk
|
||||
)
|
||||
|
||||
assert io_support.get_dummy_run_num_frames("unknown", supports_audio_input=True) == 2
|
||||
|
||||
|
||||
def test_dummy_run_image_count_resolves_hunyuan_architecture_alias() -> None:
|
||||
assert io_support.get_dummy_run_num_image_inputs("HunyuanImage3ForCausalMM") == 3
|
||||
assert io_support.get_dummy_run_num_image_inputs("unknown") == 1
|
||||
|
||||
|
||||
def test_dense_mode_does_not_build_kv_profile_request() -> None:
|
||||
engine = object.__new__(DiffusionEngine)
|
||||
engine.od_config = SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.DENSE_LEGACY)
|
||||
engine._make_dummy_request = Mock(side_effect=AssertionError("dense mode must not profile"))
|
||||
|
||||
assert engine._prepare_diffusion_kv_profile_requests() is None
|
||||
engine._make_dummy_request.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("execution_mode", "uses_dlo_dp", "max_num_seqs", "expected_profile_requests"),
|
||||
[
|
||||
(DiffusionExecutionMode.STEP_BATCH, False, 2, 2),
|
||||
(DiffusionExecutionMode.REQUEST_BATCH, True, 4, 1),
|
||||
(DiffusionExecutionMode.STEP_BATCH, True, 3, 3),
|
||||
],
|
||||
)
|
||||
def test_paged_kv_profile_requests_match_per_rank_batch(
|
||||
execution_mode: DiffusionExecutionMode,
|
||||
uses_dlo_dp: bool,
|
||||
max_num_seqs: int,
|
||||
expected_profile_requests: int,
|
||||
) -> None:
|
||||
engine = object.__new__(DiffusionEngine)
|
||||
engine.execution_mode = execution_mode
|
||||
engine.od_config = SimpleNamespace(
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
|
||||
model_class_name="HunyuanImage3ForCausalMM",
|
||||
max_num_seqs=max_num_seqs,
|
||||
parallel_config=SimpleNamespace(data_parallel_size=max_num_seqs if uses_dlo_dp else 1),
|
||||
enable_distributed_layerwise_offload=uses_dlo_dp,
|
||||
dlo_use_allgather=True,
|
||||
)
|
||||
request = OmniDiffusionRequest(
|
||||
prompt="profile",
|
||||
request_id="profile-request",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
)
|
||||
engine._make_dummy_request = Mock(return_value=request)
|
||||
prepared_layout = object()
|
||||
scheduler_kv_state = (
|
||||
DiffusionKVRequest(
|
||||
"profile-request/diffusion-kv/0",
|
||||
sequence_id=0,
|
||||
prefix_len=3,
|
||||
target_len=8,
|
||||
seq_len=12,
|
||||
),
|
||||
DiffusionKVRequest(
|
||||
"profile-request/diffusion-kv/1",
|
||||
sequence_id=1,
|
||||
prefix_len=4,
|
||||
target_len=8,
|
||||
seq_len=13,
|
||||
),
|
||||
)
|
||||
|
||||
def preprocess(req: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
req.prepared_layout = prepared_layout
|
||||
req.diffusion_kv_requests = scheduler_kv_state
|
||||
return req
|
||||
|
||||
engine._prepare_request_for_admission = Mock(side_effect=preprocess)
|
||||
|
||||
result = engine._prepare_diffusion_kv_profile_requests()
|
||||
|
||||
assert result is not None
|
||||
assert len(result) == expected_profile_requests
|
||||
assert len({profile_request.request_id for profile_request in result}) == expected_profile_requests
|
||||
assert all(profile_request.is_dummy_run() for profile_request in result)
|
||||
assert all(profile_request.prepared_layout is prepared_layout for profile_request in result)
|
||||
assert all(profile_request.diffusion_kv_requests is None for profile_request in result)
|
||||
assert len({id(profile_request.sampling_params) for profile_request in result}) == expected_profile_requests
|
||||
assert engine._diffusion_kv_profile_limits == (2, 13, 8)
|
||||
assert len(result) * engine._diffusion_kv_profile_limits[0] == expected_profile_requests * 2
|
||||
engine._make_dummy_request.assert_called_once_with(
|
||||
height=1024,
|
||||
width=1024,
|
||||
guidance_scale=5.0,
|
||||
num_image_inputs=3,
|
||||
)
|
||||
engine._prepare_request_for_admission.assert_called_once_with(request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_sequences", "seq_len", "target_len"),
|
||||
[
|
||||
(3, 16, 8),
|
||||
(1, 17, 8),
|
||||
(1, 16, 9),
|
||||
],
|
||||
)
|
||||
def test_paged_kv_admission_rejects_shape_beyond_profile_envelope(
|
||||
num_sequences: int,
|
||||
seq_len: int,
|
||||
target_len: int,
|
||||
) -> None:
|
||||
engine = object.__new__(DiffusionEngine)
|
||||
engine._diffusion_kv_profile_limits = (2, 16, 8)
|
||||
engine.pre_process_func = lambda request: request
|
||||
request = OmniDiffusionRequest(
|
||||
prompt="too large",
|
||||
request_id="too-large",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
diffusion_kv_requests=tuple(
|
||||
DiffusionKVRequest(
|
||||
f"too-large/diffusion-kv/{sequence_id}",
|
||||
sequence_id=sequence_id,
|
||||
prefix_len=0,
|
||||
target_len=target_len,
|
||||
seq_len=seq_len,
|
||||
)
|
||||
for sequence_id in range(num_sequences)
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="exceeds the startup memory-profile envelope"):
|
||||
engine._prepare_request_for_admission(request)
|
||||
|
||||
|
||||
def test_paged_kv_admission_accepts_shape_at_profile_envelope() -> None:
|
||||
engine = object.__new__(DiffusionEngine)
|
||||
engine._diffusion_kv_profile_limits = (2, 16, 8)
|
||||
engine.pre_process_func = lambda request: request
|
||||
request = OmniDiffusionRequest(
|
||||
prompt="fits",
|
||||
request_id="fits",
|
||||
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||
diffusion_kv_requests=tuple(
|
||||
DiffusionKVRequest(
|
||||
f"fits/diffusion-kv/{sequence_id}",
|
||||
sequence_id=sequence_id,
|
||||
prefix_len=8,
|
||||
target_len=8,
|
||||
seq_len=16,
|
||||
)
|
||||
for sequence_id in range(2)
|
||||
),
|
||||
)
|
||||
|
||||
assert engine._prepare_request_for_admission(request) is request
|
||||
@@ -473,6 +473,38 @@ async def test_cancelled_future_does_not_kill_busy_loop():
|
||||
_stop_engine(engine)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_kv_reservation_error_wakes_stream_without_killing_busy_loop():
|
||||
loop = asyncio.get_running_loop()
|
||||
engine = _make_engine_with_loop(loop)
|
||||
|
||||
def fail_reservation(*args, **kwargs):
|
||||
raise RuntimeError("native allocation bug")
|
||||
|
||||
engine.scheduler._diffusion_kv_manager = SimpleNamespace(
|
||||
has_request=lambda request_id: False,
|
||||
reserve_request=fail_reservation,
|
||||
free_request=lambda request_id: None,
|
||||
)
|
||||
request = _make_request("kv-error")
|
||||
request.diffusion_kv_requests = (object(),)
|
||||
try:
|
||||
response_stream = engine.async_add_req_and_stream_response(request)
|
||||
output_queue = engine._out_streams[request.request_id]
|
||||
for _ in range(300):
|
||||
if not output_queue.empty():
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert not output_queue.empty(), "terminal KV allocation error was not delivered to the request stream"
|
||||
|
||||
output = await anext(response_stream)
|
||||
assert output.error == "native allocation bug"
|
||||
assert output.finished
|
||||
assert engine.worker_thread.is_alive()
|
||||
finally:
|
||||
_stop_engine(engine)
|
||||
|
||||
|
||||
# ─────────────────────────── _RpcTask basics ───────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -512,6 +513,84 @@ def test_execute_model_accepts_bare_diffusion_output_from_single_request_pipelin
|
||||
assert runner.pipeline.last_req.num_reqs == 1
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_profile_run_executes_forward_without_scheduler_kv_validation(monkeypatch):
|
||||
runner = _make_runner(cache_backend=None, cache_backend_name="none")
|
||||
request = _make_request()
|
||||
runner._validate_diffusion_kv_metadata = Mock(side_effect=AssertionError("profile must bypass admission"))
|
||||
record_names = []
|
||||
original_execute = runner._execute_request_list
|
||||
|
||||
def execute(*args, **kwargs):
|
||||
record_names.append(kwargs["record_name"])
|
||||
return original_execute(*args, **kwargs)
|
||||
|
||||
runner._execute_request_list = execute
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
reset_peak_memory_stats = Mock()
|
||||
monkeypatch.setattr(
|
||||
model_runner_module.current_omni_platform,
|
||||
"reset_peak_memory_stats",
|
||||
reset_peak_memory_stats,
|
||||
)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
|
||||
synchronize = Mock()
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "synchronize", synchronize)
|
||||
|
||||
runner.profile_run([request])
|
||||
|
||||
assert runner.pipeline.forward_calls == 1
|
||||
assert record_names == ["pipeline_memory_profile"]
|
||||
runner._validate_diffusion_kv_metadata.assert_not_called()
|
||||
reset_peak_memory_stats.assert_not_called()
|
||||
synchronize.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_profile_run_executes_maximum_step_batch_without_resetting_peak(monkeypatch):
|
||||
runner = _make_runner(cache_backend=None, cache_backend_name=None)
|
||||
runner.pipeline = _FinalOnlyStepPipeline()
|
||||
runner.od_config.step_execution = True
|
||||
requests = [_make_request(), _make_request()]
|
||||
requests[0].request_id = "profile-0"
|
||||
requests[1].request_id = "profile-1"
|
||||
for request in requests:
|
||||
request.sampling_params.num_inference_steps = 1
|
||||
|
||||
observed_batch_rows = []
|
||||
original_denoise_step = runner.pipeline.denoise_step
|
||||
|
||||
def denoise_step(input_batch, states):
|
||||
observed_batch_rows.append((len(states), int(input_batch.latents.shape[0])))
|
||||
return original_denoise_step(input_batch, states)
|
||||
|
||||
runner.pipeline.denoise_step = denoise_step
|
||||
runner._validate_diffusion_kv_metadata = Mock(side_effect=AssertionError("profile must bypass admission"))
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
reset_peak_memory_stats = Mock()
|
||||
monkeypatch.setattr(
|
||||
model_runner_module.current_omni_platform,
|
||||
"reset_peak_memory_stats",
|
||||
reset_peak_memory_stats,
|
||||
)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_reserved", lambda: 0)
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "max_memory_allocated", lambda: 0)
|
||||
synchronize = Mock()
|
||||
monkeypatch.setattr(model_runner_module.current_omni_platform, "synchronize", synchronize)
|
||||
|
||||
runner.profile_run(requests)
|
||||
|
||||
assert observed_batch_rows == [(2, 2)]
|
||||
assert runner.state_cache == {}
|
||||
assert runner.input_batch is None
|
||||
runner._validate_diffusion_kv_metadata.assert_not_called()
|
||||
reset_peak_memory_stats.assert_not_called()
|
||||
synchronize.assert_called_once_with()
|
||||
|
||||
|
||||
class _BatchPipeline:
|
||||
"""Pipeline returning a configurable list of outputs from forward()."""
|
||||
|
||||
|
||||
@@ -52,6 +52,7 @@ def test_dummy_run_request_is_identified_by_reserved_request_id():
|
||||
|
||||
assert req.is_dummy_run()
|
||||
assert OmniDiffusionRequest.is_dummy_run_request_id(DUMMY_DIFFUSION_REQUEST_ID)
|
||||
assert OmniDiffusionRequest.is_dummy_run_request_id(f"{DUMMY_DIFFUSION_REQUEST_ID}/kv-profile-1")
|
||||
|
||||
|
||||
def test_non_dummy_request_is_not_identified_as_dummy_run():
|
||||
|
||||
@@ -8,12 +8,14 @@ from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, DiffusionRequestAbortedError
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine, DiffusionExecutionMode
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVContext, DiffusionKVRequest
|
||||
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.sched import (
|
||||
BaseScheduler,
|
||||
@@ -87,6 +89,52 @@ def _cached_ids(sched_output) -> list[str]:
|
||||
return list(sched_output.scheduled_cached_reqs.request_ids)
|
||||
|
||||
|
||||
def _initialize_paged_scheduler(
|
||||
scheduler: BaseScheduler,
|
||||
*,
|
||||
num_blocks: int = 64,
|
||||
max_num_seqs: int = 1,
|
||||
) -> None:
|
||||
native_kv_managers.register_all_kvcache_specs(None)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[KVCacheTensor(size=spec.page_size_bytes * num_blocks, shared_by=["layer0"])],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)],
|
||||
)
|
||||
scheduler.initialize(
|
||||
SimpleNamespace(
|
||||
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
|
||||
max_model_len=64,
|
||||
max_num_seqs=max_num_seqs,
|
||||
),
|
||||
kv_cache_config=config,
|
||||
scheduler_block_size=4,
|
||||
hash_block_size=4,
|
||||
kv_vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=64),
|
||||
max_in_flight_tokens=64,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _attach_diffusion_kv(request: OmniDiffusionRequest, *, seq_len: int = 8) -> None:
|
||||
request.diffusion_kv_requests = (
|
||||
DiffusionKVRequest(
|
||||
f"{request.request_id}/diffusion-kv/0",
|
||||
sequence_id=0,
|
||||
prefix_len=4,
|
||||
target_len=4,
|
||||
seq_len=seq_len,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _StubScheduler:
|
||||
def __init__(self, request: OmniDiffusionRequest, output) -> None:
|
||||
self._request = request
|
||||
@@ -400,18 +448,16 @@ class TestRequestScheduler:
|
||||
assert self.scheduler.has_requests() is False
|
||||
|
||||
def test_diffusion_kv_request_moves_to_scheduler_state(self) -> None:
|
||||
self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER))
|
||||
_initialize_paged_scheduler(self.scheduler)
|
||||
request = _make_request("diffusion-kv")
|
||||
prepared_layout = object()
|
||||
request.prepared_layout = prepared_layout
|
||||
text_context = DiffusionKVContext(context_id="text", cache_role="cross.text", num_tokens=8)
|
||||
kv_request = DiffusionKVRequest(
|
||||
"diffusion-kv/diffusion-kv/0",
|
||||
sequence_id=0,
|
||||
prefix_len=4,
|
||||
target_len=8,
|
||||
seq_len=16,
|
||||
kv_contexts=(text_context,),
|
||||
)
|
||||
request.diffusion_kv_requests = (kv_request,)
|
||||
|
||||
@@ -421,12 +467,179 @@ class TestRequestScheduler:
|
||||
|
||||
assert state is not None
|
||||
assert state.diffusion_kv_requests == (kv_request,)
|
||||
assert state.diffusion_kv_requests[0].kv_contexts == (text_context,)
|
||||
assert request.diffusion_kv_requests is None
|
||||
assert scheduler_output.scheduled_new_reqs[0].req.prepared_layout is prepared_layout
|
||||
metadata = scheduler_output.scheduled_new_reqs[0].diffusion_kv_metadata
|
||||
assert metadata is not None
|
||||
assert metadata.request_id == request.request_id
|
||||
assert len(metadata.sequences[0].block_ids[0]) == 4
|
||||
|
||||
def test_diffusion_kv_capacity_backpressures_fifo_until_blocks_are_freed(self) -> None:
|
||||
_initialize_paged_scheduler(self.scheduler, num_blocks=3, max_num_seqs=2)
|
||||
first_request = _make_request("first")
|
||||
second_request = _make_request("second")
|
||||
_attach_diffusion_kv(first_request)
|
||||
_attach_diffusion_kv(second_request)
|
||||
self.scheduler.add_request(first_request)
|
||||
self.scheduler.add_request(second_request)
|
||||
|
||||
first_output = self.scheduler.schedule()
|
||||
|
||||
assert _new_ids(first_output) == ["first"]
|
||||
assert first_output.num_waiting_reqs == 1
|
||||
|
||||
self.scheduler.update_from_output(first_output, _make_request_output("first"))
|
||||
second_output = self.scheduler.schedule()
|
||||
|
||||
assert _new_ids(second_output) == ["second"]
|
||||
assert second_output.num_waiting_reqs == 0
|
||||
# The same output carries both the release and the newly admitted block
|
||||
# table; the future Worker data plane must process finished ids first.
|
||||
assert second_output.finished_req_ids == {"first"}
|
||||
assert second_output.scheduled_new_reqs[0].diffusion_kv_metadata is not None
|
||||
|
||||
def test_impossible_diffusion_kv_capacity_finishes_only_that_request(self) -> None:
|
||||
_initialize_paged_scheduler(self.scheduler, num_blocks=2, max_num_seqs=2)
|
||||
impossible = _make_request("impossible")
|
||||
schedulable = _make_request("schedulable")
|
||||
_attach_diffusion_kv(impossible, seq_len=8)
|
||||
schedulable.diffusion_kv_requests = (
|
||||
DiffusionKVRequest(
|
||||
"schedulable/diffusion-kv/0",
|
||||
sequence_id=0,
|
||||
prefix_len=0,
|
||||
target_len=4,
|
||||
seq_len=4,
|
||||
),
|
||||
)
|
||||
self.scheduler.add_request(impossible)
|
||||
self.scheduler.add_request(schedulable)
|
||||
|
||||
sched_output = self.scheduler.schedule()
|
||||
|
||||
failed_state = self.scheduler.get_request_state("impossible")
|
||||
assert failed_state is not None
|
||||
assert failed_state.status == DiffusionRequestStatus.FINISHED_ERROR
|
||||
assert failed_state.error is not None
|
||||
assert "cannot fit even when the block pool is empty" in failed_state.error
|
||||
assert sched_output.finished_req_ids == {"impossible"}
|
||||
assert _new_ids(sched_output) == ["schedulable"]
|
||||
|
||||
finished = self.scheduler.update_from_output(
|
||||
sched_output,
|
||||
_make_request_output("schedulable"),
|
||||
)
|
||||
|
||||
assert finished == {"impossible", "schedulable"}
|
||||
|
||||
def test_impossible_diffusion_kv_capacity_does_not_block_waiters_under_load(self) -> None:
|
||||
_initialize_paged_scheduler(self.scheduler, num_blocks=4, max_num_seqs=2)
|
||||
running = _make_request("running")
|
||||
_attach_diffusion_kv(running)
|
||||
self.scheduler.add_request(running)
|
||||
assert _new_ids(self.scheduler.schedule()) == ["running"]
|
||||
|
||||
impossible = _make_request("impossible")
|
||||
impossible.diffusion_kv_requests = tuple(
|
||||
DiffusionKVRequest(
|
||||
f"impossible/diffusion-kv/{sequence_id}",
|
||||
sequence_id=sequence_id,
|
||||
prefix_len=0,
|
||||
target_len=4,
|
||||
seq_len=4,
|
||||
)
|
||||
for sequence_id in range(4)
|
||||
)
|
||||
schedulable = _make_request("schedulable")
|
||||
schedulable.diffusion_kv_requests = (
|
||||
DiffusionKVRequest(
|
||||
"schedulable/diffusion-kv/0",
|
||||
sequence_id=0,
|
||||
prefix_len=0,
|
||||
target_len=4,
|
||||
seq_len=4,
|
||||
),
|
||||
)
|
||||
self.scheduler.add_request(impossible)
|
||||
self.scheduler.add_request(schedulable)
|
||||
|
||||
sched_output = self.scheduler.schedule()
|
||||
|
||||
failed_state = self.scheduler.get_request_state("impossible")
|
||||
assert failed_state is not None
|
||||
assert failed_state.status == DiffusionRequestStatus.FINISHED_ERROR
|
||||
assert failed_state.error is not None
|
||||
assert "required_blocks=4, available_blocks=3" in failed_state.error
|
||||
assert sched_output.finished_req_ids == {"impossible"}
|
||||
assert _new_ids(sched_output) == ["schedulable"]
|
||||
|
||||
def test_diffusion_kv_internal_allocation_error_finishes_request(self, monkeypatch) -> None:
|
||||
_initialize_paged_scheduler(self.scheduler)
|
||||
request = _make_request("native-error")
|
||||
_attach_diffusion_kv(request)
|
||||
self.scheduler.add_request(request)
|
||||
manager = self.scheduler._diffusion_kv_manager
|
||||
assert manager is not None
|
||||
|
||||
def raise_native_error(*args, **kwargs):
|
||||
raise ValueError("native allocation bug")
|
||||
|
||||
monkeypatch.setattr(
|
||||
manager,
|
||||
"reserve_request",
|
||||
raise_native_error,
|
||||
)
|
||||
|
||||
sched_output = self.scheduler.schedule()
|
||||
|
||||
state = self.scheduler.get_request_state("native-error")
|
||||
assert state is not None
|
||||
assert state.status == DiffusionRequestStatus.FINISHED_ERROR
|
||||
assert state.error == "native allocation bug"
|
||||
assert sched_output.finished_req_ids == {"native-error"}
|
||||
assert sched_output.scheduled_request_ids == []
|
||||
|
||||
def test_diffusion_kv_preemption_retains_allocation(self) -> None:
|
||||
_initialize_paged_scheduler(self.scheduler, num_blocks=3)
|
||||
request = _make_request("preempted")
|
||||
_attach_diffusion_kv(request)
|
||||
self.scheduler.add_request(request)
|
||||
self.scheduler.schedule()
|
||||
manager = self.scheduler._diffusion_kv_manager
|
||||
assert manager is not None
|
||||
metadata = manager.get_metadata("preempted")
|
||||
|
||||
assert self.scheduler.preempt_request("preempted") is True
|
||||
resumed_output = self.scheduler.schedule()
|
||||
|
||||
assert resumed_output.scheduled_new_reqs == []
|
||||
assert _cached_ids(resumed_output) == ["preempted"]
|
||||
assert manager.get_metadata("preempted") == metadata
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status",
|
||||
[
|
||||
DiffusionRequestStatus.FINISHED_COMPLETED,
|
||||
DiffusionRequestStatus.FINISHED_ABORTED,
|
||||
DiffusionRequestStatus.FINISHED_ERROR,
|
||||
],
|
||||
)
|
||||
def test_diffusion_kv_terminal_status_releases_allocation(self, status: DiffusionRequestStatus) -> None:
|
||||
_initialize_paged_scheduler(self.scheduler, num_blocks=3)
|
||||
request = _make_request("terminal")
|
||||
_attach_diffusion_kv(request)
|
||||
self.scheduler.add_request(request)
|
||||
self.scheduler.schedule()
|
||||
manager = self.scheduler._diffusion_kv_manager
|
||||
assert manager is not None
|
||||
assert manager.has_request("terminal") is True
|
||||
|
||||
self.scheduler.finish_requests("terminal", status)
|
||||
|
||||
assert manager.has_request("terminal") is False
|
||||
|
||||
def test_paged_scheduler_rejects_missing_kv_request(self) -> None:
|
||||
self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER))
|
||||
_initialize_paged_scheduler(self.scheduler)
|
||||
request = _make_request("diffusion-kv")
|
||||
|
||||
with pytest.raises(ValueError, match="did not produce DiffusionKVRequest"):
|
||||
@@ -443,7 +656,7 @@ class TestRequestScheduler:
|
||||
],
|
||||
)
|
||||
def test_diffusion_kv_request_rejects_legacy_dense_kv(self, owner_name: str, field_name: str) -> None:
|
||||
self.scheduler.initialize(SimpleNamespace(diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER))
|
||||
_initialize_paged_scheduler(self.scheduler)
|
||||
request = _make_request("diffusion-kv")
|
||||
owner = request if owner_name == "request" else request.sampling_params
|
||||
setattr(owner, field_name, object())
|
||||
@@ -921,6 +1134,45 @@ class TestDiffusionEngine:
|
||||
assert engine.scheduler.max_num_running_reqs == 1
|
||||
fake_executor_cls.assert_called_once_with(od_config)
|
||||
|
||||
def test_initializes_paged_scheduler_from_native_control_plane(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
od_config = SimpleNamespace(model_class_name="mock_model", streaming_output=False)
|
||||
fake_executor_cls = mocker.Mock(return_value=mocker.Mock())
|
||||
scheduler = mocker.Mock()
|
||||
kv_cache_config = object()
|
||||
kv_vllm_config = object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.diffusion_engine.get_diffusion_post_process_func",
|
||||
lambda *args, **kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.diffusion_engine.get_diffusion_pre_process_func",
|
||||
lambda *args, **kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.diffusion_engine.DiffusionExecutor.get_class",
|
||||
lambda *args, **kwargs: fake_executor_cls,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.diffusion_engine.initialize_diffusion_kv_control_plane",
|
||||
lambda *args, **kwargs: (kv_cache_config, 16, 16, kv_vllm_config),
|
||||
)
|
||||
|
||||
engine = DiffusionEngine(od_config, scheduler=scheduler)
|
||||
|
||||
assert engine.scheduler is scheduler
|
||||
scheduler.initialize.assert_called_once_with(
|
||||
od_config,
|
||||
kv_cache_config=kv_cache_config,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=16,
|
||||
kv_vllm_config=kv_vllm_config,
|
||||
)
|
||||
|
||||
def test_scheduler_alias_keeps_default_request_scheduler(self) -> None:
|
||||
scheduler = Scheduler()
|
||||
scheduler.initialize(SimpleNamespace())
|
||||
@@ -987,6 +1239,21 @@ class TestDiffusionEngine:
|
||||
assert output.aborted is True
|
||||
assert output.abort_message == "Request req-finalize aborted."
|
||||
|
||||
def test_finalize_finished_request_returns_scheduler_admission_error(self) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
engine.scheduler = StepScheduler()
|
||||
engine.scheduler.initialize(SimpleNamespace())
|
||||
|
||||
req_id = engine.scheduler.add_request(_make_request("req-error"))
|
||||
engine.scheduler._finish_requests(
|
||||
{req_id: DiffusionRequestStatus.FINISHED_ERROR},
|
||||
{req_id: "KV request cannot fit"},
|
||||
)
|
||||
|
||||
output = engine._finalize_finished_request(req_id)
|
||||
|
||||
assert output.error == "KV request cannot fit"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_runner_output_notifies_each_chunk(self) -> None:
|
||||
engine = DiffusionEngine.__new__(DiffusionEngine)
|
||||
|
||||
@@ -13,11 +13,7 @@ import pytest
|
||||
import torch
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import (
|
||||
DiffusionWorker,
|
||||
_create_diffusion_worker_vllm_config,
|
||||
_make_diffusion_vllm_model_config,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.gpu]
|
||||
|
||||
@@ -58,34 +54,6 @@ def mock_gpu_worker(mocker: MockerFixture, mock_od_config):
|
||||
return worker
|
||||
|
||||
|
||||
def test_diffusion_vllm_model_config_supplies_dtype_for_quant_methods():
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm_omni.quantization import build_quant_config
|
||||
|
||||
od_config = SimpleNamespace(
|
||||
model="dummy",
|
||||
dtype=torch.bfloat16,
|
||||
quantization_config=build_quant_config(
|
||||
{
|
||||
"quant_method": "modelopt",
|
||||
"quant_algo": "FP8",
|
||||
"ignore": [],
|
||||
}
|
||||
),
|
||||
tf_model_config=SimpleNamespace(),
|
||||
enforce_eager=True,
|
||||
is_moe=False,
|
||||
)
|
||||
|
||||
model_config = _make_diffusion_vllm_model_config(od_config)
|
||||
|
||||
assert model_config.dtype is torch.bfloat16
|
||||
assert model_config.quantization == "modelopt"
|
||||
assert model_config.quantization_config is od_config.quantization_config
|
||||
assert model_config.is_quantized()
|
||||
|
||||
|
||||
class TestDiffusionWorkerSleep:
|
||||
"""Test DiffusionWorker.sleep method."""
|
||||
|
||||
@@ -329,49 +297,3 @@ class TestDiffusionWorkerWakeUp:
|
||||
mock_buffer2.data.copy_.assert_not_called()
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
class TestWorkerVllmConfigAdditionalConfig:
|
||||
"""Test worker-side VllmConfig construction with additional_config."""
|
||||
|
||||
def test_create_diffusion_worker_vllm_config_passes_additional_config(self, mocker: MockerFixture):
|
||||
mock_vllm_config = mocker.Mock()
|
||||
mock_vllm_cls = mocker.patch(
|
||||
"vllm_omni.diffusion.worker.diffusion_worker.VllmConfig",
|
||||
return_value=mock_vllm_config,
|
||||
)
|
||||
od_config = mocker.Mock(additional_config={"torchair_graph_config": {"enabled": True}})
|
||||
|
||||
result = _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config)
|
||||
|
||||
assert result is mock_vllm_config
|
||||
assert mock_vllm_cls.call_args.kwargs["additional_config"] == od_config.additional_config
|
||||
|
||||
def test_create_diffusion_worker_vllm_config_falls_back_when_constructor_rejects_additional_config(
|
||||
self, mocker: MockerFixture
|
||||
):
|
||||
mock_vllm_config = mocker.Mock()
|
||||
mock_vllm_cls = mocker.patch(
|
||||
"vllm_omni.diffusion.worker.diffusion_worker.VllmConfig",
|
||||
side_effect=[
|
||||
TypeError("VllmConfig.__init__() got an unexpected keyword argument 'additional_config'"),
|
||||
mock_vllm_config,
|
||||
],
|
||||
)
|
||||
od_config = mocker.Mock(additional_config={"ascend_scheduler_config": {"foo": "bar"}})
|
||||
|
||||
result = _create_diffusion_worker_vllm_config(torch.device("cpu"), od_config)
|
||||
|
||||
assert result is mock_vllm_config
|
||||
assert mock_vllm_cls.call_count == 2
|
||||
assert getattr(mock_vllm_config, "additional_config") == od_config.additional_config
|
||||
|
||||
def test_create_diffusion_worker_vllm_config_reraises_other_type_errors(self, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.diffusion.worker.diffusion_worker.VllmConfig",
|
||||
side_effect=TypeError("additional_config values must be hashable"),
|
||||
)
|
||||
od_config = mocker.Mock(additional_config={"ascend_scheduler_config": {"foo": "bar"}})
|
||||
|
||||
with pytest.raises(TypeError, match="hashable"):
|
||||
_create_diffusion_worker_vllm_config(torch.device("cpu"), od_config)
|
||||
@@ -413,6 +413,26 @@ class TestRequestModeDispatch:
|
||||
assert [output.result.error for output in result.runner_outputs] == ["A", "B"]
|
||||
executor.collective_rpc.assert_called_once()
|
||||
|
||||
def test_dlo_dp_forwards_request_metadata_envelopes_as_one_wave(self):
|
||||
executor, _, _ = _make_executor(num_gpus=2)
|
||||
executor.od_config = SimpleNamespace(
|
||||
step_execution=False,
|
||||
parallel_config=SimpleNamespace(data_parallel_size=2),
|
||||
enable_distributed_layerwise_offload=True,
|
||||
dlo_use_allgather=True,
|
||||
)
|
||||
executor.collective_rpc = Mock(return_value=[_tagged_output("A"), _tagged_output("B")])
|
||||
scheduler_output = _make_sched_output("A", "B")
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
new_req.diffusion_kv_metadata = SimpleNamespace(request_id=new_req.request_id)
|
||||
|
||||
result = executor.execute_request(scheduler_output)
|
||||
|
||||
assert [output.result.error for output in result.runner_outputs] == ["A", "B"]
|
||||
forwarded_envelopes = executor.collective_rpc.call_args.kwargs["args"][0]
|
||||
assert forwarded_envelopes is scheduler_output.scheduled_new_reqs
|
||||
assert [envelope.diffusion_kv_metadata.request_id for envelope in forwarded_envelopes] == ["A", "B"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[
|
||||
|
||||
Reference in new issue
Block a user