[Diffusion] Register runtime cache for direct DLO H2D
Avoid recurrent mmap-to-pinned packing in no-AllGather DLO by registering transform-complete runtime-cache mappings under an explicit per-worker budget. Copy mapped tensors directly into the rotating device buffers while preserving bounded-staging fallback and ordered shutdown cleanup. Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
67f15e73cd
commit
23ab14a15b
18 files changed
+976
-74
No files matched your search
@@ -333,6 +333,8 @@ def test_serve_cli_forwards_distributed_offload_residency():
|
||||
"/var/cache/vllm-omni-dlo",
|
||||
"--dlo-runtime-cache-lock-timeout",
|
||||
"30",
|
||||
"--dlo-runtime-cache-pin-limit-gib",
|
||||
"80",
|
||||
"--dlo-resident-layers",
|
||||
"20",
|
||||
]
|
||||
@@ -347,12 +349,14 @@ def test_serve_cli_forwards_distributed_offload_residency():
|
||||
assert args.dlo_enable_runtime_cache is True
|
||||
assert args.dlo_runtime_cache_dir == "/var/cache/vllm-omni-dlo"
|
||||
assert args.dlo_runtime_cache_lock_timeout == 30
|
||||
assert args.dlo_runtime_cache_pin_limit_gib == 80
|
||||
assert args.dlo_resident_layers == 20
|
||||
assert engine_args["enable_distributed_layerwise_offload"] is True
|
||||
assert engine_args["dlo_use_allgather"] is False
|
||||
assert engine_args["dlo_enable_runtime_cache"] is True
|
||||
assert engine_args["dlo_runtime_cache_dir"] == "/var/cache/vllm-omni-dlo"
|
||||
assert engine_args["dlo_runtime_cache_lock_timeout"] == 30
|
||||
assert engine_args["dlo_runtime_cache_pin_limit_gib"] == 80
|
||||
assert engine_args["dlo_resident_layers"] == 20
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user