[Feat] Add native Mooncake KV transfer from AR to DiT (#7166)
Signed-off-by: asukaqaq-s <1311722138@qq.com> Signed-off-by: Acerak01-fy <wfy2003324@163.com> Co-authored-by: Acerak01-fy <wfy2003324@163.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
53 files changed
+2564
-204
No files matched your search
@@ -2395,13 +2395,20 @@ def test_dist_stage_runtime_applies_local_dp_to_stage_config(typed):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_replicas", [1, 2, 3])
|
||||
def test_typed_diffusion_replicas_share_one_config_between_planning_and_launch(mocker, num_replicas):
|
||||
@pytest.mark.parametrize("kv_owner", [None, "connector_config", "diffusion_config"])
|
||||
def test_typed_diffusion_replicas_share_one_config_between_planning_and_launch(mocker, num_replicas, kv_owner):
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.engine import stage_runtime as runtime_module
|
||||
|
||||
stage = StageConfigFactory.create_typed_default_diffusion(
|
||||
"generic-diffusion", {"model_class_name": "QwenImagePipeline"}
|
||||
).stage_configs[0]
|
||||
if kv_owner is not None:
|
||||
from vllm.config import KVTransferConfig
|
||||
|
||||
getattr(stage, kv_owner).kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_consumer", engine_id="dit"
|
||||
)
|
||||
devices = ",".join(str(i) for i in range(num_replicas))
|
||||
stage.runtime_config.devices = devices
|
||||
stage.runtime_config.num_replicas = num_replicas
|
||||
@@ -2432,11 +2439,114 @@ def test_typed_diffusion_replicas_share_one_config_between_planning_and_launch(m
|
||||
for i, plan in enumerate(replicas):
|
||||
assert plan.metadata.runtime_cfg is plan.stage_cfg.runtime_config
|
||||
assert plan.stage_cfg.runtime_config.devices == str(i)
|
||||
if num_replicas > 1:
|
||||
if num_replicas > 1 or kv_owner is not None:
|
||||
assert plan.stage_cfg is not stage
|
||||
if kv_owner is not None:
|
||||
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||
|
||||
assert getattr(stage, kv_owner).kv_transfer_config.engine_id == "dit"
|
||||
engine_args = build_engine_args_dict_from_omni_stage_config(plan.stage_cfg, "dummy-model")
|
||||
assert engine_args["kv_transfer_config"].engine_id == f"dit-s0-r{i}"
|
||||
assert runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=1) is client
|
||||
assert launch.call_args.kwargs["stage_config"] is plan.stage_cfg
|
||||
assert launch.call_args.kwargs["metadata"] is plan.metadata
|
||||
assert launch.call_args.kwargs["use_inline"] is (num_replicas == 1)
|
||||
|
||||
assert launch.call_count == 2 * num_replicas
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||
@pytest.mark.parametrize("num_replicas", [1, 2])
|
||||
def test_native_kv_producer_replica_identity_and_bootstrap_are_isolated(mocker, typed, num_replicas):
|
||||
from vllm.config import KVTransferConfig
|
||||
|
||||
from vllm_omni.config.omni_config import VllmOmniARStageConfig
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
from vllm_omni.engine import stage_runtime as runtime_module
|
||||
|
||||
kv_config = KVTransferConfig(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_producer",
|
||||
engine_id="ar",
|
||||
kv_ip="127.0.0.1",
|
||||
kv_connector_extra_config={"bootstrap_port": 9100},
|
||||
)
|
||||
if typed:
|
||||
stage = VllmOmniARStageConfig(stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="ar"))
|
||||
stage.connector_config.kv_transfer_config = kv_config
|
||||
runtime_cfg = stage.runtime_config
|
||||
else:
|
||||
runtime_cfg = OmegaConf.create({"devices": "0,1", "env": {"KEEP": "value"}})
|
||||
stage = types.SimpleNamespace(stage_id=0, engine_args={"kv_transfer_config": kv_config}, runtime=runtime_cfg)
|
||||
metadata = _make_llm_metadata(0)
|
||||
mocker.patch.object(
|
||||
runtime_module,
|
||||
"extract_legacy_stage_metadata",
|
||||
side_effect=lambda cfg: types.SimpleNamespace(**{**metadata.__dict__, "runtime_cfg": cfg.runtime}),
|
||||
)
|
||||
runtime_cfg.devices = "0,1"
|
||||
runtime_cfg.env = {"KEEP": "value"}
|
||||
runtime = StageRuntime(
|
||||
stage_configs=[stage],
|
||||
model="dummy-model",
|
||||
config_path="dummy-config",
|
||||
stage_init_timeout=1,
|
||||
async_chunk=False,
|
||||
)
|
||||
mocker.patch.object(runtime_module, "get_stage_connector_spec", return_value={})
|
||||
mocker.patch.object(runtime_module, "resolve_omni_kv_config_for_stage", return_value=(None, None, None))
|
||||
for builder in ("build_engine_args_dict", "build_engine_args_dict_from_omni_stage_config"):
|
||||
mocker.patch.object(runtime_module, builder, return_value={})
|
||||
mocker.patch.object(
|
||||
runtime_module,
|
||||
"build_vllm_config",
|
||||
return_value=(types.SimpleNamespace(kv_transfer_config=kv_config), object),
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
plans = runtime._build_logical_stage_init_plans(None, [num_replicas], {})
|
||||
for i, replica in enumerate(plans[0].replicas):
|
||||
config = replica.stage_vllm_config.kv_transfer_config
|
||||
assert config.engine_id == f"ar-s0-r{i}"
|
||||
assert config.kv_connector_extra_config["bootstrap_addr"] == f"http://127.0.0.1:{9100 + i}"
|
||||
assert dict(replica.metadata.runtime_cfg.env) == {
|
||||
"KEEP": "value",
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": str(9100 + i),
|
||||
}
|
||||
if typed:
|
||||
assert replica.metadata.runtime_cfg is replica.stage_cfg.runtime_config
|
||||
assert kv_config.engine_id == "ar"
|
||||
assert kv_config.kv_connector_extra_config == {"bootstrap_port": 9100}
|
||||
assert dict(runtime_cfg.env) == {"KEEP": "value"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"roles, sources, async_chunk, valid",
|
||||
[
|
||||
(["kv_producer", "kv_consumer"], [0], False, True),
|
||||
(["kv_producer", "kv_consumer"], [0], True, False),
|
||||
([None, "kv_consumer"], [0], False, False),
|
||||
(["kv_producer", None], [0], False, False),
|
||||
([None, "kv_producer", "kv_consumer"], [1], False, False),
|
||||
(["kv_producer", "kv_consumer"], [1], False, False),
|
||||
([None, None], [0], False, True),
|
||||
],
|
||||
)
|
||||
def test_native_kv_topology_rejects_silent_legacy_fallback(roles, sources, async_chunk, valid):
|
||||
plans = []
|
||||
for stage_id, role in enumerate(roles):
|
||||
diffusion = stage_id == len(roles) - 1
|
||||
config = types.SimpleNamespace(kv_role=role) if role else None
|
||||
replica = types.SimpleNamespace(
|
||||
metadata=types.SimpleNamespace(stage_type="diffusion" if diffusion else "llm", engine_input_source=sources),
|
||||
stage_vllm_config=None if diffusion else types.SimpleNamespace(kv_transfer_config=config),
|
||||
stage_cfg=types.SimpleNamespace(engine_args={"kv_transfer_config": config}),
|
||||
)
|
||||
plans.append(types.SimpleNamespace(stage_id=stage_id, replicas=[replica]))
|
||||
runtime = object.__new__(StageRuntime)
|
||||
runtime._async_chunk = async_chunk
|
||||
if valid:
|
||||
runtime._validate_native_kv_topology(plans)
|
||||
else:
|
||||
with pytest.raises(ValueError, match="two-stage"):
|
||||
runtime._validate_native_kv_topology(plans)
|
||||
@@ -22,6 +22,7 @@ from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
from vllm_omni.engine import OmniEngineCoreOutput
|
||||
from vllm_omni.engine.errors import NativeKVHandoffError
|
||||
from vllm_omni.engine.messages import (
|
||||
AbortRequestMessage,
|
||||
AbortResultMessage,
|
||||
@@ -1414,6 +1415,105 @@ async def test_handle_streaming_update_unknown_request_is_dropped() -> None:
|
||||
assert "req-unknown" not in orchestrator.request_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("native_kv", [False, True])
|
||||
async def test_add_request_attaches_native_kv_ticket_before_dispatch(mocker, native_kv) -> None:
|
||||
sampling = SamplingParams(extra_args={"keep": "sampling"})
|
||||
prompt = SimpleNamespace(sampling_params=SamplingParams(extra_args={"keep": "prompt"}))
|
||||
source = StagePool(
|
||||
0,
|
||||
FakeStageClient(),
|
||||
stage_vllm_config=SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(kv_role="kv_producer") if native_kv else None
|
||||
),
|
||||
)
|
||||
target = StagePool(1, FakeStageClient(stage_type="diffusion", final_output=True))
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=asyncio.Queue(),
|
||||
stage_pools=[source, target],
|
||||
)
|
||||
|
||||
async def check_dispatch(request_id, req_state, request, **kwargs):
|
||||
assert req_state.native_kv_transfer_id == ("xfer-req" if native_kv else None)
|
||||
for params, preserved in ((sampling, "sampling"), (request.sampling_params, "prompt")):
|
||||
assert params.extra_args["keep"] == preserved
|
||||
if native_kv:
|
||||
assert params.extra_args["kv_transfer_params"] == {
|
||||
"transfer_id": "xfer-req",
|
||||
"do_remote_decode": True,
|
||||
"do_remote_prefill": False,
|
||||
}
|
||||
else:
|
||||
assert "kv_transfer_params" not in params.extra_args
|
||||
return 0
|
||||
|
||||
dispatch = mocker.patch.object(source, "submit_initial", side_effect=check_dispatch)
|
||||
await orchestrator._handle_add_request(
|
||||
StageSubmissionMessage(
|
||||
type="add_request",
|
||||
request_id="req",
|
||||
prompt=prompt,
|
||||
original_prompt={},
|
||||
output_prompt_text=None,
|
||||
sampling_params_list=[sampling, OmniDiffusionSamplingParams()],
|
||||
final_stage_id=1,
|
||||
preprocess_ms=0,
|
||||
request_timestamp=0,
|
||||
enqueue_ts=0,
|
||||
)
|
||||
)
|
||||
dispatch.assert_awaited_once()
|
||||
|
||||
|
||||
def test_native_handoff_uses_bound_replica_and_reports_missing_binding(mocker) -> None:
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.stage_pools = [mocker.Mock()]
|
||||
orchestrator.stage_pools[0].get_bound_client.return_value = None
|
||||
req_state = SimpleNamespace(native_kv_transfer_id="xfer-req")
|
||||
output = SimpleNamespace(kv_transfer_params={"num_transfer_tokens": 4})
|
||||
|
||||
with pytest.raises(NativeKVHandoffError, match="bound AR replica"):
|
||||
orchestrator._diffusion_submit_kwargs("req", 0, SimpleNamespace(engine_input_source=[0]), req_state, output)
|
||||
orchestrator.stage_pools[0].get_bound_client.assert_called_once_with("req")
|
||||
# The pool default points at replica 0; this request actually used replica 1.
|
||||
orchestrator.stage_pools[0].stage_vllm_config = SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(engine_id="ar-0")
|
||||
)
|
||||
config = SimpleNamespace(engine_id="ar-1", kv_connector_extra_config={"bootstrap_addr": "http://host:8999"})
|
||||
orchestrator.stage_pools[0].get_bound_client.return_value = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=config)
|
||||
)
|
||||
params = orchestrator._diffusion_submit_kwargs(
|
||||
"req", 0, SimpleNamespace(engine_input_source=[0]), req_state, output
|
||||
)
|
||||
assert params["kv_transfer_params"]["remote_engine_id"] == "ar-1"
|
||||
assert params["kv_transfer_params"]["remote_bootstrap_addr"] == "http://host:8999"
|
||||
assert "remote_engine_id" not in output.kv_transfer_params
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_handoff_failure_is_request_scoped(mocker):
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
fail = mocker.patch.object(orchestrator, "_fail_request_client_error", new_callable=mocker.AsyncMock)
|
||||
|
||||
def lost_binding():
|
||||
raise NativeKVHandoffError("bound AR replica is unavailable")
|
||||
|
||||
assert not await orchestrator._dispatch_or_fail_request(
|
||||
lost_binding, req_id="req", stage_id=1, operation="inter-stage forward"
|
||||
)
|
||||
fail.assert_awaited_once_with(
|
||||
"req",
|
||||
1,
|
||||
"bound AR replica is unavailable",
|
||||
status_code=502,
|
||||
error_type="NativeKVHandoffError",
|
||||
release_owners=True,
|
||||
)
|
||||
|
||||
|
||||
async def test_handle_streaming_update_passes_prompt_text_to_stage_pool() -> None:
|
||||
class RecordingPool:
|
||||
def __init__(self) -> None:
|
||||
|
||||
Reference in new issue
Block a user