[Bugfix] Restore BAGEL CFG KV transfers (#5620)
Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
This commit is contained in:
1 parent
185e3d6df6
commit
6f98c807d2
8 files changed
+160
-8
No files matched your search
@@ -147,7 +147,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
return None
|
||||
|
||||
def _request_omits_kv_transfer_to_next_stage(self, request: Request) -> bool:
|
||||
"""True when orchestrator will not run stage 1+ for this request (e.g. text-only).
|
||||
"""True when this stage-zero-final request does not need downstream KV.
|
||||
|
||||
The result is cached per request to avoid repeated deserialization of
|
||||
additional_information on every scheduler tick.
|
||||
@@ -162,7 +162,7 @@ class OmniARScheduler(OmniSchedulerMixin, VLLMScheduler):
|
||||
result = False
|
||||
else:
|
||||
info = deserialize_additional_information(payload)
|
||||
result = info.get("omni_final_stage_id") == 0
|
||||
result = info.get("omni_final_stage_id") == 0 and not bool(info.get("omni_force_kv_transfer", False))
|
||||
|
||||
self._omits_kv_transfer_cache[rid] = result
|
||||
return result
|
||||
|
||||
@@ -403,6 +403,7 @@ class OmniKVTransferManager:
|
||||
self._prefetch_executor: ThreadPoolExecutor | None = (
|
||||
ThreadPoolExecutor(max_workers=1, thread_name_prefix="kv-prefetch") if async_prefetch else None
|
||||
)
|
||||
|
||||
self._prefetch_futures: dict[str, Any] = {}
|
||||
self._bg_copy_stream: current_omni_platform.Stream | None = None
|
||||
|
||||
@@ -415,6 +416,11 @@ class OmniKVTransferManager:
|
||||
except Exception as e:
|
||||
logger.warning("Failed to eagerly initialize sender connector: %s", e)
|
||||
|
||||
@property
|
||||
def tp_topology(self) -> KVTPTopology:
|
||||
"""Return the effective topology after config and runtime detection."""
|
||||
return self._tp_topo
|
||||
|
||||
@property
|
||||
def topo_config(self) -> _TransferTopoConfig:
|
||||
"""Lazily build the parallel topology snapshot on first access."""
|
||||
|
||||
@@ -77,12 +77,17 @@ def upgrade_to_omni_request(
|
||||
def apply_omni_final_stage_metadata(
|
||||
request: EngineCoreRequest,
|
||||
final_stage_id: int,
|
||||
*,
|
||||
force_kv_transfer: bool = False,
|
||||
) -> EngineCoreRequest:
|
||||
"""Tag a request so the AR scheduler can skip DiT KV for stage zero."""
|
||||
"""Tag a request with its final stage and optional KV-transfer override."""
|
||||
merged: dict[str, Any] = {}
|
||||
if isinstance(request, OmniEngineCoreRequest) and request.additional_information is not None:
|
||||
merged = deserialize_additional_information(request.additional_information)
|
||||
merged["omni_final_stage_id"] = final_stage_id
|
||||
merged.pop("omni_force_kv_transfer", None)
|
||||
if force_kv_transfer:
|
||||
merged["omni_force_kv_transfer"] = True
|
||||
payload = serialize_additional_information(merged)
|
||||
return OmniEngineCoreRequest.from_request(
|
||||
request,
|
||||
|
||||
@@ -798,11 +798,9 @@ class AsyncOmniEngine:
|
||||
supported_tasks=self.supported_tasks,
|
||||
)
|
||||
request.external_req_id = cid
|
||||
# Companions are stage-0-final: without this worker-visible tag
|
||||
# the AR runner would still emit downstream connector payloads
|
||||
# for them (the orchestrator-side final_stage_id=0 registration
|
||||
# alone does not reach the worker).
|
||||
request = apply_omni_final_stage_metadata(request, 0)
|
||||
# Companions are stage-0-final for ordinary downstream payloads,
|
||||
# but diffusion still needs their CFG KV caches.
|
||||
request = apply_omni_final_stage_metadata(request, 0, force_kv_transfer=True)
|
||||
|
||||
# Registration of this companion on stage-0's output processor is
|
||||
# deferred to Orchestrator._handle_add_companion, which routes
|
||||
|
||||
@@ -122,6 +122,19 @@ class OmniConnectorModelRunnerMixin:
|
||||
|
||||
# -- heterogeneous TP rank support --
|
||||
rank_cfg = self._parse_rank_mapping(model_config)
|
||||
if self._kv_transfer_manager is not None:
|
||||
topology = getattr(self._kv_transfer_manager, "tp_topology", None)
|
||||
effective_mapping = (
|
||||
getattr(topology, "source_tp_size", None),
|
||||
getattr(topology, "target_tp_size", None),
|
||||
getattr(topology, "local_rank", None),
|
||||
)
|
||||
if all(isinstance(value, int) for value in effective_mapping):
|
||||
rank_cfg = {
|
||||
"from_tp": effective_mapping[0],
|
||||
"to_tp": effective_mapping[1],
|
||||
"local_rank": effective_mapping[2],
|
||||
}
|
||||
self._from_tp: int = rank_cfg["from_tp"]
|
||||
self._to_tp: int = rank_cfg["to_tp"]
|
||||
self._local_rank: int = rank_cfg["local_rank"]
|
||||
@@ -1275,6 +1288,10 @@ class OmniConnectorModelRunnerMixin:
|
||||
For heterogeneous TP receive, the local rank is the target rank and must
|
||||
fetch one or more source-rank shards keyed as ``from_rank -> to_rank``.
|
||||
"""
|
||||
if self._from_tp <= 1 and self._to_tp <= 1:
|
||||
resolved_to_stage = self._next_stage_id if to_stage is None else to_stage
|
||||
return [f"omni_{from_stage}_to_{resolved_to_stage}_kv_cache_{req_id}"]
|
||||
|
||||
remote_ranks = self.get_kv_remote_ranks()
|
||||
return [
|
||||
self.get_kv_connector_key(
|
||||
@@ -1307,6 +1324,10 @@ class OmniConnectorModelRunnerMixin:
|
||||
chunk_id: int = 0,
|
||||
) -> list[str]:
|
||||
"""Build send-side connector keys for this rank's KV shard(s)."""
|
||||
if self._from_tp <= 1 and self._to_tp <= 1:
|
||||
resolved_to_stage = self._next_stage_id if to_stage is None else to_stage
|
||||
return [f"omni_{from_stage}_to_{resolved_to_stage}_kv_cache_{req_id}"]
|
||||
|
||||
target_ranks = self.get_kv_target_ranks_for_send()
|
||||
return [
|
||||
self.get_kv_connector_key(
|
||||
|
||||
Reference in new issue
Block a user