[Bugfix] Restore BAGEL CFG KV transfers (#5620)
Signed-off-by: lishunyang12 <lishunyang03@gmail.com> Co-authored-by: lishunyang12 <lishunyang03@gmail.com>
This commit is contained in:
1 parent
185e3d6df6
commit
6f98c807d2
8 files changed
+160
-8
No files matched your search
@@ -122,6 +122,19 @@ class OmniConnectorModelRunnerMixin:
|
||||
|
||||
# -- heterogeneous TP rank support --
|
||||
rank_cfg = self._parse_rank_mapping(model_config)
|
||||
if self._kv_transfer_manager is not None:
|
||||
topology = getattr(self._kv_transfer_manager, "tp_topology", None)
|
||||
effective_mapping = (
|
||||
getattr(topology, "source_tp_size", None),
|
||||
getattr(topology, "target_tp_size", None),
|
||||
getattr(topology, "local_rank", None),
|
||||
)
|
||||
if all(isinstance(value, int) for value in effective_mapping):
|
||||
rank_cfg = {
|
||||
"from_tp": effective_mapping[0],
|
||||
"to_tp": effective_mapping[1],
|
||||
"local_rank": effective_mapping[2],
|
||||
}
|
||||
self._from_tp: int = rank_cfg["from_tp"]
|
||||
self._to_tp: int = rank_cfg["to_tp"]
|
||||
self._local_rank: int = rank_cfg["local_rank"]
|
||||
@@ -1275,6 +1288,10 @@ class OmniConnectorModelRunnerMixin:
|
||||
For heterogeneous TP receive, the local rank is the target rank and must
|
||||
fetch one or more source-rank shards keyed as ``from_rank -> to_rank``.
|
||||
"""
|
||||
if self._from_tp <= 1 and self._to_tp <= 1:
|
||||
resolved_to_stage = self._next_stage_id if to_stage is None else to_stage
|
||||
return [f"omni_{from_stage}_to_{resolved_to_stage}_kv_cache_{req_id}"]
|
||||
|
||||
remote_ranks = self.get_kv_remote_ranks()
|
||||
return [
|
||||
self.get_kv_connector_key(
|
||||
@@ -1307,6 +1324,10 @@ class OmniConnectorModelRunnerMixin:
|
||||
chunk_id: int = 0,
|
||||
) -> list[str]:
|
||||
"""Build send-side connector keys for this rank's KV shard(s)."""
|
||||
if self._from_tp <= 1 and self._to_tp <= 1:
|
||||
resolved_to_stage = self._next_stage_id if to_stage is None else to_stage
|
||||
return [f"omni_{from_stage}_to_{resolved_to_stage}_kv_cache_{req_id}"]
|
||||
|
||||
target_ranks = self.get_kv_target_ranks_for_send()
|
||||
return [
|
||||
self.get_kv_connector_key(
|
||||
|
||||
Reference in new issue
Block a user