Signed-off-by: Kushan A. <kahmadian@nvidia.com> Signed-off-by: kushanam <kushanam@users.noreply.github.com> Signed-off-by: Kushan A <kahmadian@nvidia.com> Signed-off-by: kahmadian <kahmadian@users.noreply.github.com> Co-authored-by: kushanam <kushanam@users.noreply.github.com> Co-authored-by: kahmadian <kahmadian@users.noreply.github.com>
This commit is contained in:
30 files changed
+2722
-51
No files matched your search
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Composable parallel strategies for vLLM-Omni.
|
||||
|
||||
A small, runtime-agnostic vocabulary for declaring a stage's parallel layout
|
||||
(:class:`StrategySpec` + mesh/routing/aggregation patterns), plus the logic to
|
||||
translate a per-role stack of specs into concrete engine sizing and overlay it
|
||||
onto merged stage configs.
|
||||
|
||||
The spec/pattern *types* are deliberately pure data with no engine imports; the
|
||||
translation and apply logic lives alongside them and consumes vLLM-Omni's own
|
||||
config objects.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.composable_parallel.aggregation import (
|
||||
AGGREGATION_PATTERN_VARIANTS,
|
||||
AggregationConflictError,
|
||||
AggregationPattern,
|
||||
AllGather,
|
||||
Combine,
|
||||
FanInByStage,
|
||||
GatherDim,
|
||||
StitchPipeline,
|
||||
StitchSpatial,
|
||||
TakeRank,
|
||||
Union,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.apply import (
|
||||
StrategyApplyError,
|
||||
StrategyApplyResult,
|
||||
apply_strategy_specs,
|
||||
check_device_layout,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.routing import (
|
||||
ROUTING_PATTERN_VARIANTS,
|
||||
Broadcast,
|
||||
DuplicateWithCondTag,
|
||||
PartitionByHash,
|
||||
PipelineMicrobatch,
|
||||
RouteByExpert,
|
||||
RouteByStage,
|
||||
RoutingKeyError,
|
||||
RoutingPattern,
|
||||
ShardSequence,
|
||||
ShardSpatial,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.spec import (
|
||||
MESH_AXIS_KINDS,
|
||||
KernelSpec,
|
||||
LayerHookSpec,
|
||||
MeshAxisKind,
|
||||
MeshAxisSpec,
|
||||
SpecMergeConflictError,
|
||||
StrategySpec,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.translator import (
|
||||
AxisTranslationError,
|
||||
OmniParallelConfig,
|
||||
translate_strategy_stack,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# spec types
|
||||
"MeshAxisKind",
|
||||
"MESH_AXIS_KINDS",
|
||||
"MeshAxisSpec",
|
||||
"LayerHookSpec",
|
||||
"KernelSpec",
|
||||
"StrategySpec",
|
||||
"SpecMergeConflictError",
|
||||
# routing patterns
|
||||
"RoutingPattern",
|
||||
"RoutingKeyError",
|
||||
"Broadcast",
|
||||
"PartitionByHash",
|
||||
"ShardSequence",
|
||||
"ShardSpatial",
|
||||
"RouteByExpert",
|
||||
"PipelineMicrobatch",
|
||||
"DuplicateWithCondTag",
|
||||
"RouteByStage",
|
||||
"ROUTING_PATTERN_VARIANTS",
|
||||
# aggregation patterns
|
||||
"AggregationPattern",
|
||||
"AggregationConflictError",
|
||||
"TakeRank",
|
||||
"Union",
|
||||
"GatherDim",
|
||||
"AllGather",
|
||||
"StitchSpatial",
|
||||
"StitchPipeline",
|
||||
"Combine",
|
||||
"FanInByStage",
|
||||
"AGGREGATION_PATTERN_VARIANTS",
|
||||
# translation
|
||||
"OmniParallelConfig",
|
||||
"translate_strategy_stack",
|
||||
"AxisTranslationError",
|
||||
# apply
|
||||
"apply_strategy_specs",
|
||||
"StrategyApplyResult",
|
||||
"check_device_layout",
|
||||
"StrategyApplyError",
|
||||
]
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Closed aggregation-pattern hierarchy for declarative parallel strategies.
|
||||
|
||||
An aggregation pattern describes how the per-worker results of one mesh axis
|
||||
are recombined into a single logical result (e.g. take one tensor-parallel
|
||||
rank's output, union disjoint data-parallel shards). The hierarchy is *closed*:
|
||||
subclasses may only be defined in this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AggregationPattern:
|
||||
"""Base class for closed aggregation patterns. Subclass only within this module."""
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||
super().__init_subclass__(**kwargs)
|
||||
if cls.__module__ != __name__:
|
||||
raise TypeError("AggregationPattern is closed to external subclassing")
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class AggregationConflictError(ValueError):
|
||||
"""Raised when two aggregation inputs conflict for the same request id."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TakeRank(AggregationPattern):
|
||||
"""Select one rank's result (TP — all ranks agree)."""
|
||||
|
||||
rank: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.rank, int) or isinstance(self.rank, bool):
|
||||
raise TypeError(f"TakeRank.rank must be int, got {type(self.rank).__name__}")
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "take_rank"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Union(AggregationPattern):
|
||||
"""Union of disjoint replica results (DP, HSDP)."""
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "union"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatherDim(AggregationPattern):
|
||||
"""Gather along a dimension and materialize on one designated rank.
|
||||
|
||||
Default designated rank is axis-rank 0 unless adapter overrides.
|
||||
"""
|
||||
|
||||
dim: int = 1
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "gather_dim"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AllGather(AggregationPattern):
|
||||
"""All ranks materialize the same aggregate result across the axis."""
|
||||
|
||||
dim: int | None = None
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "all_gather"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StitchSpatial(AggregationPattern):
|
||||
"""Stitch spatial patches (VAE-PP)."""
|
||||
|
||||
grid: tuple[int, int] = (1, 1)
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "stitch_spatial"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StitchPipeline(AggregationPattern):
|
||||
"""Stitch pipeline stage outputs (PP)."""
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "stitch_pipeline"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Combine(AggregationPattern):
|
||||
"""Custom reducer combination (CFG). Adapter resolves reducer_id."""
|
||||
|
||||
reducer_id: str = "default"
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "combine"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FanInByStage(AggregationPattern):
|
||||
"""Fan-in results routed by stage (stage replicas)."""
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "fan_in_by_stage"
|
||||
|
||||
|
||||
AGGREGATION_PATTERN_VARIANTS: tuple[type[AggregationPattern], ...] = (
|
||||
TakeRank,
|
||||
Union,
|
||||
GatherDim,
|
||||
AllGather,
|
||||
StitchSpatial,
|
||||
StitchPipeline,
|
||||
Combine,
|
||||
FanInByStage,
|
||||
)
|
||||
|
||||
ReducerFn = Callable[..., object]
|
||||
@@ -0,0 +1,264 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Apply declarative parallel strategies onto merged stage configs.
|
||||
|
||||
This is the *override-after-merge* seam. ``merge_pipeline_deploy`` first fuses
|
||||
the pipeline topology + deploy YAML into a list of ``StageConfig`` objects; this
|
||||
module then overlays the engine sizing derived from a per-role
|
||||
``StrategySpec`` stack onto those stages.
|
||||
|
||||
Design rules (one writer per axis):
|
||||
|
||||
* A strategy only writes the axes it actually declares. If a stack has no ``tp``
|
||||
axis, this module never touches ``tensor_parallel_size`` — it does not force a
|
||||
default of 1.
|
||||
* **Conflict-on-explicit.** If a knob was already set explicitly by the deploy
|
||||
YAML and the strategy derives a different value, we raise rather than silently
|
||||
override. Equal values are a no-op; unset values are filled.
|
||||
* **Pre-spawn device check.** When a stage declares ``devices``, the device
|
||||
count must be consistent with the engine world size (``tp * dp * pp``) and the
|
||||
replica count, so misconfigurations fail at config time instead of at spawn.
|
||||
|
||||
``omni_lb_policy`` is *not* a per-stage config knob: omni reads it once at
|
||||
orchestrator construction (``AsyncOmniEngine``), not from stage configs. So a
|
||||
stage_replica axis's derived policy is surfaced on :class:`StrategyApplyResult`
|
||||
for the caller to pass to the orchestrator; it is never silently written into a
|
||||
stage where it would be ignored.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, NoReturn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.config.composable_parallel.spec import StrategySpec
|
||||
from vllm_omni.config.composable_parallel.translator import (
|
||||
OmniParallelConfig,
|
||||
translate_strategy_stack,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Role key -> the StageConfig fields each declared axis writes.
|
||||
_ENGINE_FIELD_BY_KIND: dict[str, str] = {
|
||||
"tp": "tensor_parallel_size",
|
||||
"dp": "data_parallel_size",
|
||||
"pp": "pipeline_parallel_size",
|
||||
}
|
||||
|
||||
RoleKey = str
|
||||
|
||||
|
||||
class StrategyApplyError(ValueError):
|
||||
"""Error for applying a strategy onto stage configs.
|
||||
|
||||
A single error type (rather than per-cause subclasses) keeps the public
|
||||
surface small and consistent with the rest of the codebase; the specific
|
||||
cause — an unmatched role, a conflict with an explicit deploy value, or a
|
||||
device-count mismatch — is in the message and is logged before the raise so
|
||||
it stays visible even when the type is unavailable to a caller (e.g. across a
|
||||
server boundary).
|
||||
"""
|
||||
|
||||
|
||||
def _fail(msg: str) -> NoReturn:
|
||||
"""Log and raise :class:`StrategyApplyError`."""
|
||||
logger.error("[composable_parallel] %s", msg)
|
||||
raise StrategyApplyError(msg)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyApplyResult:
|
||||
"""Outcome of applying strategies to a stage list.
|
||||
|
||||
``stages`` is the same list passed in (mutated in place and returned for
|
||||
convenience). ``omni_lb_policy`` is the pipeline-wide load-balancer policy
|
||||
derived from any stage_replica axes (``None`` if none declared one); the
|
||||
caller wires it into the orchestrator, since omni does not read it from
|
||||
stage configs.
|
||||
|
||||
``per_role_config`` is keyed by the role key the caller supplied — a
|
||||
``model_stage`` name (str). ``per_stage_config`` is the same information
|
||||
keyed by the resolved integer ``stage_id`` — handy for re-validating a
|
||||
stage after later layers (e.g. CLI overrides) have run.
|
||||
"""
|
||||
|
||||
stages: list[Any]
|
||||
omni_lb_policy: str | None = None
|
||||
per_role_config: dict[RoleKey, OmniParallelConfig] = field(default_factory=dict)
|
||||
per_stage_config: dict[int, OmniParallelConfig] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _resolve_stage(stages: Sequence[Any], key: RoleKey) -> Any:
|
||||
"""Find the single stage matching ``key`` by ``model_stage`` name.
|
||||
|
||||
Role keys are ``model_stage`` names (str) — the same human-readable names
|
||||
that strategy.yaml authors use to refer to a stage (e.g. ``"thinker"``).
|
||||
"""
|
||||
matches = [s for s in stages if getattr(s, "model_stage", None) == key]
|
||||
descriptor = f"model_stage={key!r}"
|
||||
|
||||
if not matches:
|
||||
available = ", ".join(f"{getattr(s, 'model_stage', '?')!r}(id={getattr(s, 'stage_id', '?')})" for s in stages)
|
||||
_fail(f"strategy role {descriptor} did not match any stage; available stages: {available}")
|
||||
if len(matches) > 1:
|
||||
_fail(f"strategy role {descriptor} is ambiguous; it matched {len(matches)} stages")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _set_explicit(engine_args: dict[str, Any], field_name: str, value: Any, *, role: RoleKey) -> None:
|
||||
"""Write ``field_name`` with conflict-on-explicit semantics.
|
||||
|
||||
``engine_args`` is the raw mapping loaded straight from the deploy YAML
|
||||
(before the engine-args dataclass is constructed), so a key is *present* only
|
||||
when the YAML set it explicitly. We therefore key off membership rather than
|
||||
``.get``: an explicit ``None`` in the YAML is a real, present value and must
|
||||
not be conflated with a key miss (which ``.get`` would report as ``None``).
|
||||
"""
|
||||
if field_name in engine_args and engine_args[field_name] != value:
|
||||
existing = engine_args[field_name]
|
||||
_fail(
|
||||
f"role {role!r}: strategy derives {field_name}={value} but the deploy config already "
|
||||
f"set {field_name}={existing}. Remove one of them (the strategy is the single writer "
|
||||
"for the axes it declares)."
|
||||
)
|
||||
engine_args[field_name] = value
|
||||
|
||||
|
||||
def _set_num_replicas(runtime: dict[str, Any], value: int, *, role: RoleKey) -> None:
|
||||
"""Write ``num_replicas`` with conflict-on-explicit (treating 1 as unset)."""
|
||||
existing = runtime.get("num_replicas", 1)
|
||||
if existing not in (1, value):
|
||||
_fail(
|
||||
f"role {role!r}: strategy derives num_replicas={value} but the deploy config already "
|
||||
f"set num_replicas={existing}. Remove one of them."
|
||||
)
|
||||
runtime["num_replicas"] = value
|
||||
|
||||
|
||||
def _parse_device_count(devices: Any) -> int | None:
|
||||
"""Return the number of device ids in a ``devices`` value, or None."""
|
||||
if devices is None:
|
||||
return None
|
||||
if isinstance(devices, (list, tuple)):
|
||||
return len([d for d in devices])
|
||||
text = str(devices).strip()
|
||||
if not text:
|
||||
return None
|
||||
return len([d for d in text.split(",") if d.strip()])
|
||||
|
||||
|
||||
def check_device_layout(
|
||||
devices: Any,
|
||||
*,
|
||||
tensor_parallel_size: int,
|
||||
data_parallel_size: int,
|
||||
pipeline_parallel_size: int,
|
||||
num_replicas: int,
|
||||
role: RoleKey,
|
||||
) -> None:
|
||||
"""Validate a stage's device count against its world size (× replicas in pool mode).
|
||||
|
||||
Raises :class:`StrategyApplyError` when the declared ``devices`` count is
|
||||
neither the per-replica world size nor the full ``num_replicas`` pool.
|
||||
Exposed publicly so the resolved (post-CLI) layout can be re-checked after
|
||||
later override layers, not just the strategy-derived snapshot.
|
||||
"""
|
||||
count = _parse_device_count(devices)
|
||||
if count is None:
|
||||
return
|
||||
world = int(tensor_parallel_size) * int(data_parallel_size) * int(pipeline_parallel_size)
|
||||
replicas = int(num_replicas or 1)
|
||||
# Two valid shapes: a single per-replica template (== world) or the full
|
||||
# pool across replicas (== replicas * world). This mirrors omni's own
|
||||
# per-replica device splitter.
|
||||
valid = {world, replicas * world}
|
||||
if count not in valid:
|
||||
_fail(
|
||||
f"role {role!r}: declared {count} device id(s) but the strategy world size is {world} "
|
||||
f"(tp={tensor_parallel_size} * dp={data_parallel_size} * pp={pipeline_parallel_size}). "
|
||||
f"Provide either {world} (per-replica template) or {replicas * world} "
|
||||
f"(num_replicas={replicas} pool)."
|
||||
)
|
||||
|
||||
|
||||
def _check_devices(runtime: dict[str, Any], cfg: OmniParallelConfig, *, role: RoleKey) -> None:
|
||||
"""Pre-spawn check on the strategy-derived snapshot (see :func:`check_device_layout`)."""
|
||||
check_device_layout(
|
||||
runtime.get("devices"),
|
||||
tensor_parallel_size=cfg.tensor_parallel_size,
|
||||
data_parallel_size=cfg.data_parallel_size,
|
||||
pipeline_parallel_size=cfg.pipeline_parallel_size,
|
||||
num_replicas=int(runtime.get("num_replicas", 1) or 1),
|
||||
role=role,
|
||||
)
|
||||
|
||||
|
||||
def _apply_to_stage(stage: Any, cfg: OmniParallelConfig, *, role: RoleKey) -> None:
|
||||
engine_args = stage.yaml_engine_args
|
||||
runtime = stage.yaml_runtime
|
||||
|
||||
declared = set(cfg.l1_owners.keys())
|
||||
for kind, field_name in _ENGINE_FIELD_BY_KIND.items():
|
||||
if kind in declared:
|
||||
_set_explicit(engine_args, field_name, getattr(cfg, field_name), role=role)
|
||||
if "ep" in declared and cfg.enable_expert_parallel:
|
||||
_set_explicit(engine_args, "enable_expert_parallel", True, role=role)
|
||||
if "stage_replica" in declared:
|
||||
_set_num_replicas(runtime, cfg.stage_replica_size, role=role)
|
||||
|
||||
_check_devices(runtime, cfg, role=role)
|
||||
|
||||
|
||||
def apply_strategy_specs(
|
||||
stages: list[Any],
|
||||
strategy_specs: Mapping[RoleKey, Sequence[StrategySpec]],
|
||||
) -> StrategyApplyResult:
|
||||
"""Overlay per-role strategy specs onto a merged stage list.
|
||||
|
||||
Args:
|
||||
stages: the ``list[StageConfig]`` returned by ``merge_pipeline_deploy``.
|
||||
strategy_specs: maps a role (a ``model_stage`` name, e.g. ``"thinker"``)
|
||||
to that stage's stack of ``StrategySpec`` (one per declared mesh
|
||||
axis). Role keys are ``model_stage`` names (str).
|
||||
|
||||
Returns:
|
||||
A :class:`StrategyApplyResult` with the mutated stages and the derived
|
||||
pipeline-wide ``omni_lb_policy`` (if any stage_replica axis set one).
|
||||
|
||||
Raises:
|
||||
StrategyApplyError: a role matched zero or multiple stages, a derived
|
||||
value conflicts with an explicit deploy value (including two roles
|
||||
deriving different ``omni_lb_policy`` values), or a stage's device
|
||||
count is inconsistent with its world size.
|
||||
AxisTranslationError: the spec stack is invalid/unsupported.
|
||||
NotImplementedError: the spec stack requests routing not built yet.
|
||||
"""
|
||||
result = StrategyApplyResult(stages=stages)
|
||||
lb_policy: str | None = None
|
||||
lb_owner: RoleKey | None = None
|
||||
|
||||
for key, specs in strategy_specs.items():
|
||||
stage = _resolve_stage(stages, key)
|
||||
cfg = translate_strategy_stack(specs)
|
||||
_apply_to_stage(stage, cfg, role=key)
|
||||
result.per_role_config[key] = cfg
|
||||
stage_id = getattr(stage, "stage_id", None)
|
||||
if stage_id is not None:
|
||||
result.per_stage_config[int(stage_id)] = cfg
|
||||
|
||||
if cfg.stage_replica_size > 1 and cfg.omni_lb_policy is not None:
|
||||
if lb_policy is not None and lb_policy != cfg.omni_lb_policy:
|
||||
_fail(
|
||||
f"role {key!r} derives omni_lb_policy={cfg.omni_lb_policy!r} but role "
|
||||
f"{lb_owner!r} already derived {lb_policy!r}. The load-balancer policy is "
|
||||
"pipeline-wide; only one value is allowed."
|
||||
)
|
||||
lb_policy = cfg.omni_lb_policy
|
||||
lb_owner = key
|
||||
|
||||
result.omni_lb_policy = lb_policy
|
||||
return result
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Closed routing-pattern hierarchy for declarative parallel strategies.
|
||||
|
||||
A routing pattern describes how a request batch is distributed across the
|
||||
workers of one mesh axis (e.g. broadcast to every tensor-parallel rank, hash
|
||||
partition across data-parallel ranks). The hierarchy is *closed*: subclasses
|
||||
may only be defined in this module, so the set of patterns is a fixed,
|
||||
exhaustively matchable contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoutingPattern:
|
||||
"""Base class for closed routing patterns. Subclass only within this module."""
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||
super().__init_subclass__(**kwargs)
|
||||
if cls.__module__ != __name__:
|
||||
raise TypeError("RoutingPattern is closed to external subclassing")
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RoutingKeyError(KeyError):
|
||||
"""Raised when a routing key is missing from a request."""
|
||||
|
||||
def __init__(self, request_id: object, missing_key: str) -> None:
|
||||
self.request_id = request_id
|
||||
self.missing_key = missing_key
|
||||
super().__init__(f"Missing routing key {missing_key!r} for request_id={request_id!r}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Broadcast(RoutingPattern):
|
||||
"""Every worker in the group sees the whole batch (TP, dense EP)."""
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "broadcast"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PartitionByHash(RoutingPattern):
|
||||
"""Stable hash partition by request key (DP, HSDP)."""
|
||||
|
||||
key: str = "request_id"
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "partition_by_hash"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ShardSequence(RoutingPattern):
|
||||
"""Shard along a sequence dimension (SP-Ulysses, SP-Ring, CP)."""
|
||||
|
||||
dim: int = 1
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "shard_sequence"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ShardSpatial(RoutingPattern):
|
||||
"""Shard along a spatial grid (VAE-PP)."""
|
||||
|
||||
grid: tuple[int, int] = (1, 1)
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "shard_spatial"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteByExpert(RoutingPattern):
|
||||
"""Sparse MoE expert routing."""
|
||||
|
||||
router_id: str = "default"
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "route_by_expert"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineMicrobatch(RoutingPattern):
|
||||
"""Pipeline parallelism microbatch routing (PP)."""
|
||||
|
||||
microbatch_size: int = 1
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "pipeline_microbatch"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DuplicateWithCondTag(RoutingPattern):
|
||||
"""Duplicate batch with a conditional tag (CFG)."""
|
||||
|
||||
flag_name: str = "guidance_branch"
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "duplicate_with_cond_tag"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteByStage(RoutingPattern):
|
||||
"""Route requests by pipeline stage policy (stage replicas).
|
||||
|
||||
The ``random``/``round_robin``/``least_queue`` policies describe stateless
|
||||
load balancing that a runtime (e.g. an engine's built-in DP load balancer,
|
||||
or the omni StagePool balancer) can realize. ``hash`` describes
|
||||
deterministic, key-stable routing that a load balancer cannot honor on its
|
||||
own (see ``PartitionByHash``).
|
||||
"""
|
||||
|
||||
routing_policy: Literal["random", "round_robin", "least_queue", "hash"] = "round_robin"
|
||||
|
||||
def pattern_kind(self) -> str:
|
||||
return "route_by_stage"
|
||||
|
||||
|
||||
ROUTING_PATTERN_VARIANTS: tuple[type[RoutingPattern], ...] = (
|
||||
Broadcast,
|
||||
PartitionByHash,
|
||||
ShardSequence,
|
||||
ShardSpatial,
|
||||
RouteByExpert,
|
||||
PipelineMicrobatch,
|
||||
DuplicateWithCondTag,
|
||||
RouteByStage,
|
||||
)
|
||||
|
||||
# Optional callable slot for Combine-style custom routing reducers in adapters.
|
||||
RouterFn = Callable[..., object]
|
||||
@@ -0,0 +1,136 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Core declarative types for parallel-strategy specification.
|
||||
|
||||
A :class:`StrategySpec` declares one parallelism scheme: the mesh axis it sizes
|
||||
(kind + degree), how a batch is routed across that axis, and how per-worker
|
||||
results are aggregated back. A *stack* of specs (one per axis) fully describes a
|
||||
stage's parallel layout in a runtime-agnostic, data-only form that can be
|
||||
translated into concrete engine sizing.
|
||||
|
||||
The mesh-axis kinds below enumerate every parallelism dimension the contract can
|
||||
describe. Only a subset is wired end-to-end today (see ``translator.py``):
|
||||
|
||||
* **Wired** (translatable now): ``tp``, ``dp``, ``pp``, ``ep``,
|
||||
``stage_replica``.
|
||||
* **Reserved** (declared in the type system but not yet translatable — the
|
||||
translator raises ``AxisTranslationError`` for them): ``sp_ulysses``,
|
||||
``sp_ring``, ``cfg``, ``vae_pp``, ``hsdp``, ``stage_pp``, ``cp``.
|
||||
|
||||
Reserved kinds fail fast at translation time rather than silently doing nothing,
|
||||
so declaring one is an explicit "not yet" rather than a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Literal
|
||||
|
||||
from vllm_omni.config.composable_parallel.aggregation import AggregationPattern
|
||||
from vllm_omni.config.composable_parallel.routing import RoutingPattern
|
||||
|
||||
|
||||
class MeshAxisKind(str, Enum):
|
||||
"""Enumerates every parallelism dimension the strategy contract can describe.
|
||||
|
||||
Subclassing ``str`` keeps existing string comparisons and dict keys working
|
||||
(``MeshAxisKind.TP == "tp"`` and both hash the same), while making the type
|
||||
refactor-safe: a typo'd kind is an ``AttributeError`` on the enum rather than
|
||||
a silently-wrong bare string.
|
||||
"""
|
||||
|
||||
TP = "tp"
|
||||
DP = "dp"
|
||||
PP = "pp"
|
||||
EP = "ep"
|
||||
SP_ULYSSES = "sp_ulysses"
|
||||
SP_RING = "sp_ring"
|
||||
CFG = "cfg"
|
||||
VAE_PP = "vae_pp"
|
||||
HSDP = "hsdp"
|
||||
STAGE_PP = "stage_pp"
|
||||
STAGE_REPLICA = "stage_replica"
|
||||
CP = "cp"
|
||||
|
||||
|
||||
# Bare string values, kept as a plain tuple so membership checks and error
|
||||
# messages read as the kind names (``"tp"``) rather than enum reprs. ``str``-enum
|
||||
# equality means both ``"tp"`` and ``MeshAxisKind.TP`` satisfy ``in`` checks.
|
||||
MESH_AXIS_KINDS: tuple[str, ...] = tuple(k.value for k in MeshAxisKind)
|
||||
|
||||
HookCategory = Literal[
|
||||
"linear",
|
||||
"attention_pre",
|
||||
"attention_post",
|
||||
"ffn_pre",
|
||||
"ffn_post",
|
||||
"moe_dispatch",
|
||||
"other",
|
||||
]
|
||||
|
||||
HOOK_CATEGORY_ORDER: dict[HookCategory, int] = {
|
||||
"linear": 0,
|
||||
"attention_pre": 1,
|
||||
"attention_post": 2,
|
||||
"ffn_pre": 3,
|
||||
"ffn_post": 4,
|
||||
"moe_dispatch": 5,
|
||||
"other": 6,
|
||||
}
|
||||
|
||||
|
||||
class SpecMergeConflictError(ValueError):
|
||||
"""Raised when merged hook/kernel specs have conflicting declarations."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MeshAxisSpec:
|
||||
"""Declares one axis of a process mesh (kind + size)."""
|
||||
|
||||
kind: MeshAxisKind
|
||||
size: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.kind not in MESH_AXIS_KINDS:
|
||||
raise ValueError(f"MeshAxisSpec.kind must be one of {MESH_AXIS_KINDS}, got {self.kind!r}")
|
||||
if self.size <= 0:
|
||||
raise ValueError(f"MeshAxisSpec.size must be > 0, got {self.size}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LayerHookSpec:
|
||||
"""L2 hook slot referenced by StrategySpec; resolved by the model walker."""
|
||||
|
||||
hook_id: str
|
||||
target: str | None = None
|
||||
category: HookCategory = "other"
|
||||
priority: int = 0
|
||||
axis_index: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KernelSpec:
|
||||
"""L3 kernel slot referenced by StrategySpec; resolved by the kernel registry."""
|
||||
|
||||
kernel_id: str
|
||||
target: str | None = None
|
||||
category: HookCategory = "other"
|
||||
priority: int = 0
|
||||
axis_index: int = 0
|
||||
group_axis_kind: MeshAxisKind | None = None
|
||||
requires_collective: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StrategySpec:
|
||||
"""Declarative contract for one parallelism scheme (data + hook/kernel slots)."""
|
||||
|
||||
name: str
|
||||
mesh_axis: MeshAxisSpec
|
||||
routing: RoutingPattern
|
||||
aggregation: AggregationPattern
|
||||
layer_hook_specs: tuple[LayerHookSpec, ...] = ()
|
||||
kernel_specs: tuple[KernelSpec, ...] = ()
|
||||
shard_extension: Mapping[str, Any] = field(default_factory=dict)
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Load per-role parallel strategies from a strategy file.
|
||||
|
||||
The strategy file is a small, optional companion to the deploy config. It maps a
|
||||
logical role (``model_stage``, e.g. ``thinker``) to a list of axis declarations,
|
||||
each of which becomes one :class:`StrategySpec`. YAML is the only format wired up
|
||||
today, but the module/symbol names stay format-agnostic so a different source can
|
||||
be added without renaming the public surface::
|
||||
|
||||
strategies:
|
||||
thinker:
|
||||
- axis: tp
|
||||
size: 2
|
||||
talker:
|
||||
- axis: stage_replica
|
||||
size: 2
|
||||
routing: round_robin # random | round_robin | least_queue
|
||||
code2wav:
|
||||
- axis: stage_replica
|
||||
size: 2
|
||||
routing: round_robin
|
||||
|
||||
Per-axis fields:
|
||||
|
||||
* ``axis`` (required): mesh-axis kind (``tp``, ``dp``, ``pp``, ``ep``,
|
||||
``stage_replica`` are translatable today).
|
||||
* ``size`` (required): axis degree (> 0).
|
||||
* ``routing`` (optional): stateless policy for ``dp`` / ``stage_replica`` axes
|
||||
(``random`` / ``round_robin`` / ``least_queue``). Ignored for axes whose
|
||||
routing is fixed (``tp`` broadcast, ``pp`` microbatch).
|
||||
* ``l1_owner`` (optional): ``engine`` or ``delegated``; overrides the default
|
||||
owner for the axis (recorded in ``shard_extension``).
|
||||
* ``name`` (optional): a label for the spec; defaults to the axis kind.
|
||||
|
||||
Routing/aggregation patterns are filled in from sensible per-kind defaults so
|
||||
the file stays compact; the translator validates the combination.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from vllm_omni.config.composable_parallel.aggregation import (
|
||||
AggregationPattern,
|
||||
FanInByStage,
|
||||
StitchPipeline,
|
||||
TakeRank,
|
||||
Union,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.routing import (
|
||||
Broadcast,
|
||||
PipelineMicrobatch,
|
||||
RouteByStage,
|
||||
RoutingPattern,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.spec import MeshAxisSpec, StrategySpec
|
||||
from vllm_omni.config.yaml_util import load_yaml_config, to_dict
|
||||
|
||||
# Per-kind defaults for routing + aggregation, so the file only needs axis+size.
|
||||
_ROUTING_POLICY_KINDS = frozenset({"dp", "stage_replica"})
|
||||
|
||||
|
||||
class StrategyLoadError(ValueError):
|
||||
"""Raised when a strategy file is malformed."""
|
||||
|
||||
|
||||
def _default_routing(kind: str, routing_policy: str | None) -> RoutingPattern:
|
||||
if kind in _ROUTING_POLICY_KINDS:
|
||||
return RouteByStage(routing_policy=routing_policy or "round_robin") # type: ignore[arg-type]
|
||||
if kind == "pp":
|
||||
return PipelineMicrobatch()
|
||||
# tp, ep, and any other kind default to broadcast (the translator rejects
|
||||
# kinds it cannot handle before it ever inspects routing).
|
||||
return Broadcast()
|
||||
|
||||
|
||||
def _default_aggregation(kind: str) -> AggregationPattern:
|
||||
if kind == "stage_replica":
|
||||
return FanInByStage()
|
||||
if kind == "pp":
|
||||
return StitchPipeline()
|
||||
if kind in ("dp", "ep"):
|
||||
return Union()
|
||||
return TakeRank()
|
||||
|
||||
|
||||
def _build_spec(role: str, entry: Mapping[str, Any]) -> StrategySpec:
|
||||
if "axis" not in entry:
|
||||
raise StrategyLoadError(f"role {role!r}: each strategy entry needs an 'axis' field")
|
||||
if "size" not in entry:
|
||||
raise StrategyLoadError(f"role {role!r}: strategy entry for axis {entry['axis']!r} needs a 'size'")
|
||||
|
||||
kind = str(entry["axis"])
|
||||
try:
|
||||
size = int(entry["size"])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise StrategyLoadError(f"role {role!r}: axis {kind!r} size must be an integer, got {entry['size']!r}") from exc
|
||||
|
||||
routing_policy = entry.get("routing")
|
||||
if routing_policy is not None and kind not in _ROUTING_POLICY_KINDS:
|
||||
raise StrategyLoadError(
|
||||
f"role {role!r}: axis {kind!r} does not accept a 'routing' policy (only {sorted(_ROUTING_POLICY_KINDS)} do)"
|
||||
)
|
||||
|
||||
shard_extension: dict[str, Any] = {}
|
||||
l1_owner = entry.get("l1_owner")
|
||||
if l1_owner is not None:
|
||||
shard_extension["l1_owner"] = str(l1_owner)
|
||||
|
||||
return StrategySpec(
|
||||
name=str(entry.get("name", kind)),
|
||||
mesh_axis=MeshAxisSpec(kind=kind, size=size), # type: ignore[arg-type]
|
||||
routing=_default_routing(kind, routing_policy),
|
||||
aggregation=_default_aggregation(kind),
|
||||
shard_extension=shard_extension,
|
||||
)
|
||||
|
||||
|
||||
def parse_strategy_specs(data: Mapping[str, Any]) -> dict[str, list[StrategySpec]]:
|
||||
"""Parse an already-loaded strategy mapping into per-role spec stacks."""
|
||||
strategies = data.get("strategies", data)
|
||||
if not isinstance(strategies, Mapping):
|
||||
raise StrategyLoadError("strategy file must contain a 'strategies' mapping of role -> list of axes")
|
||||
|
||||
result: dict[str, list[StrategySpec]] = {}
|
||||
for role, entries in strategies.items():
|
||||
if not isinstance(entries, (list, tuple)):
|
||||
raise StrategyLoadError(
|
||||
f"role {role!r}: expected a list of axis declarations, got {type(entries).__name__}"
|
||||
)
|
||||
specs: list[StrategySpec] = []
|
||||
for entry in entries:
|
||||
if not isinstance(entry, Mapping):
|
||||
raise StrategyLoadError(
|
||||
f"role {role!r}: each axis declaration must be a mapping, got {type(entry).__name__}"
|
||||
)
|
||||
specs.append(_build_spec(str(role), dict(entry)))
|
||||
result[str(role)] = specs
|
||||
return result
|
||||
|
||||
|
||||
def load_strategy_specs(path: str) -> dict[str, list[StrategySpec]]:
|
||||
"""Load and parse a strategy file into per-role spec stacks."""
|
||||
data = to_dict(load_yaml_config(path))
|
||||
if not isinstance(data, Mapping):
|
||||
raise StrategyLoadError(f"strategy file {path!r} did not parse to a mapping")
|
||||
return parse_strategy_specs(data)
|
||||
@@ -0,0 +1,378 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Translate a ``StrategySpec`` stack into vLLM-Omni parallel sizing.
|
||||
|
||||
Read a stack of strategy specs (one per mesh axis) and work out the
|
||||
tensor/data/pipeline parallel sizes, the dense-EP flag, and the number of stage
|
||||
replicas, plus who owns each axis's request routing. The result is a plain,
|
||||
CPU-computable sizing struct (:class:`OmniParallelConfig`) keyed by the real
|
||||
``OmniEngineArgs`` / ``EngineArgs`` field names, so the deploy layer can splat it
|
||||
onto a stage's engine args.
|
||||
|
||||
The distinction this module enforces — engine data parallelism (a true vLLM
|
||||
intra-engine world dimension) vs. omni stage replicas (independent engines fanned
|
||||
out by omni's coordinator) — is documented on the axis validators that depend on
|
||||
it (see :func:`_validate_dp` and :func:`_stage_replica_lb_policy`). Routing we do
|
||||
not support yet (key-stable / affinity routing) raises ``NotImplementedError``;
|
||||
any other invalid spec raises :class:`AxisTranslationError`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Literal, NoReturn, cast, get_args
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.config.composable_parallel.routing import (
|
||||
Broadcast,
|
||||
PartitionByHash,
|
||||
PipelineMicrobatch,
|
||||
RouteByStage,
|
||||
RoutingPattern,
|
||||
)
|
||||
from vllm_omni.config.composable_parallel.spec import MeshAxisKind, StrategySpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Who owns an axis's request routing. A closed string type so an unexpected raw
|
||||
# value is caught statically and at runtime (``_VALID_L1_OWNERS`` is derived from
|
||||
# it, keeping one source of truth) rather than silently flowing through.
|
||||
L1Owner = Literal["delegated", "engine"]
|
||||
|
||||
# DP, TP, PP, (dense) EP and stage_replica translate today. DP/TP/PP are true
|
||||
# world dimensions vLLM realizes intra-engine; dense EP is a flag over the
|
||||
# existing TP*DP ranks; stage_replica is an omni-coordinator-level fan-out of
|
||||
# independent engine replicas (NOT a vLLM world dimension). The remaining kinds
|
||||
# (sequence/context parallel, CFG, VAE pipelines, ...) need per-layer hooks or
|
||||
# custom collectives and arrive in later stages.
|
||||
_SUPPORTED_KINDS: tuple[MeshAxisKind, ...] = ("dp", "tp", "pp", "ep", "stage_replica")
|
||||
|
||||
# Which EngineArgs field each axis kind *sizes*. EP and stage_replica are
|
||||
# deliberately absent: EP is not an independent world dimension but an
|
||||
# ``enable_expert_parallel`` flag whose degree equals tp*dp; stage_replica is a
|
||||
# per-stage deploy ``num_replicas`` count, not an intra-engine world dimension.
|
||||
_AXIS_TO_ENGINE_FIELD: dict[MeshAxisKind, str] = {
|
||||
"tp": "tensor_parallel_size",
|
||||
"dp": "data_parallel_size",
|
||||
"pp": "pipeline_parallel_size",
|
||||
}
|
||||
|
||||
# Default L1 owner per axis kind. ``stage_replica`` is the only delegated axis:
|
||||
# omni's coordinator/StagePool owns routing across replicas. ``dp`` is engine
|
||||
# data parallelism, realized by vLLM's own intra-engine DP load balancer.
|
||||
_DEFAULT_L1_OWNER: dict[MeshAxisKind, L1Owner] = {
|
||||
"dp": "engine",
|
||||
"tp": "engine",
|
||||
"pp": "engine",
|
||||
"ep": "engine",
|
||||
"stage_replica": "delegated",
|
||||
}
|
||||
|
||||
# How a RouteByStage policy maps onto omni's load balancer policy string. These
|
||||
# are the stateless options omni can actually do; note there's no "hash" here,
|
||||
# because omni has no key-stable balancer.
|
||||
_STAGE_POLICY_TO_OMNI_LB: dict[str, str] = {
|
||||
"random": "random",
|
||||
"round_robin": "round-robin",
|
||||
"least_queue": "least-queue-length",
|
||||
}
|
||||
|
||||
_VALID_L1_OWNERS: frozenset[str] = frozenset(get_args(L1Owner))
|
||||
|
||||
|
||||
class AxisTranslationError(ValueError):
|
||||
"""Error for an invalid or unsupported strategy spec.
|
||||
|
||||
A single error type (rather than a tree of subclasses) keeps the public
|
||||
surface small and consistent with the rest of the codebase; the specific
|
||||
cause is in the message and is logged before the raise so it is visible even
|
||||
when the type is unavailable to a caller (e.g. across a server boundary).
|
||||
|
||||
Strategies that are *valid but not built yet* — key-stable / affinity routing
|
||||
today — raise ``NotImplementedError`` instead, to distinguish "we haven't
|
||||
implemented this" from "your config is wrong".
|
||||
"""
|
||||
|
||||
|
||||
def _fail(msg: str) -> NoReturn:
|
||||
"""Log and raise :class:`AxisTranslationError` for an invalid/unsupported spec."""
|
||||
logger.error("[composable_parallel] %s", msg)
|
||||
raise AxisTranslationError(msg)
|
||||
|
||||
|
||||
def _not_implemented(msg: str) -> NoReturn:
|
||||
"""Log and raise ``NotImplementedError`` for a valid-but-unbuilt strategy."""
|
||||
logger.error("[composable_parallel] %s", msg)
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OmniParallelConfig:
|
||||
"""Result of translating a spec stack into omni parallel sizing."""
|
||||
|
||||
tensor_parallel_size: int = 1
|
||||
data_parallel_size: int = 1
|
||||
pipeline_parallel_size: int = 1
|
||||
# Dense expert parallelism: a flag over the TP*DP ranks, not a world axis.
|
||||
enable_expert_parallel: bool = False
|
||||
# Number of independent omni stage replicas (the ``stage_replica`` axis).
|
||||
# This is NOT a vLLM world dimension; it is the per-stage deploy
|
||||
# ``num_replicas`` count. 1 means a single (un-replicated) engine.
|
||||
stage_replica_size: int = 1
|
||||
# The omni StagePool LB policy string for the (delegated) stage_replica
|
||||
# axis, if any. Only meaningful when ``stage_replica_size > 1``.
|
||||
omni_lb_policy: str | None = None
|
||||
# axis kind -> resolved L1 owner ("delegated" | "engine").
|
||||
l1_owners: Mapping[MeshAxisKind, L1Owner] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def world_size(self) -> int:
|
||||
# EP and stage_replica are intentionally excluded. EP reuses the TP*DP
|
||||
# ranks rather than adding a dimension; stage_replica spins up separate
|
||||
# engines (each its own world), not extra ranks in this engine's group.
|
||||
# This product is what vLLM calls ``world_size_across_dp`` (tp * pp * dp).
|
||||
return self.tensor_parallel_size * self.data_parallel_size * self.pipeline_parallel_size
|
||||
|
||||
@property
|
||||
def delegated_axes(self) -> tuple[MeshAxisKind, ...]:
|
||||
return tuple(kind for kind, owner in self.l1_owners.items() if owner == "delegated")
|
||||
|
||||
def as_engine_kwargs(self) -> dict[str, object]:
|
||||
"""Return per-stage kwargs keyed by real OmniEngineArgs/EngineArgs field names.
|
||||
|
||||
Two derived values are intentionally *not* emitted here because they are
|
||||
not per-stage engine args:
|
||||
|
||||
* ``stage_replica_size`` — a per-stage deploy ``num_replicas`` knob
|
||||
(``StageDeployConfig``); the deploy layer consumes it separately.
|
||||
* ``omni_lb_policy`` — a pipeline-wide load-balancer policy the engine
|
||||
reads once at construction (see ``StrategyApplyResult.omni_lb_policy``);
|
||||
it is applied at the orchestrator level, not folded into per-stage args.
|
||||
"""
|
||||
kwargs: dict[str, object] = {
|
||||
"tensor_parallel_size": self.tensor_parallel_size,
|
||||
"data_parallel_size": self.data_parallel_size,
|
||||
"pipeline_parallel_size": self.pipeline_parallel_size,
|
||||
}
|
||||
if self.enable_expert_parallel:
|
||||
kwargs["enable_expert_parallel"] = True
|
||||
return kwargs
|
||||
|
||||
|
||||
def _is_affinity_dp_routing(routing: RoutingPattern) -> bool:
|
||||
"""True when DP routing demands key-stable (hash) placement."""
|
||||
if isinstance(routing, PartitionByHash):
|
||||
return True
|
||||
if isinstance(routing, RouteByStage) and routing.routing_policy == "hash":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_l1_owner(spec: StrategySpec) -> L1Owner:
|
||||
kind = spec.mesh_axis.kind
|
||||
declared_owner = spec.shard_extension.get("l1_owner")
|
||||
owner = declared_owner
|
||||
if owner is None:
|
||||
owner = _DEFAULT_L1_OWNER.get(kind, "engine")
|
||||
if kind not in _DEFAULT_L1_OWNER:
|
||||
logger.debug(
|
||||
"[composable_parallel] axis %r has unknown kind %r with no default l1_owner; falling back to %r",
|
||||
spec.name,
|
||||
kind,
|
||||
owner,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"[composable_parallel] axis %r (kind %r) declared no l1_owner; using default %r",
|
||||
spec.name,
|
||||
kind,
|
||||
owner,
|
||||
)
|
||||
if owner not in _VALID_L1_OWNERS:
|
||||
# Distinguish a bad value supplied via shard_extension from a bad default
|
||||
# indexed out of _DEFAULT_L1_OWNER, so the source of the invalid owner is
|
||||
# debuggable rather than swallowed.
|
||||
source = "shard_extension" if declared_owner is not None else "_DEFAULT_L1_OWNER"
|
||||
logger.debug(
|
||||
"[composable_parallel] axis %r (kind %r) resolved invalid l1_owner %r from %s; valid owners are %s",
|
||||
spec.name,
|
||||
kind,
|
||||
owner,
|
||||
source,
|
||||
sorted(_VALID_L1_OWNERS),
|
||||
)
|
||||
_fail(f"axis {kind!r} has invalid l1_owner {owner!r}; expected one of {sorted(_VALID_L1_OWNERS)}")
|
||||
return cast(L1Owner, owner)
|
||||
|
||||
|
||||
def _validate_dp(spec: StrategySpec, owner: L1Owner) -> None:
|
||||
"""Validate an engine data-parallel axis.
|
||||
|
||||
DP is realized intra-engine by vLLM's own DP load balancer, so it is
|
||||
engine-owned and emits no ``omni_lb_policy`` (that string configures omni's
|
||||
*replica* balancer, a different layer). Key-stable (hash) request affinity is
|
||||
not something vLLM's DP LB guarantees, so it is rejected rather than silently
|
||||
dropped.
|
||||
"""
|
||||
if owner != "engine":
|
||||
_fail(
|
||||
f"dp axis {spec.name!r} is engine data parallelism realized intra-engine by "
|
||||
f"vLLM's DP load balancer; l1_owner must be 'engine', got {owner!r}. For "
|
||||
"omni-coordinator-level request fan-out across independent replicas, use a "
|
||||
"'stage_replica' axis."
|
||||
)
|
||||
if _is_affinity_dp_routing(spec.routing):
|
||||
_not_implemented(
|
||||
f"dp axis {spec.name!r} requests key-stable (hash) routing, which vLLM's "
|
||||
"intra-engine DP load balancer does not guarantee — not supported yet. Use "
|
||||
"RouteByStage(random|round_robin|least_queue) for stateless DP balancing."
|
||||
)
|
||||
if not isinstance(spec.routing, RouteByStage):
|
||||
_fail(
|
||||
f"dp axis {spec.name!r} expects RouteByStage(random|round_robin|least_queue) routing, "
|
||||
f"got {type(spec.routing).__name__}"
|
||||
)
|
||||
if spec.routing.routing_policy not in _STAGE_POLICY_TO_OMNI_LB:
|
||||
# Recognized-but-unimplemented routing (key-stable/hash) is already
|
||||
# handled above via _is_affinity_dp_routing -> _not_implemented. Anything
|
||||
# left here is an unknown/invalid policy value (e.g. a typo from YAML),
|
||||
# i.e. invalid input -> AxisTranslationError, not NotImplementedError.
|
||||
_fail(
|
||||
f"dp axis {spec.name!r} has invalid routing_policy "
|
||||
f"{spec.routing.routing_policy!r}; expected one of "
|
||||
f"{sorted(_STAGE_POLICY_TO_OMNI_LB)}."
|
||||
)
|
||||
|
||||
|
||||
def _stage_replica_lb_policy(spec: StrategySpec, owner: L1Owner) -> str:
|
||||
"""Return the omni StagePool LB policy for a delegated stage_replica axis.
|
||||
|
||||
A ``stage_replica`` axis is *not* a vLLM world dimension: it stands up N
|
||||
independent engine replicas of one pipeline stage, coordinated by the omni
|
||||
coordinator and balanced over a StagePool with stateless policies only
|
||||
(random / round-robin / least-queue-length). It maps to the per-stage
|
||||
``num_replicas`` count plus the pipeline-level ``omni_lb_policy`` string, so
|
||||
its ``l1_owner`` must be ``"delegated"`` (omni owns the routing). Key-stable
|
||||
(hash) routing is rejected because omni has no key-stable balancer yet.
|
||||
"""
|
||||
if owner != "delegated":
|
||||
_fail(
|
||||
f"stage_replica axis {spec.name!r} must be 'delegated' to omni's StagePool load "
|
||||
f"balancer; got owner {owner!r}. Replica routing is owned by omni's coordinator."
|
||||
)
|
||||
|
||||
routing = spec.routing
|
||||
if _is_affinity_dp_routing(routing):
|
||||
_not_implemented(
|
||||
f"stage_replica axis {spec.name!r} requests key-stable (hash) routing, which needs a "
|
||||
"dedicated load balancer — not implemented yet. Use "
|
||||
"RouteByStage(random|round_robin|least_queue) to delegate to omni's load balancer."
|
||||
)
|
||||
if not isinstance(routing, RouteByStage):
|
||||
_fail(
|
||||
f"stage_replica axis {spec.name!r} expects RouteByStage(random|round_robin|least_queue) "
|
||||
f"routing, got {type(routing).__name__}"
|
||||
)
|
||||
policy = _STAGE_POLICY_TO_OMNI_LB.get(routing.routing_policy)
|
||||
if policy is None:
|
||||
_not_implemented(
|
||||
f"stage_replica axis {spec.name!r} routing_policy {routing.routing_policy!r} has no omni LB policy"
|
||||
)
|
||||
return policy
|
||||
|
||||
|
||||
def _validate_tp(spec: StrategySpec, owner: L1Owner) -> None:
|
||||
if not isinstance(spec.routing, Broadcast):
|
||||
_fail(f"tp axis {spec.name!r} expects Broadcast routing, got {type(spec.routing).__name__}")
|
||||
if owner != "engine":
|
||||
_fail(f"tp axis {spec.name!r} is realized intra-engine; l1_owner must be 'engine', got {owner!r}")
|
||||
|
||||
|
||||
def _validate_pp(spec: StrategySpec, owner: L1Owner) -> None:
|
||||
if not isinstance(spec.routing, PipelineMicrobatch):
|
||||
_fail(f"pp axis {spec.name!r} expects PipelineMicrobatch routing, got {type(spec.routing).__name__}")
|
||||
if owner != "engine":
|
||||
_fail(f"pp axis {spec.name!r} is realized intra-engine; l1_owner must be 'engine', got {owner!r}")
|
||||
|
||||
|
||||
def _validate_ep(spec: StrategySpec, owner: L1Owner) -> None:
|
||||
# Dense EP: every rank still sees the whole batch, experts are sharded
|
||||
# across ranks inside the engine (sparse MoE all-to-all is a later stage).
|
||||
if not isinstance(spec.routing, Broadcast):
|
||||
_fail(
|
||||
f"ep axis {spec.name!r} expects Broadcast routing (dense expert parallel), "
|
||||
f"got {type(spec.routing).__name__}"
|
||||
)
|
||||
if owner != "engine":
|
||||
_fail(f"ep axis {spec.name!r} is realized intra-engine; l1_owner must be 'engine', got {owner!r}")
|
||||
|
||||
|
||||
def translate_strategy_stack(specs: Sequence[StrategySpec]) -> OmniParallelConfig:
|
||||
"""Translate a spec stack into an ``OmniParallelConfig``.
|
||||
|
||||
Supported kinds: dp (engine data parallel), tp, pp, (dense) ep, and
|
||||
stage_replica (omni replicas). Raises ``NotImplementedError`` for deferred
|
||||
(affinity / key-stable) routing, and :class:`AxisTranslationError` for any
|
||||
other invalid spec (a kind not yet translatable, a repeated kind, an owner
|
||||
incompatible with the axis kind, or unsupported routing). The EP degree must
|
||||
equal tensor_parallel_size * data_parallel_size.
|
||||
"""
|
||||
sizes: dict[str, int] = {"tensor_parallel_size": 1, "data_parallel_size": 1, "pipeline_parallel_size": 1}
|
||||
owners: dict[MeshAxisKind, L1Owner] = {}
|
||||
omni_lb_policy: str | None = None
|
||||
stage_replica_size = 1
|
||||
enable_expert_parallel = False
|
||||
ep_size: int | None = None
|
||||
|
||||
for spec in specs:
|
||||
kind = spec.mesh_axis.kind
|
||||
if kind not in _SUPPORTED_KINDS:
|
||||
_fail(
|
||||
f"axis kind {kind!r} is not translatable yet (supported: {list(_SUPPORTED_KINDS)}); "
|
||||
"it is designed-for and lands in a later stage"
|
||||
)
|
||||
if kind in owners:
|
||||
_fail(f"axis kind {kind!r} appears more than once in the spec stack")
|
||||
|
||||
owner = _resolve_l1_owner(spec)
|
||||
if kind == "dp":
|
||||
_validate_dp(spec, owner)
|
||||
elif kind == "tp":
|
||||
_validate_tp(spec, owner)
|
||||
elif kind == "pp":
|
||||
_validate_pp(spec, owner)
|
||||
elif kind == "ep":
|
||||
_validate_ep(spec, owner)
|
||||
enable_expert_parallel = True
|
||||
ep_size = spec.mesh_axis.size
|
||||
elif kind == "stage_replica":
|
||||
omni_lb_policy = _stage_replica_lb_policy(spec, owner)
|
||||
stage_replica_size = spec.mesh_axis.size
|
||||
|
||||
if kind in _AXIS_TO_ENGINE_FIELD:
|
||||
sizes[_AXIS_TO_ENGINE_FIELD[kind]] = spec.mesh_axis.size
|
||||
owners[kind] = owner
|
||||
|
||||
if enable_expert_parallel:
|
||||
# Dense EP shards experts across exactly the TP*DP ranks, so the declared
|
||||
# EP degree must match that product (it is not its own world dimension,
|
||||
# and PP is excluded). An EP axis of size 1 is a degenerate no-op that
|
||||
# still sets the flag; it is only valid when TP*DP == 1.
|
||||
ep_ranks = sizes["tensor_parallel_size"] * sizes["data_parallel_size"]
|
||||
if ep_size != ep_ranks:
|
||||
_fail(
|
||||
f"ep axis size {ep_size} must equal tensor_parallel_size*data_parallel_size "
|
||||
f"(={ep_ranks}); dense expert parallelism shards experts across exactly those ranks"
|
||||
)
|
||||
|
||||
return OmniParallelConfig(
|
||||
tensor_parallel_size=sizes["tensor_parallel_size"],
|
||||
data_parallel_size=sizes["data_parallel_size"],
|
||||
pipeline_parallel_size=sizes["pipeline_parallel_size"],
|
||||
enable_expert_parallel=enable_expert_parallel,
|
||||
stage_replica_size=stage_replica_size,
|
||||
omni_lb_policy=omni_lb_policy,
|
||||
l1_owners=owners,
|
||||
)
|
||||
Reference in new issue
Block a user