[Core] Retain structured configs through runtime startup (#6849)

Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
This commit is contained in:
maithilijoshi20 authored and GitHub committed 2026-09-16 20:47:38 -07:00
1 parent becfefd440
commit f1a6e7ce2a
49 files changed
+1856 -331

No files matched your search

+8 -1
View File
@@ -272,7 +272,14 @@ class OmniEngineBase:
self.num_stages = len(self.stage_configs)
self.async_chunk = any(
bool(getattr(getattr(stage, "engine_args", None), "async_chunk", False)) for stage in self.stage_configs
bool(
getattr(
getattr(stage, "connector_config", None),
"async_chunk",
getattr(getattr(stage, "engine_args", None), "async_chunk", False),
)
)
for stage in self.stage_configs
)
self.stage_pools: list[StagePool] = []
self.stage_clients: list[StageClient] = [] # logical-stage view for external readers
+6 -1
View File
@@ -287,8 +287,13 @@ def check_admission(
def _diffusion_utilization(replica: Any) -> float | None:
"""Best-effort gpu_memory_utilization for a diffusion replica from raw args."""
"""Resolve a diffusion replica's memory budget from typed or legacy config."""
stage_cfg = getattr(replica, "stage_cfg", None)
cache_config = getattr(stage_cfg, "cache_config", None)
util = getattr(cache_config, "gpu_memory_utilization", None)
if util is not None:
return float(util)
engine_args = getattr(stage_cfg, "engine_args", None)
if isinstance(engine_args, dict):
util = engine_args.get("gpu_memory_utilization")
+11 -6
View File
@@ -17,6 +17,7 @@ from types import SimpleNamespace
from typing import Any
import msgspec
import torch
import zmq
from omegaconf import OmegaConf
from vllm.config import VllmConfig
@@ -33,6 +34,7 @@ from vllm.v1.engine.utils import (
)
from vllm.v1.executor import Executor
from vllm_omni.config.omni_config import BaseVllmOmniStageConfig
from vllm_omni.distributed.omni_connectors.utils import initialization
from vllm_omni.engine import stage_init_utils
from vllm_omni.engine.stage_init_utils import (
@@ -67,11 +69,13 @@ def _serialize_stage_config(stage_config: Any) -> Any:
"""Convert a stage config to msgpack-friendly builtins."""
if stage_config is None or isinstance(stage_config, (str, bytes, int, float, bool)):
return stage_config
if isinstance(stage_config, torch.dtype):
return str(stage_config).removeprefix("torch.")
if OmegaConf.is_config(stage_config):
return _serialize_stage_config(OmegaConf.to_container(stage_config, resolve=True))
if dataclasses.is_dataclass(stage_config):
if dataclasses.is_dataclass(stage_config) and not isinstance(stage_config, type):
return _serialize_stage_config(dataclasses.asdict(stage_config))
if isinstance(stage_config, dict):
@@ -1403,7 +1407,7 @@ def get_headless_replica_devices(
omni_dp_size_local: int,
) -> list[str | None]:
"""Return per-replica device slices for a headless stage."""
runtime_cfg = getattr(stage_cfg, "runtime", None)
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", None))
devices_str: str | None = None
if runtime_cfg is not None:
devices_str = (
@@ -1509,10 +1513,11 @@ def launch_headless_diffusion_replicas(
stage_id,
)
# Headless diffusion startup and its downstream helpers still consume the
# legacy StageConfig shape; switch this with the coordinated RFC #4021
# stage-init cutover.
metadata = stage_init_utils.extract_legacy_stage_metadata(stage_cfg)
metadata = (
stage_init_utils.extract_stage_metadata_from_omni_stage_config(stage_cfg)
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
else stage_init_utils.extract_legacy_stage_metadata(stage_cfg)
)
if omni_conn_cfg:
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
# Headless single-stage launch must still infer cross-stage TP topology
+76 -40
View File
@@ -402,6 +402,11 @@ def _tp_size_for_stage(stage_configs: Sequence[Any], stage_id: Any) -> int | Non
for stage_cfg in stage_configs:
if str(getattr(stage_cfg, "stage_id", None)) not in id_strs:
continue
if isinstance(stage_cfg, BaseVllmOmniStageConfig):
try:
return max(1, int(stage_cfg.parallel_config.tensor_parallel_size))
except (TypeError, ValueError):
return 1
engine_args = getattr(stage_cfg, "engine_args", None)
if engine_args is None:
return 1
@@ -489,11 +494,15 @@ def inject_kv_stage_info(stage_cfg: Any, stage_id: int, stage_configs: Sequence[
rank mappings automatically.
"""
try:
engine_args = stage_cfg.engine_args
if hasattr(engine_args, "get"):
omni_kv = engine_args.get("omni_kv_config", None)
connector_config = getattr(stage_cfg, "connector_config", None)
if connector_config is not None:
omni_kv = connector_config.omni_kv_config
else:
omni_kv = getattr(engine_args, "omni_kv_config", None)
engine_args = stage_cfg.engine_args
if hasattr(engine_args, "get"):
omni_kv = engine_args.get("omni_kv_config", None)
else:
omni_kv = getattr(engine_args, "omni_kv_config", None)
if omni_kv is None:
return
@@ -504,7 +513,11 @@ def inject_kv_stage_info(stage_cfg: Any, stage_id: int, stage_configs: Sequence[
if "stage_id" not in omni_kv:
omni_kv["stage_id"] = stage_id
engine_input_source = getattr(stage_cfg, "engine_input_source", None)
engine_input_source = (
getattr(stage_cfg, "input_sources", None)
if connector_config is not None
else getattr(stage_cfg, "engine_input_source", None)
)
if engine_input_source is not None:
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("engine_input_source", list(engine_input_source))
@@ -703,17 +716,16 @@ def _resolve_omni_metadata_hook(path: str | None) -> Callable | None:
def extract_stage_metadata_from_omni_stage_config(
stage_config: BaseVllmOmniStageConfig,
) -> StageMetadata:
"""Project one typed stage config into metadata for a future cutover.
This projection is not used by production startup yet. Current replica
layout, engine-argument, remote-diffusion, and platform setup paths still
require the legacy StageConfig/OmegaConf shape.
"""
"""Project one typed stage config into production runtime metadata."""
stage_type: Literal["llm", "diffusion"] = "diffusion" if stage_config.stage_type == StageType.DIFFUSION else "llm"
sampling_params_cls = SamplingParams if stage_type == "llm" else OmniDiffusionSamplingParams
sampling_params: OmniSamplingParams = sampling_params_cls(
**(stage_config.model_config.default_sampling_params or {})
)
pooling_config = stage_config.pooling_config
if stage_type == "llm" and (pooling_config.runner or "").lower() == "pooling":
sampling_params: OmniSamplingParams | PoolingParams = (
copy.deepcopy(pooling_config.default_pooling_params) or PoolingParams()
)
else:
sampling_params_cls = SamplingParams if stage_type == "llm" else OmniDiffusionSamplingParams
sampling_params = sampling_params_cls(**(stage_config.model_config.default_sampling_params or {}))
custom_process_input_func = _resolve_omni_metadata_hook(stage_config.custom_process_input_func)
if stage_type == "diffusion":
@@ -878,7 +890,9 @@ def _get_local_llm_parallel_sizes(
unset.
"""
if engine_args is None:
engine_args = getattr(stage_cfg, "engine_args", {})
engine_args = getattr(stage_cfg, "parallel_config", None)
if engine_args is None:
engine_args = getattr(stage_cfg, "engine_args", {})
tp_size = int(_get_attr_or_item(engine_args, "tensor_parallel_size", 1) or 1)
pp_size = int(_get_attr_or_item(engine_args, "pipeline_parallel_size", 1) or 1)
local_dp_size = _get_attr_or_item(engine_args, "data_parallel_size_local", None)
@@ -889,9 +903,12 @@ def _get_local_llm_parallel_sizes(
def get_stage_devices_per_replica(stage_cfg: Any, engine_args: Any | None = None) -> int:
"""Return the number of devices consumed by one replica of *stage_cfg*."""
if engine_args is None:
engine_args = getattr(stage_cfg, "engine_args", {})
if getattr(stage_cfg, "stage_type", "llm") == "diffusion":
typed_parallel_config = getattr(stage_cfg, "parallel_config", None)
if typed_parallel_config is not None:
return max(1, int(typed_parallel_config.world_size))
if engine_args is None:
engine_args = getattr(stage_cfg, "engine_args", {})
parallel_config = _get_attr_or_item(engine_args, "parallel_config")
if parallel_config is None:
return 1
@@ -933,7 +950,7 @@ def compute_replica_layout(
"""
replicas_per_stage: list[int] = []
for stage_cfg in stage_configs:
runtime_cfg = getattr(stage_cfg, "runtime", {})
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", {}))
num_replicas = int(
runtime_cfg.get("num_replicas", 1)
if hasattr(runtime_cfg, "get")
@@ -948,7 +965,7 @@ def compute_replica_layout(
num_replicas = replicas_per_stage[stage_id]
if num_replicas <= 1:
continue
runtime_cfg = getattr(stage_cfg, "runtime", {})
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", {}))
devices_str = (
runtime_cfg.get("devices") if hasattr(runtime_cfg, "get") else getattr(runtime_cfg, "devices", None)
)
@@ -1086,15 +1103,31 @@ def _project_omni_stage_engine_args(
diffusion_stage = cast(VllmOmniDiffusionStageConfig, stage_config)
engine_args.update(_project_omni_config_fields(diffusion_stage.diffusion_config))
model_excluded_fields = {
"default_sampling_params",
"has_sampling_extra_args",
}
runtime_excluded_fields = {"devices", "num_replicas", "env", "num_gpus"}
if not is_diffusion:
# These values configure OmniDiffusionConfig or its worker process;
# OmniEngineArgs has no matching fields for LLM stages.
model_excluded_fields.update(
{
"disable_autocast",
"enable_multithread_weight_load",
"num_weight_load_threads",
}
)
runtime_excluded_fields.add("log_level")
for config, excluded_fields in (
(
stage_config.model_config,
frozenset({"default_sampling_params", "has_sampling_extra_args"}),
frozenset(model_excluded_fields),
),
(
stage_config.runtime_config,
frozenset({"devices", "num_replicas", "env", "num_gpus"})
| (frozenset({"additional_config"}) if is_diffusion else frozenset()),
frozenset(runtime_excluded_fields) | (frozenset({"additional_config"}) if is_diffusion else frozenset()),
),
):
engine_args.update(
@@ -1103,6 +1136,11 @@ def _project_omni_stage_engine_args(
exclude=excluded_fields,
)
)
pooling_config = stage_config.pooling_config
if pooling_config.runner is not None:
engine_args["runner"] = pooling_config.runner
if pooling_config.pooling_output_decoder is not None:
engine_args["pooling_output_decoder"] = pooling_config.pooling_output_decoder
if is_diffusion:
for config, field_map in (
@@ -1330,11 +1368,9 @@ def build_engine_args_dict(
stage_connector_spec: dict[str, Any] | None = None,
cli_tokenizer: str | None = None,
) -> dict[str, Any]:
"""Build engine arguments through the stable production entry point.
"""Preserve the legacy engine-argument API for compatibility callers.
Production inputs still use the legacy stage representation. Keep that
compatibility choice behind this function so callers do not bind directly
to a representation-specific implementation.
Typed runtime stages use ``build_engine_args_dict_from_omni_stage_config``.
"""
return build_legacy_engine_args_dict(
stage_config,
@@ -1350,13 +1386,7 @@ def build_engine_args_dict_from_omni_stage_config(
stage_connector_spec: dict[str, Any] | None = None,
cli_tokenizer: str | None = None,
) -> dict[str, Any]:
"""Project one typed stage config into backend engine arguments.
This projection is prepared for the RFC #4021 stage-init cutover. Current
production startup reaches the legacy implementation through
``build_engine_args_dict`` while strategy and startup-plan inputs still
use the legacy representation.
"""
"""Project one typed production stage config into backend engine arguments."""
engine_args_dict = _project_omni_stage_engine_args(stage_config)
_apply_rocm_attention_backend(engine_args_dict, stage_config.stage_type)
return _finalize_engine_args_dict(
@@ -1391,7 +1421,7 @@ def _check_stage_device_layout(stage_config: Any, engine_args_dict: dict[str, An
"""
from vllm_omni.config.composable_parallel import StrategyApplyError, check_device_layout
runtime = getattr(stage_config, "runtime", None)
runtime = getattr(stage_config, "runtime_config", getattr(stage_config, "runtime", None))
devices = _get_attr_or_item(runtime, "devices", None) if runtime is not None else None
if devices is None:
# No explicit placement -> vLLM assigns devices itself; nothing to check.
@@ -1452,10 +1482,12 @@ def build_vllm_config(
(vllm_config, executor_class)
"""
if engine_args_dict is None:
engine_args_dict = build_engine_args_dict(
stage_config,
model,
stage_connector_spec=stage_connector_spec,
engine_args_dict = (
build_engine_args_dict_from_omni_stage_config(
stage_config, model, stage_connector_spec=stage_connector_spec
)
if isinstance(stage_config, BaseVllmOmniStageConfig)
else build_engine_args_dict(stage_config, model, stage_connector_spec=stage_connector_spec)
)
filtered_engine_args_dict = filter_dataclass_kwargs(OmniEngineArgs, engine_args_dict)
@@ -1927,7 +1959,11 @@ def build_diffusion_config(
) -> Any:
"""Build diffusion config for a stage."""
engine_args_dict = build_engine_args_dict(stage_cfg, model)
engine_args_dict = (
build_engine_args_dict_from_omni_stage_config(stage_cfg, model)
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
else build_engine_args_dict(stage_cfg, model)
)
od_config = OmniDiffusionConfig.from_kwargs(**engine_args_dict)
num_devices_per_stage = od_config.parallel_config.world_size
+57 -36
View File
@@ -16,9 +16,9 @@ from dataclasses import dataclass
from typing import Any, cast
import janus
from omegaconf import OmegaConf
from vllm.logger import init_logger
from vllm_omni.config.omni_config import BaseVllmOmniStageConfig
from vllm_omni.distributed.omni_connectors.utils.initialization import (
resolve_omni_kv_config_for_stage,
)
@@ -49,11 +49,13 @@ from vllm_omni.engine.stage_init_utils import (
_inject_inferred_kv_tp_topology,
acquire_device_locks,
build_engine_args_dict,
build_engine_args_dict_from_omni_stage_config,
build_llm_stage_output_processor,
build_vllm_config,
compute_replica_layout,
device_overlap_group_keys,
extract_legacy_stage_metadata,
extract_stage_metadata_from_omni_stage_config,
get_stage_connector_spec,
inject_kv_stage_info,
inject_omni_kv_connector_config,
@@ -345,12 +347,12 @@ class StageRuntime:
"""Build startup plans for every logical stage and replica."""
stage_plans: list[LogicalStageInitPlan] = []
# RFC #4021 transition boundary: stage planning still relies on legacy
# StageConfig.runtime and StageConfig.engine_args for replica and engine
# setup. Keep metadata extraction on the legacy path until the
# coordinated stage-init cutover.
for stage_idx, stage_cfg in enumerate(self._stage_configs):
base_metadata = extract_legacy_stage_metadata(stage_cfg)
base_metadata = (
extract_stage_metadata_from_omni_stage_config(stage_cfg)
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
else extract_legacy_stage_metadata(stage_cfg)
)
stage_id = int(base_metadata.stage_id)
if stage_id != stage_idx:
raise ValueError(
@@ -374,14 +376,20 @@ class StageRuntime:
executor_class = None
engine_args_dict = None
if base_metadata.stage_type != "diffusion":
# The stable adapter entry point still receives the same
# legacy stage object as replica planning. Its implementation
# switches only at the coordinated RFC #4021 cutover.
engine_args_dict = build_engine_args_dict(
stage_cfg,
self._model,
stage_connector_spec=stage_connector_spec,
cli_tokenizer=self._tokenizer,
engine_args_dict = (
build_engine_args_dict_from_omni_stage_config(
stage_cfg,
self._model,
stage_connector_spec=stage_connector_spec,
cli_tokenizer=self._tokenizer,
)
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
else build_engine_args_dict(
stage_cfg,
self._model,
stage_connector_spec=stage_connector_spec,
cli_tokenizer=self._tokenizer,
)
)
inject_omni_kv_connector_config(
engine_args_dict,
@@ -401,11 +409,20 @@ class StageRuntime:
)
for replica_id in range(num_replicas):
replica_cfg = copy.deepcopy(stage_cfg) if replica_id > 0 else stage_cfg
# Keep the logical stage's device pool intact; each replica owns
# the same config throughout planning, metadata and launch.
replica_cfg = copy.deepcopy(stage_cfg) if num_replicas > 1 else stage_cfg
if stage_idx in replica_devices_map:
replica_cfg.runtime.devices = replica_devices_map[stage_idx][replica_id]
devices = replica_devices_map[stage_idx][replica_id]
runtime_cfg = getattr(replica_cfg, "runtime_config", getattr(replica_cfg, "runtime", None))
if runtime_cfg is not None:
runtime_cfg.devices = devices
replica_metadata = extract_legacy_stage_metadata(replica_cfg)
replica_metadata = (
extract_stage_metadata_from_omni_stage_config(replica_cfg)
if isinstance(replica_cfg, BaseVllmOmniStageConfig)
else extract_legacy_stage_metadata(replica_cfg)
)
replica_metadata.replica_id = replica_id
if launch_mode == "remote" and replica_metadata.stage_type != "diffusion":
replica_metadata.runtime_cfg = None
@@ -859,17 +876,25 @@ class StageRuntime:
raise RuntimeError("Omni KV connector requires source and destination stages")
inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to)
inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self._stage_configs)
engine_args = getattr(plan.stage_cfg, "engine_args", {})
inline_diffusion = (
engine_args.get("inline_diffusion", False)
if hasattr(engine_args, "get")
else getattr(engine_args, "inline_diffusion", False)
)
custom_pipeline_args = (
engine_args.get("custom_pipeline_args")
if hasattr(engine_args, "get")
else getattr(engine_args, "custom_pipeline_args", None)
)
if isinstance(plan.stage_cfg, BaseVllmOmniStageConfig):
inline_diffusion = plan.stage_cfg.stage_pipeline_config.inline_diffusion
custom_pipeline_args = getattr(
getattr(plan.stage_cfg, "diffusion_config", None),
"custom_pipeline_args",
None,
)
else:
engine_args = getattr(plan.stage_cfg, "engine_args", {})
inline_diffusion = (
engine_args.get("inline_diffusion", False)
if hasattr(engine_args, "get")
else getattr(engine_args, "inline_diffusion", False)
)
custom_pipeline_args = (
engine_args.get("custom_pipeline_args")
if hasattr(engine_args, "get")
else getattr(engine_args, "custom_pipeline_args", None)
)
client, resources = launch_diffusion_stage_replica(
model=self._model,
stage_config=plan.stage_cfg,
@@ -1028,7 +1053,7 @@ class DistStageRuntime(StageRuntime):
for idx, stage_cfg in enumerate(self._stage_configs):
stage_id = int(getattr(stage_cfg, "stage_id", idx))
runtime_cfg = getattr(stage_cfg, "runtime", None)
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", None))
if runtime_cfg is None:
continue
if stage_id == target_stage_id:
@@ -1100,13 +1125,9 @@ class DistStageRuntime(StageRuntime):
if registered_stage_cfg is None:
raise ValueError(f"Remote stage {plan.metadata.stage_id} registered without stage config")
# Remote diffusion registration still transports the legacy mapping
# shape. Reconstruct and project that shape until its RFC #4021 cutover.
metadata = (
extract_legacy_stage_metadata(OmegaConf.create(registered_stage_cfg))
if plan.metadata.stage_type == "diffusion"
else copy.deepcopy(plan.metadata)
)
# Registration is transport-only: the head-side typed plan remains
# authoritative for metadata and topology.
metadata = extract_stage_metadata_from_omni_stage_config(plan.stage_cfg)
metadata.replica_id = plan.replica_id
ctx = StageRemoteFactoryContext(
stage_id=plan.metadata.stage_id,