[Entrypoint][Refactor]Stage CLI Refactor (#2020)

Signed-off-by: wuhang <wuhang6@huawei.com>
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
wuhangandDidan Deng authored and GitHub committed 2026-04-10 14:40:37 +08:00
1 parent f3f2dc590c
commit cb91cbe61e
19 files changed
+3773 -273

No files matched your search

+22 -1
View File
@@ -86,7 +86,11 @@ class OmniEngineArgs(EngineArgs):
Adds omni-specific configuration fields for multi-stage pipeline
processing and output type specification.
Args:
stage_id: Identifier for the stage in a multi-stage pipeline (default: 0)
stage_id: Identifier for the stage in a multi-stage pipeline.
Defaults to 0 for per-stage engine construction. The CLI-level
single-stage selector remains optional on the parsed argparse
namespace and should not be forwarded as a nullable per-stage
engine argument.
model_stage: Stage type identifier, e.g., "thinker" or "talker"
(default: "thinker")
model_arch: Model architecture name
@@ -105,6 +109,18 @@ class OmniEngineArgs(EngineArgs):
worker_type: Model Type, e.g., "ar" or "generation"
task_type: Default task type for TTS models (CustomVoice, VoiceDesign, or Base).
If not specified, will be inferred from model path.
omni_master_address: TCP address that the OmniMasterServer (running
inside AsyncOmniEngine) listens on for engine core registrations.
Required when single-stage mode is active.
omni_master_port: TCP port for the OmniMasterServer registration
socket. Required when single-stage mode is active.
stage_configs_path: Optional path to a JSON/YAML file containing
stage configurations for the multi-stage pipeline. If None,
stage configs are resolved from the model's default configuration.
output_modalities: Optional list of output modality names to enable
(e.g. ["text", "audio"]). If None, all modalities supported by
the model are used.
log_stats: Whether to log engine statistics. Defaults to False.
"""
stage_id: int = 0
@@ -119,6 +135,11 @@ class OmniEngineArgs(EngineArgs):
quantization_config: Any | None = None
worker_type: str | None = None
task_type: str | None = None
omni_master_address: str | None = None
omni_master_port: int | None = None
stage_configs_path: str | None = None
output_modalities: list[str] | None = None
log_stats: bool = False
def __post_init__(self) -> None:
load_omni_general_plugins()
+392 -134
View File
@@ -18,12 +18,10 @@ import time
import uuid
import weakref
from collections.abc import Mapping, Sequence
from contextlib import ExitStack
from dataclasses import asdict
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from vllm_omni.engine.arg_utils import OmniEngineArgs
import janus
import torch
from omegaconf import OmegaConf
@@ -34,26 +32,36 @@ from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.input_processor import InputProcessor
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
from vllm_omni.diffusion.stage_diffusion_proc import (
complete_diffusion_handshake,
spawn_diffusion_proc,
)
from vllm_omni.distributed.omni_connectors.utils.initialization import (
resolve_omni_kv_config_for_stage,
)
from vllm_omni.engine import (
OmniEngineCoreRequest,
)
from vllm_omni.engine import OmniEngineCoreRequest
from vllm_omni.engine.orchestrator import Orchestrator
from vllm_omni.engine.output_processor import MultimodalOutputProcessor
from vllm_omni.engine.serialization import (
deserialize_additional_information,
serialize_additional_information,
)
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
from vllm_omni.engine.stage_engine_core_proc import (
complete_stage_handshake,
spawn_stage_core,
)
from vllm_omni.engine.stage_engine_startup import (
OmniMasterServer,
connect_remote_engine_cores,
launch_omni_core_engines,
register_stage_with_omni_master,
)
from vllm_omni.engine.stage_init_utils import (
StartedLlmStage,
acquire_device_locks,
build_diffusion_config,
build_engine_args_dict,
build_vllm_config,
cleanup_failed_stage_initialization,
@@ -62,17 +70,20 @@ from vllm_omni.engine.stage_init_utils import (
finalize_initialized_stages,
get_stage_connector_spec,
initialize_diffusion_stage,
inject_kv_stage_info,
load_omni_transfer_config_for_model,
prepare_engine_environment,
release_device_locks,
setup_stage_devices,
terminate_alive_proc,
)
from vllm_omni.entrypoints.utils import (
load_and_resolve_stage_configs,
)
from vllm_omni.entrypoints.utils import load_and_resolve_stage_configs
from vllm_omni.inputs.preprocess import OmniInputPreprocessor
from vllm_omni.platforms import current_omni_platform
if TYPE_CHECKING:
from vllm_omni.engine.arg_utils import OmniEngineArgs
logger = init_logger(__name__)
@@ -86,39 +97,6 @@ def _patch_generation_config_if_needed(model_config: Any) -> None:
model_config.try_get_generation_config = lambda: {}
def _inject_kv_stage_info(stage_cfg: Any, stage_id: int) -> None:
"""Inject stage_id and engine_input_source into omni_kv_config.
OmniKVTransferManager needs stage_id to compute recv_stages for the
receiving side. In the old Omni architecture, OmniDiffusion.__init__
performed this injection; replicate it here for AsyncOmniEngine.
"""
try:
engine_args = stage_cfg.engine_args
if hasattr(engine_args, "get"):
omni_kv = engine_args.get("omni_kv_config", None)
else:
omni_kv = getattr(engine_args, "omni_kv_config", None)
if omni_kv is None:
return
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("stage_id", stage_id)
elif hasattr(omni_kv, "__setitem__"):
if "stage_id" not in omni_kv:
omni_kv["stage_id"] = stage_id
engine_input_source = getattr(stage_cfg, "engine_input_source", None)
if engine_input_source is not None:
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("engine_input_source", list(engine_input_source))
elif hasattr(omni_kv, "__setitem__") and "engine_input_source" not in omni_kv:
omni_kv["engine_input_source"] = list(engine_input_source)
except Exception as e:
logger.debug("Failed to inject stage info into omni_kv_config: %s", e)
def _inject_global_id(target: Any, request_id: str) -> None:
"""Inject global_request_id into a prompt dict's additional_information."""
if isinstance(target, dict):
@@ -255,6 +233,7 @@ class AsyncOmniEngine:
stage_init_timeout: int = 300,
init_timeout: int = 600,
diffusion_batch_size: int = 1,
single_stage_mode: bool = False,
**kwargs: Any,
) -> None:
self.model = model
@@ -274,6 +253,31 @@ class AsyncOmniEngine:
ea_dict.pop("model", None)
kwargs = {**ea_dict, **kwargs}
# ------------------------------------------------------------------ #
# Single-stage mode detection #
# ------------------------------------------------------------------ #
# Single-stage mode is enabled when the caller explicitly passes #
# single_stage_mode=True, or when a stage_id is provided in the args. #
_stage_id_kwarg = kwargs.get("stage_id")
if isinstance(_stage_id_kwarg, int) and not single_stage_mode:
single_stage_mode = True
self.single_stage_mode: bool = single_stage_mode
self._single_stage_id_filter: int | None = (
int(_stage_id_kwarg) if single_stage_mode and isinstance(_stage_id_kwarg, int) else None
)
self._omni_master_address: str | None = kwargs.get("omni_master_address")
self._omni_master_port: int | None = kwargs.get("omni_master_port")
self._omni_master_server: OmniMasterServer | None = None
if single_stage_mode:
logger.info(
"[AsyncOmniEngine] Single-stage mode enabled (stage_id_filter=%s, master=%s:%s)",
self._single_stage_id_filter,
self._omni_master_address,
self._omni_master_port,
)
self.config_path, self.stage_configs = self._resolve_stage_configs(model, kwargs)
self.num_stages = len(self.stage_configs)
@@ -350,61 +354,89 @@ class AsyncOmniEngine:
started_stage: StartedLlmStage | None = None
lock_fds: list[int] = []
device_control_env = current_omni_platform.device_control_env_var
try:
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(metadata.stage_id, metadata.runtime_cfg)
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
)
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
if omni_conn_cfg:
omni_kv = engine_args_dict.get("omni_kv_config") or {}
if not isinstance(omni_kv, dict):
omni_kv = dict(omni_kv)
omni_kv["connector_config"] = omni_conn_cfg
omni_kv["omni_from_stage"] = omni_from
omni_kv["omni_to_stage"] = omni_to
omni_kv.setdefault("stage_id", metadata.stage_id)
engine_args_dict["omni_kv_config"] = omni_kv
vllm_config, executor_class = build_vllm_config(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
engine_args_dict=engine_args_dict,
)
lock_fds = acquire_device_locks(
metadata.stage_id,
engine_args_dict,
stage_init_timeout,
)
addresses, proc, handshake_address = spawn_stage_core(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
)
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
proc=proc,
addresses=addresses,
)
logger.info("[AsyncOmniEngine] Stage %s engine launch started", metadata.stage_id)
# Keep the stage-specific device visibility until vLLM
# finishes starting all child processes.
complete_stage_handshake(proc, handshake_address, addresses, vllm_config)
logger.info("[AsyncOmniEngine] Stage %s engine startup completed", metadata.stage_id)
finally:
if previous_visible_devices is None:
current_omni_platform.unset_device_control_env_var()
else:
current_omni_platform.set_device_control_env_var(previous_visible_devices)
proc = None
handshake_address = None
with ExitStack() as launch_stack:
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(metadata.stage_id, metadata.runtime_cfg)
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
)
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
if omni_conn_cfg:
omni_kv = engine_args_dict.get("omni_kv_config") or {}
if not isinstance(omni_kv, dict):
omni_kv = dict(omni_kv)
omni_kv["connector_config"] = omni_conn_cfg
omni_kv["omni_from_stage"] = omni_from
omni_kv["omni_to_stage"] = omni_to
omni_kv.setdefault("stage_id", metadata.stage_id)
engine_args_dict["omni_kv_config"] = omni_kv
vllm_config, executor_class = build_vllm_config(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
engine_args_dict=engine_args_dict,
)
lock_fds = acquire_device_locks(
metadata.stage_id,
engine_args_dict,
stage_init_timeout,
)
if self.single_stage_mode and self._omni_master_server is not None:
engine_manager, coordinator, addresses = launch_stack.enter_context(
launch_omni_core_engines(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
omni_master_server=self._omni_master_server,
stage_id=metadata.stage_id,
stage_config=stage_cfg,
)
)
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
addresses=addresses,
engine_manager=engine_manager,
coordinator=coordinator,
)
else:
addresses, proc, handshake_address = spawn_stage_core(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
)
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
addresses=addresses,
proc=proc,
)
logger.info("[AsyncOmniEngine] Stage %s engine launch started", metadata.stage_id)
# Keep the stage-specific device visibility until vLLM
# finishes starting all child processes.
if self.single_stage_mode and self._omni_master_server is not None:
launch_stack.close()
else:
assert proc is not None
assert handshake_address is not None
complete_stage_handshake(proc, handshake_address, addresses, vllm_config)
logger.info("[AsyncOmniEngine] Stage %s engine startup completed", metadata.stage_id)
finally:
if previous_visible_devices is None:
current_omni_platform.unset_device_control_env_var()
else:
current_omni_platform.set_device_control_env_var(previous_visible_devices)
assert started_stage is not None
return started_stage
@@ -416,13 +448,138 @@ class AsyncOmniEngine:
if lock_fds:
release_device_locks(lock_fds)
def _create_remote_llm_stage(
self,
stage_cfg: Any,
metadata: Any,
stage_connector_spec: dict[str, Any],
stage_init_timeout: int,
omni_master_server: OmniMasterServer,
) -> StartedLlmStage:
"""Attach to a remote engine core and wait for its startup handshake."""
started_stage: StartedLlmStage | None = None
try:
raw_stage_cfg = omni_master_server.get_stage_config(
metadata.stage_id,
timeout_s=stage_init_timeout,
)
if raw_stage_cfg is None:
raise ValueError(f"Remote stage {metadata.stage_id} registered without stage config")
stage_cfg = OmegaConf.create(raw_stage_cfg)
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
)
vllm_config, executor_class = build_vllm_config(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
engine_args_dict=engine_args_dict,
)
vllm_config.parallel_config.data_parallel_size_local = 0
launch_cm = connect_remote_engine_cores(
vllm_config=vllm_config,
omni_master_server=omni_master_server,
stage_id=metadata.stage_id,
)
logger.info("[AsyncOmniEngine] Stage %s remote engine handshake started", metadata.stage_id)
with launch_cm as (engine_manager, coordinator, addresses):
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
engine_manager=engine_manager,
coordinator=coordinator,
addresses=addresses,
)
logger.info("[AsyncOmniEngine] Stage %s remote engine startup completed", metadata.stage_id)
assert started_stage is not None
return started_stage
except Exception:
if started_stage is not None:
close_started_llm_stage(started_stage)
raise
def _launch_diffusion_stage(
self,
stage_cfg: Any,
metadata: Any,
omni_master_server: OmniMasterServer,
) -> StageDiffusionClient:
"""Launch a local diffusion stage on OmniMasterServer-allocated sockets."""
proc = None
try:
od_config = build_diffusion_config(self.model, stage_cfg, metadata)
handshake_address, request_address, response_address = register_stage_with_omni_master(
omni_master_address=omni_master_server.address,
omni_master_port=omni_master_server.port,
omni_stage_id=metadata.stage_id,
omni_stage_config=stage_cfg,
return_addresses=True,
)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion registration completed",
metadata.stage_id,
)
proc, _, _, _ = spawn_diffusion_proc(
self.model,
od_config,
handshake_address=handshake_address,
request_address=request_address,
response_address=response_address,
)
complete_diffusion_handshake(proc, handshake_address)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion startup completed",
metadata.stage_id,
)
return StageDiffusionClient.from_addresses(
metadata,
request_address=request_address,
response_address=response_address,
proc=proc,
batch_size=self.diffusion_batch_size,
)
except Exception:
if proc is not None:
terminate_alive_proc(proc)
raise
def _create_remote_diffusion_stage(
self,
metadata: Any,
stage_init_timeout: int,
omni_master_server: OmniMasterServer,
) -> StageDiffusionClient:
"""Attach to a remote diffusion stage registered with OmniMasterServer."""
remote_stage_cfg = OmegaConf.create(
omni_master_server.get_stage_config(
metadata.stage_id,
timeout_s=stage_init_timeout,
)
)
remote_metadata = extract_stage_metadata(remote_stage_cfg)
addresses = omni_master_server.get_zmq_addresses(metadata.stage_id)
logger.info(
"[AsyncOmniEngine] Stage %s remote diffusion startup completed",
metadata.stage_id,
)
return StageDiffusionClient.from_addresses(
remote_metadata,
request_address=addresses.inputs[0],
response_address=addresses.outputs[0],
batch_size=self.diffusion_batch_size,
)
def _attach_llm_stage(
self,
started: StartedLlmStage,
) -> tuple[Any, Any, Any, InputProcessor | None]:
"""Attach a READY LLM stage to the orchestrator event loop."""
client_addresses = {
client_addresses: dict[str, str] = {
"input_address": started.addresses.inputs[0],
"output_address": started.addresses.outputs[0],
}
@@ -430,14 +587,18 @@ class AsyncOmniEngine:
client_addresses["stats_update_address"] = started.addresses.frontend_stats_publish_address
try:
stage_client = StageEngineCoreClient(
stage_client = StageEngineCoreClientBase.make_async_mp_client(
vllm_config=started.vllm_config,
executor_class=started.executor_class,
metadata=started.metadata,
client_addresses=client_addresses,
proc=started.proc,
engine_manager=started.engine_manager,
coordinator=started.coordinator,
)
started.proc = None
started.engine_manager = None
started.coordinator = None
except Exception:
close_started_llm_stage(started)
raise
@@ -493,7 +654,7 @@ class AsyncOmniEngine:
output_processors: list[Any | None] = [None] * num_stages
stage_vllm_configs: list[Any | None] = [None] * num_stages
input_processor: InputProcessor | None = None
llm_stage_ids: list[int] = []
llm_stage_positions: list[int] = []
llm_launch_futures: dict[int, concurrent.futures.Future[StartedLlmStage]] = {}
started_llm_stages: dict[int, StartedLlmStage] = {}
llm_stage_launch_lock = threading.Lock()
@@ -507,45 +668,102 @@ class AsyncOmniEngine:
prepare_engine_environment()
omni_transfer_config = load_omni_transfer_config_for_model(self.model, self.config_path)
# ------------------------------------------------------------------ #
# Single-stage mode: start OmniMasterServer before launching stages. #
# ------------------------------------------------------------------ #
if self.single_stage_mode:
if not self._omni_master_address or not self._omni_master_port:
raise ValueError(
"AsyncOmniEngine single_stage_mode requires both "
"omni_master_address and omni_master_port to be set."
)
# Collect all configured stage IDs for pre-allocation.
all_stage_ids: list[int] = []
seen_stage_ids: set[int] = set()
for i, sc in enumerate(self.stage_configs):
stage_id = int(getattr(sc, "stage_id", i))
if stage_id in seen_stage_ids:
raise ValueError(
f"Duplicate stage_id {stage_id!r} detected among configured stages; stage_ids must be unique."
)
seen_stage_ids.add(stage_id)
all_stage_ids.append(stage_id)
self._omni_master_server = OmniMasterServer(
master_address=self._omni_master_address,
master_port=self._omni_master_port,
stage_ids=all_stage_ids,
)
self._omni_master_server.start()
logger.info(
"[AsyncOmniEngine] OmniMasterServer started for stages %s",
all_stage_ids,
)
try:
with concurrent.futures.ThreadPoolExecutor(
max_workers=max(1, llm_stage_count),
thread_name_prefix="llm-stage-launch",
) as launch_executor:
for stage_id, stage_cfg in enumerate(self.stage_configs):
logger.info("[AsyncOmniEngine] Initializing stage %s", stage_id)
for stage_idx, stage_cfg in enumerate(self.stage_configs):
metadata = extract_stage_metadata(stage_cfg)
configured_stage_id = metadata.stage_id
logger.info("[AsyncOmniEngine] Initializing stage %s", configured_stage_id)
if metadata.prompt_expand_func is not None:
prompt_expand_func = metadata.prompt_expand_func
if self.single_stage_mode:
metadata.runtime_cfg = None
stage_connector_spec = get_stage_connector_spec(
omni_transfer_config=omni_transfer_config,
stage_id=stage_id,
stage_id=configured_stage_id,
async_chunk=async_chunk,
)
omni_kv_connector = resolve_omni_kv_config_for_stage(omni_transfer_config, stage_id)
omni_kv_connector = resolve_omni_kv_config_for_stage(omni_transfer_config, configured_stage_id)
if metadata.stage_type == "diffusion":
is_remote_diffusion_stage = (
self.single_stage_mode
and self._single_stage_id_filter is not None
and configured_stage_id != self._single_stage_id_filter
)
if is_remote_diffusion_stage:
assert self._omni_master_server is not None
stage_clients[stage_idx] = self._create_remote_diffusion_stage(
metadata,
stage_init_timeout,
self._omni_master_server,
)
continue
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(stage_id, metadata.runtime_cfg)
setup_stage_devices(configured_stage_id, metadata.runtime_cfg)
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
if omni_conn_cfg:
from vllm_omni.entrypoints.utils import inject_omni_kv_config
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
_inject_kv_stage_info(stage_cfg, stage_id)
stage_clients[stage_id] = initialize_diffusion_stage(
self.model,
stage_cfg,
metadata,
batch_size=self.diffusion_batch_size,
)
inject_kv_stage_info(stage_cfg, configured_stage_id)
if self.single_stage_mode:
assert self._omni_master_server is not None
stage_clients[stage_idx] = self._launch_diffusion_stage(
stage_cfg,
metadata,
self._omni_master_server,
)
else:
stage_clients[stage_idx] = initialize_diffusion_stage(
self.model,
stage_cfg,
metadata,
batch_size=self.diffusion_batch_size,
)
logger.info(
"[AsyncOmniEngine] Stage %s initialized (diffusion, batch_size=%d)",
stage_id,
configured_stage_id,
self.diffusion_batch_size,
)
finally:
@@ -555,30 +773,58 @@ class AsyncOmniEngine:
current_omni_platform.set_device_control_env_var(previous_visible_devices)
continue
llm_stage_ids.append(stage_id)
llm_launch_futures[stage_id] = launch_executor.submit(
self._launch_llm_stage,
stage_cfg,
metadata,
stage_connector_spec,
stage_init_timeout,
llm_stage_launch_lock,
omni_kv_connector,
)
llm_stage_positions.append(stage_idx)
# In single-stage mode, stages that don't match the local
# stage_id filter are skipped.
if (
self.single_stage_mode
and self._single_stage_id_filter is not None
and configured_stage_id != self._single_stage_id_filter
):
assert self._omni_master_server is not None
llm_launch_futures[stage_idx] = launch_executor.submit(
self._create_remote_llm_stage,
stage_cfg,
metadata,
stage_connector_spec,
stage_init_timeout,
self._omni_master_server,
)
else:
llm_launch_futures[stage_idx] = launch_executor.submit(
self._launch_llm_stage,
stage_cfg,
metadata,
stage_connector_spec,
stage_init_timeout,
llm_stage_launch_lock,
omni_kv_connector,
)
concurrent.futures.wait(list(llm_launch_futures.values()))
for stage_id in llm_stage_ids:
started_llm_stages[stage_id] = llm_launch_futures[stage_id].result()
for stage_idx in llm_stage_positions:
started_llm_stages[stage_idx] = llm_launch_futures[stage_idx].result()
for stage_id in llm_stage_ids:
started = started_llm_stages[stage_id]
stage_client, output_processor, vllm_config, stage0_input_processor = self._attach_llm_stage(started)
stage_clients[stage_id] = stage_client
output_processors[stage_id] = output_processor
stage_vllm_configs[stage_id] = vllm_config
if stage0_input_processor is not None:
input_processor = stage0_input_processor
attach_futures: dict[concurrent.futures.Future[tuple[Any, Any, Any, InputProcessor | None]], int] = {}
with concurrent.futures.ThreadPoolExecutor(
max_workers=max(1, len(llm_stage_positions)),
thread_name_prefix="llm-stage-attach",
) as attach_executor:
for stage_idx in llm_stage_positions:
attach_futures[attach_executor.submit(self._attach_llm_stage, started_llm_stages[stage_idx])] = (
stage_idx
)
for future in concurrent.futures.as_completed(attach_futures):
stage_idx = attach_futures[future]
stage_client, output_processor, vllm_config, stage0_input_processor = future.result()
stage_clients[stage_idx] = stage_client
output_processors[stage_idx] = output_processor
stage_vllm_configs[stage_idx] = vllm_config
if stage0_input_processor is not None:
input_processor = stage0_input_processor
initialized_stage_clients, default_sampling_params_list, stage_metadata = finalize_initialized_stages(
stage_clients,
@@ -595,8 +841,13 @@ class AsyncOmniEngine:
)
cleanup_failed_stage_initialization(
stage_clients,
[started_llm_stages[stage_id] for stage_id in llm_stage_ids if stage_id in started_llm_stages],
[started_llm_stages[stage_idx] for stage_idx in llm_stage_positions if stage_idx in started_llm_stages],
)
if self._omni_master_server is not None:
try:
self._omni_master_server.stop()
except Exception:
logger.exception("[AsyncOmniEngine] Failed to stop OmniMasterServer during stage-init cleanup")
raise
self.stage_clients = initialized_stage_clients
@@ -1310,3 +1561,10 @@ class AsyncOmniEngine:
q.close()
except Exception:
pass
if self._omni_master_server is not None:
try:
self._omni_master_server.stop()
except Exception:
logger.exception("[AsyncOmniEngine] Failed to stop OmniMasterServer during shutdown")
self._omni_master_server = None
+72 -13
View File
@@ -12,7 +12,7 @@ from urllib.parse import urlparse
from vllm.logger import init_logger
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.core_client import AsyncMPClient
from vllm.v1.engine.core_client import AsyncMPClient, DPLBAsyncMPClient
from vllm_omni.distributed.omni_connectors.utils.initialization import KV_TRANSFER_PORT_OFFSET
from vllm_omni.engine.stage_init_utils import StageMetadata
@@ -25,18 +25,54 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class StageEngineCoreClient(AsyncMPClient):
"""Stage async client that inherits from vLLM's AsyncMPClient.
class StageEngineCoreClientBase:
"""Shared stage-aware behavior for async EngineCore clients.
Fully reuses AsyncMPClient for:
The concrete transport/load-balancing behavior is supplied by the
multiprocessing client subclass in the MRO.
Fully reuses the underlying vLLM async MP client ``__init__`` for:
- ZMQ setup, sockets
- outputs_queue, output_queue_task
- All utility methods (get_output_async, abort_requests_async, etc.)
The subprocess is spawned externally via ``spawn_stage_core`` /
``complete_stage_handshake`` from *stage_engine_core_proc.py*.
In single-stage CLI mode, the client may instead attach to an
``engine_manager`` / ``coordinator`` pair created elsewhere.
"""
@staticmethod
def make_async_mp_client(
vllm_config: Any,
executor_class: type,
metadata: StageMetadata,
client_addresses: dict[str, str] | None = None,
proc: Any = None,
engine_manager: Any = None,
coordinator: Any = None,
client_count: int = 1,
client_index: int = 0,
) -> StageEngineCoreClient | DPLBStageEngineCoreClient:
"""Create the appropriate stage async client for the DP mode."""
parallel_config = vllm_config.parallel_config
client_args = dict(
vllm_config=vllm_config,
executor_class=executor_class,
metadata=metadata,
client_addresses=client_addresses,
proc=proc,
engine_manager=engine_manager,
coordinator=coordinator,
client_count=client_count,
client_index=client_index,
)
if parallel_config.data_parallel_size > 1 and not parallel_config.data_parallel_external_lb:
return DPLBStageEngineCoreClient(**client_args)
return StageEngineCoreClient(**client_args)
def __init__(
self,
vllm_config: Any,
@@ -85,8 +121,10 @@ class StageEngineCoreClient(AsyncMPClient):
self._kv_sender_info: dict[str, Any] | None = None
self._kv_sender_initialized = False
client_name = self.__class__.__name__
logger.info(
"[StageEngineCoreClient] Stage-%s initializing EngineCore",
"[%s] Stage-%s initializing EngineCore",
client_name,
self.stage_id,
)
try:
@@ -98,23 +136,30 @@ class StageEngineCoreClient(AsyncMPClient):
client_count=client_count,
client_index=client_index,
)
if engine_manager is not None:
self.resources.engine_manager = engine_manager
if coordinator is not None:
self.resources.coordinator = coordinator
except Exception:
logger.exception(
"[StageEngineCoreClient] Stage-%s EngineCore init failed",
"[%s] Stage-%s EngineCore init failed",
client_name,
self.stage_id,
)
try:
self.shutdown()
except Exception as shutdown_error:
logger.warning(
"[StageEngineCoreClient] Stage-%s cleanup after init failure failed: %s",
"[%s] Stage-%s cleanup after init failure failed: %s",
client_name,
self.stage_id,
shutdown_error,
)
raise
self._initialize_kv_sender_endpoint()
logger.info(
"[StageEngineCoreClient] Stage-%s EngineCore running",
"[%s] Stage-%s EngineCore running",
client_name,
self.stage_id,
)
@@ -122,7 +167,12 @@ class StageEngineCoreClient(AsyncMPClient):
async def add_request_async(self, request: EngineCoreRequest) -> None:
"""Add request to the stage engine core."""
logger.info(f"[StageEngineCoreClient] Stage-{self.stage_id} adding request: {request.request_id}")
logger.info(
"[%s] Stage-%s adding request: %s",
self.__class__.__name__,
self.stage_id,
request.request_id,
)
await super().add_request_async(request)
# ==================== Stage Methods ====================
@@ -287,9 +337,9 @@ class StageEngineCoreClient(AsyncMPClient):
) -> Any:
"""Forward control RPCs to the underlying AsyncMPClient stage engine.
Each ``StageEngineCoreClient`` already represents one logical stage, so
stage-scoped control operations should be executed here and then fanned
in-core across the workers managed by this EngineCore client.
Each stage client already represents one logical stage, so stage-scoped
control operations should be executed here and then fanned in-core
across the workers managed by this EngineCore client.
"""
return await super().collective_rpc_async(
method=method,
@@ -299,10 +349,19 @@ class StageEngineCoreClient(AsyncMPClient):
)
def shutdown(self) -> None:
"""Shutdown ZMQ connections and the subprocess."""
"""Shutdown managed resources and any externally spawned subprocess."""
super().shutdown()
if self._proc is not None and self._proc.is_alive():
self._proc.terminate()
self._proc.join(timeout=5)
if self._proc.is_alive():
self._proc.kill()
self._proc = None
class StageEngineCoreClient(StageEngineCoreClientBase, AsyncMPClient):
"""Stage async client backed by vLLM's ``AsyncMPClient``."""
class DPLBStageEngineCoreClient(StageEngineCoreClientBase, DPLBAsyncMPClient):
"""Stage async client backed by vLLM's ``DPLBAsyncMPClient``."""
+599
View File
@@ -0,0 +1,599 @@
"""Helpers for launching and handshaking omni engine cores."""
from __future__ import annotations
import contextlib
import dataclasses
import threading
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Any
import msgspec
import zmq
from omegaconf import OmegaConf
from vllm.config import CacheConfig, VllmConfig
from vllm.logger import init_logger
from vllm.utils.network_utils import get_open_port, zmq_socket_ctx
from vllm.v1.engine.coordinator import DPCoordinator
from vllm.v1.engine.utils import (
STARTUP_POLL_PERIOD_MS,
CoreEngine,
CoreEngineProcManager,
CoreEngineState,
EngineHandshakeMetadata,
EngineZmqAddresses,
wait_for_engine_startup,
)
from vllm.v1.executor import Executor
logger = init_logger(__name__)
# Poll period (ms) used by the registration/handshake loop.
_POLL_PERIOD_MS = 5_000
# Default timeout (s) for a stage to send READY.
_DEFAULT_STARTUP_TIMEOUT_S = 300
def _serialize_stage_config(stage_config: Any) -> Any:
"""Convert a stage config to msgpack-friendly builtins."""
if stage_config is None or isinstance(stage_config, (str, bytes, int, float, bool)):
return stage_config
if OmegaConf.is_config(stage_config):
return _serialize_stage_config(OmegaConf.to_container(stage_config, resolve=True))
if dataclasses.is_dataclass(stage_config):
return _serialize_stage_config(dataclasses.asdict(stage_config))
if isinstance(stage_config, dict):
return {key: _serialize_stage_config(value) for key, value in stage_config.items() if not callable(value)}
if isinstance(stage_config, (list, tuple, set)):
return [_serialize_stage_config(item) for item in stage_config if not callable(item)]
if hasattr(stage_config, "items"):
return {key: _serialize_stage_config(value) for key, value in stage_config.items() if not callable(value)}
if hasattr(stage_config, "__dict__"):
return {
key: _serialize_stage_config(value)
for key, value in vars(stage_config).items()
if not key.startswith("_") and not callable(value)
}
return stage_config
# ---------------------------------------------------------------------------
# Per-stage address allocation
# ---------------------------------------------------------------------------
@dataclass
class StageAllocation:
"""ZMQ addresses reserved for a single stage."""
# Per-stage handshake socket (OmniMasterServer binds, engine connects)
handshake_bind_address: str
handshake_connect_address: str
# Input channel: client binds ROUTER, engine connects DEALER
input_bind_address: str
input_connect_address: str
# Output channel: client binds PULL, engine connects PUSH
output_bind_address: str
output_connect_address: str
@dataclass(frozen=True)
class StageCoordinatorAddresses:
"""Optional DP coordinator addresses registered for a stage."""
coordinator_input: str | None = None
coordinator_output: str | None = None
frontend_stats_publish_address: str | None = None
# ---------------------------------------------------------------------------
# OmniMasterServer
# ---------------------------------------------------------------------------
class OmniMasterServer:
"""Registration server for single-stage engine startup."""
def __init__(
self,
master_address: str,
master_port: int,
stage_ids: list[int],
) -> None:
self._address = master_address
self._port = master_port
self._allocations: dict[int, StageAllocation] = {}
self._stage_configs: dict[int, Any] = {}
self._stage_coordinator_addresses: dict[int, StageCoordinatorAddresses] = {}
self._stage_config_events: dict[int, threading.Event] = {}
self._thread: threading.Thread | None = None
self._stop_event = threading.Event()
for sid in stage_ids:
self._stage_config_events[sid] = threading.Event()
self._stage_coordinator_addresses[sid] = StageCoordinatorAddresses()
hs_port = get_open_port()
inp_port = get_open_port()
out_port = get_open_port()
self._allocations[sid] = StageAllocation(
handshake_bind_address=f"tcp://{master_address}:{hs_port}",
handshake_connect_address=f"tcp://{master_address}:{hs_port}",
input_bind_address=f"tcp://{master_address}:{inp_port}",
input_connect_address=f"tcp://{master_address}:{inp_port}",
output_bind_address=f"tcp://{master_address}:{out_port}",
output_connect_address=f"tcp://{master_address}:{out_port}",
)
logger.info(
"[OmniMasterServer] Pre-allocated addresses for stages %s (master=%s:%d)",
list(stage_ids),
master_address,
master_port,
)
# ------------------------------------------------------------------
# Public helpers
# ------------------------------------------------------------------
@property
def address(self) -> str:
"""Return the registration address exposed to stage launchers."""
return self._address
@property
def port(self) -> int:
"""Return the registration port exposed to stage launchers."""
return self._port
def get_allocation(self, stage_id: int) -> StageAllocation:
"""Return the full address allocation for *stage_id*."""
return self._allocations[stage_id]
def register_stage_config(
self,
stage_id: int,
stage_config: Any,
coordinator_addresses: StageCoordinatorAddresses | None = None,
) -> None:
"""Store the latest stage registration payload for *stage_id*."""
if stage_id not in self._allocations:
raise KeyError(stage_id)
self._stage_configs[stage_id] = stage_config
if coordinator_addresses is not None:
self._stage_coordinator_addresses[stage_id] = coordinator_addresses
self._stage_config_events[stage_id].set()
def get_stage_config(self, stage_id: int, timeout_s: float | None = None) -> Any:
"""Return the stage config for *stage_id*, waiting if necessary."""
if stage_id not in self._allocations:
raise KeyError(stage_id)
if stage_id in self._stage_configs:
return self._stage_configs[stage_id]
if not self._stage_config_events[stage_id].wait(timeout=timeout_s):
raise TimeoutError(f"Timed out waiting for stage config for stage {stage_id}.")
return self._stage_configs[stage_id]
def get_stage_coordinator_addresses(
self,
stage_id: int,
timeout_s: float | None = None,
) -> StageCoordinatorAddresses:
"""Return the registered coordinator addresses for *stage_id*."""
if stage_id not in self._allocations:
raise KeyError(stage_id)
if not self._stage_config_events[stage_id].is_set():
if not self._stage_config_events[stage_id].wait(timeout=timeout_s):
raise TimeoutError(f"Timed out waiting for stage registration for stage {stage_id}.")
return self._stage_coordinator_addresses[stage_id]
def get_client_addresses(self, stage_id: int) -> dict[str, str]:
"""Return the addresses the client-side sockets should *bind* to."""
alloc = self._allocations[stage_id]
return {
"input_address": alloc.input_bind_address,
"output_address": alloc.output_bind_address,
}
def get_zmq_addresses(self, stage_id: int) -> EngineZmqAddresses:
"""Return EngineZmqAddresses using the *bind* (client) side addresses."""
alloc = self._allocations[stage_id]
return EngineZmqAddresses(
inputs=[alloc.input_bind_address],
outputs=[alloc.output_bind_address],
)
def get_engine_zmq_addresses(self, stage_id: int) -> EngineZmqAddresses:
"""Return EngineZmqAddresses using the *connect* (engine) addresses."""
alloc = self._allocations[stage_id]
return EngineZmqAddresses(
inputs=[alloc.input_connect_address],
outputs=[alloc.output_connect_address],
)
# ------------------------------------------------------------------
# Lifecycle
# ------------------------------------------------------------------
def start(self) -> None:
"""Start the background server thread."""
self._thread = threading.Thread(
target=self._run,
name="OmniMasterServer",
daemon=True,
)
self._thread.start()
logger.info(
"[OmniMasterServer] Listening on tcp://%s:%d",
self.address,
self.port,
)
def stop(self) -> None:
"""Signal stop and join the background thread."""
self._stop_event.set()
if self._thread is not None:
self._thread.join(timeout=10)
# ------------------------------------------------------------------
# Internal server logic
# ------------------------------------------------------------------
def _run(self) -> None:
ctx = zmq.Context()
try:
self._serve(ctx)
except Exception:
logger.exception("[OmniMasterServer] Server thread crashed")
finally:
ctx.term()
def _serve(self, ctx: zmq.Context) -> None: # type: ignore[type-arg]
# Registration socket for the initial stage registration.
# Per-stage handshake sockets are bound by the launch helpers.
reg_socket: zmq.Socket = ctx.socket(zmq.ROUTER) # type: ignore[attr-defined]
reg_socket.bind(f"tcp://{self.address}:{self.port}")
poller = zmq.Poller()
poller.register(reg_socket, zmq.POLLIN)
pending: set[int] = set(self._allocations.keys())
while pending and not self._stop_event.is_set():
events: list[tuple[zmq.Socket, int]] = poller.poll(_POLL_PERIOD_MS) # type: ignore[assignment]
if not events:
logger.debug("[OmniMasterServer] Still waiting for registration from stages: %s", pending)
continue
for sock, _ in events:
if sock is reg_socket:
sid = self._handle_registration(reg_socket)
if sid is not None:
pending.discard(sid)
# Cleanup
reg_socket.close(linger=0)
logger.info("[OmniMasterServer] All stages registered; server thread exiting.")
def _handle_registration(self, reg_socket: zmq.Socket) -> int | None: # type: ignore[type-arg]
"""Receive a stage registration and reply with the handshake address.
Returns the registered stage_id on success, or None on failure.
"""
frames = reg_socket.recv_multipart()
if len(frames) < 2:
logger.warning(
"[OmniMasterServer] Unexpected registration frame count: %d",
len(frames),
)
return None
identity = frames[0]
msg_bytes = frames[-1]
try:
msg = msgspec.msgpack.decode(msg_bytes)
except Exception as exc:
logger.warning("[OmniMasterServer] Failed to decode registration message: %s", exc)
return None
stage_id: int | None = msg.get("stage_id")
if stage_id not in self._allocations:
logger.warning(
"[OmniMasterServer] Received registration for unknown stage_id=%s",
stage_id,
)
return None
self.register_stage_config(
stage_id,
msg.get("stage_config"),
coordinator_addresses=StageCoordinatorAddresses(
coordinator_input=msg.get("coordinator_input"),
coordinator_output=msg.get("coordinator_output"),
frontend_stats_publish_address=msg.get("frontend_stats_publish_address"),
),
)
alloc = self._allocations[stage_id]
response = msgspec.msgpack.encode(
{
"handshake_address": alloc.handshake_connect_address,
"input_address": alloc.input_bind_address,
"output_address": alloc.output_bind_address,
}
)
# ROUTER-DEALER: reply is [identity, payload] (no empty delimiter).
reg_socket.send_multipart([identity, response])
logger.info(
"[OmniMasterServer] Stage %d registered; assigned handshake=%s",
stage_id,
alloc.handshake_connect_address,
)
return stage_id
def register_stage_with_omni_master(
*,
omni_master_address: str,
omni_master_port: int,
omni_stage_id: int,
omni_stage_config: Any = None,
coordinator: DPCoordinator | None = None,
return_addresses: bool = False,
) -> str | tuple[str, str, str]:
"""Register a stage with the omni master server.
Returns the per-stage handshake address by default. When
``return_addresses`` is true, also returns the stage input/output
addresses allocated by the master.
"""
reg_ctx = zmq.Context()
try:
reg_sock: zmq.Socket = reg_ctx.socket(zmq.DEALER) # type: ignore[attr-defined]
try:
reg_sock.connect(f"tcp://{omni_master_address}:{omni_master_port}")
payload = {
"stage_id": omni_stage_id,
"stage_config": _serialize_stage_config(omni_stage_config),
}
if coordinator is not None:
coordinator_input, coordinator_output = coordinator.get_engine_socket_addresses()
payload["coordinator_input"] = coordinator_input
payload["coordinator_output"] = coordinator_output
payload["frontend_stats_publish_address"] = coordinator.get_stats_publish_address()
reg_sock.send(msgspec.msgpack.encode(payload))
timeout_ms = _DEFAULT_STARTUP_TIMEOUT_S * 1_000
if not reg_sock.poll(timeout=timeout_ms):
raise RuntimeError(
f"Timed out waiting for registration "
f"response from OmniMasterServer "
f"({omni_master_address}:{omni_master_port}) "
f"for stage {omni_stage_id}."
)
response_bytes = reg_sock.recv()
response = msgspec.msgpack.decode(response_bytes)
handshake_address: str = response["handshake_address"]
input_address: str = response["input_address"]
output_address: str = response["output_address"]
logger.info(
"Stage %d registered; handshake_address=%s",
omni_stage_id,
handshake_address,
)
finally:
reg_sock.close(linger=0)
finally:
reg_ctx.term()
if return_addresses:
return handshake_address, input_address, output_address
return handshake_address
def _wait_for_omni_engine_startup(
handshake_socket: zmq.Socket,
engine_addresses: EngineZmqAddresses,
engines: list[CoreEngine],
cache_config: CacheConfig,
) -> None:
"""Wait for omni-managed engines to finish the HELLO/READY handshake."""
conn_pending = len(engines)
start_pending = 0
poller = zmq.Poller()
poller.register(handshake_socket, zmq.POLLIN)
while conn_pending or start_pending:
events = poller.poll(STARTUP_POLL_PERIOD_MS)
if not events:
logger.debug(
"[omni] Waiting for %d engine(s) to connect, %d to start.",
conn_pending,
start_pending,
)
continue
eng_identity, msg_bytes = handshake_socket.recv_multipart()
eng_index = int.from_bytes(eng_identity, "little")
engine = next((e for e in engines if e.identity == eng_identity), None)
if engine is None:
raise RuntimeError(f"[omni] Handshake message from unexpected engine rank: {eng_index}")
msg = msgspec.msgpack.decode(msg_bytes)
status: str = msg["status"]
if status == "HELLO" and engine.state == CoreEngineState.NEW:
init_message = msgspec.msgpack.encode(
EngineHandshakeMetadata(addresses=engine_addresses, parallel_config={})
)
handshake_socket.send_multipart((eng_identity, init_message), copy=False)
conn_pending -= 1
start_pending += 1
engine.state = CoreEngineState.CONNECTED
logger.debug("[omni] HELLO from engine %d", eng_index)
elif status == "READY" and engine.state == CoreEngineState.CONNECTED:
num_gpu_blocks = (cache_config.num_gpu_blocks or 0) + msg["num_gpu_blocks"]
cache_config.num_gpu_blocks = num_gpu_blocks
if engine_addresses.frontend_stats_publish_address is None:
engine_addresses.frontend_stats_publish_address = msg.get("dp_stats_address")
start_pending -= 1
engine.state = CoreEngineState.READY
logger.debug("[omni] READY from engine %d (num_gpu_blocks=%d)", eng_index, msg["num_gpu_blocks"])
else:
raise RuntimeError(f"[omni] Unexpected status '{status}' from engine {eng_index} in state {engine.state}.")
@contextlib.contextmanager
def connect_remote_engine_cores(
vllm_config: VllmConfig,
omni_master_server: OmniMasterServer,
stage_id: int,
) -> Iterator[tuple[None, DPCoordinator | None, EngineZmqAddresses]]:
"""Wait for remote engine cores to connect through the omni handshake."""
addresses = omni_master_server.get_zmq_addresses(stage_id)
parallel_config = vllm_config.parallel_config
# Mirror the engine-count logic from launch_omni_core_engines.
remote_engine_count = (
parallel_config.data_parallel_size_local
if parallel_config.data_parallel_size_local is not None and parallel_config.data_parallel_size_local > 0
else max(1, parallel_config.data_parallel_size)
)
start_index = parallel_config.data_parallel_rank if parallel_config.data_parallel_rank is not None else 0
coordinator = None
registered_coordinator_addresses = omni_master_server.get_stage_coordinator_addresses(stage_id)
addresses.coordinator_input = registered_coordinator_addresses.coordinator_input
addresses.coordinator_output = registered_coordinator_addresses.coordinator_output
addresses.frontend_stats_publish_address = registered_coordinator_addresses.frontend_stats_publish_address
engines_to_handshake = [CoreEngine(index=start_index + i, local=False) for i in range(remote_engine_count)]
logger.info(
"Waiting for %d remote engine(s) for stage %d",
remote_engine_count,
stage_id,
)
handshake_bind_address = omni_master_server.get_allocation(stage_id).handshake_bind_address
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
yield None, coordinator, addresses
_wait_for_omni_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
vllm_config.cache_config,
)
@contextlib.contextmanager
def launch_omni_core_engines(
vllm_config: VllmConfig,
executor_class: type[Executor],
log_stats: bool,
omni_master_server: OmniMasterServer,
stage_id: int,
stage_config: Any = None,
) -> Iterator[tuple[CoreEngineProcManager, DPCoordinator | None, EngineZmqAddresses]]:
"""Launch local engine cores using the omni registration flow."""
addresses = omni_master_server.get_zmq_addresses(stage_id)
parallel_config = vllm_config.parallel_config
# Determine the number of local engines and their ranks.
local_engine_count = (
parallel_config.data_parallel_size_local
if parallel_config.data_parallel_size_local is not None and parallel_config.data_parallel_size_local > 0
else max(1, parallel_config.data_parallel_size)
)
dp_rank = parallel_config.data_parallel_rank if parallel_config.data_parallel_rank is not None else 0
local_start_index = 0
start_index = dp_rank
# Run the DP Coordinator process with rank 0 when in online DP mode.
# The coordinator is needed for:
# 1. Internal/hybrid LB: collecting and publishing queue stats
# 2. MoE models: wave coordination in addition to stats
run_coordinator = vllm_config.needs_dp_coordinator and dp_rank == 0
if run_coordinator:
coordinator = DPCoordinator(
parallel_config,
enable_wave_coordination=vllm_config.model_config.is_moe,
)
addresses.coordinator_input, addresses.coordinator_output = coordinator.get_engine_socket_addresses()
addresses.frontend_stats_publish_address = coordinator.get_stats_publish_address()
logger.info(
"[omni] Started DP Coordinator process for stage %d (PID: %d)",
stage_id,
coordinator.proc.pid,
)
else:
coordinator = None
logger.info(
"Starting %d local engine(s) for stage %d (dp_rank=%d)",
local_engine_count,
stage_id,
dp_rank,
)
# Register the stage once and reuse the returned per-stage handshake
# address for all local engine-core processes.
handshake_address = register_stage_with_omni_master(
omni_master_address=omni_master_server.address,
omni_master_port=omni_master_server.port,
omni_stage_id=stage_id,
omni_stage_config=stage_config,
coordinator=coordinator,
)
# One CoreEngine entry per local engine so wait_for_engine_startup can
# track the HELLO/READY handshake for each of them.
engines_to_handshake = [CoreEngine(index=start_index + i, local=True) for i in range(local_engine_count)]
# Bind the pre-allocated handshake socket for this stage.
handshake_bind_address = omni_master_server.get_allocation(stage_id).handshake_bind_address
with zmq_socket_ctx(handshake_bind_address, zmq.ROUTER, bind=True) as handshake_socket:
local_engine_manager = CoreEngineProcManager(
local_engine_count=local_engine_count,
start_index=start_index,
local_start_index=local_start_index,
vllm_config=vllm_config,
local_client=True,
handshake_address=handshake_address,
executor_class=executor_class,
log_stats=log_stats,
)
yield local_engine_manager, coordinator, addresses
# Wait for all local engine-core processes to complete the
# standard HELLO/READY handshake — mirrors launch_core_engines.
coordinated_dp = parallel_config.data_parallel_size > 1 and vllm_config.model_config.is_moe
wait_for_engine_startup(
handshake_socket,
addresses,
engines_to_handshake,
parallel_config,
coordinated_dp,
vllm_config.cache_config,
local_engine_manager,
coordinator.proc if coordinator else None,
)
+80 -35
View File
@@ -101,6 +101,34 @@ def resolve_worker_cls(engine_args: dict[str, Any]) -> None:
raise ValueError(f"Unknown worker_type: {worker_type}")
def inject_kv_stage_info(stage_cfg: Any, stage_id: int) -> None:
"""Inject stage metadata into omni_kv_config when present."""
try:
engine_args = stage_cfg.engine_args
if hasattr(engine_args, "get"):
omni_kv = engine_args.get("omni_kv_config", None)
else:
omni_kv = getattr(engine_args, "omni_kv_config", None)
if omni_kv is None:
return
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("stage_id", stage_id)
elif hasattr(omni_kv, "__setitem__"):
if "stage_id" not in omni_kv:
omni_kv["stage_id"] = stage_id
engine_input_source = getattr(stage_cfg, "engine_input_source", None)
if engine_input_source is not None:
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("engine_input_source", list(engine_input_source))
elif hasattr(omni_kv, "__setitem__") and "engine_input_source" not in omni_kv:
omni_kv["engine_input_source"] = list(engine_input_source)
except Exception as e:
logger.debug("Failed to inject stage info into omni_kv_config: %s", e)
@dataclass
class StageMetadata:
"""Lightweight stage attributes extracted from stage_config."""
@@ -129,8 +157,10 @@ class StartedLlmStage:
metadata: Any
vllm_config: Any
executor_class: type
proc: Any
addresses: Any
proc: Any = None
engine_manager: Any = None
coordinator: Any = None
def extract_stage_metadata(stage_config: Any) -> StageMetadata:
@@ -263,6 +293,7 @@ def build_vllm_config(
model: str,
stage_connector_spec: dict[str, Any] | None = None,
engine_args_dict: dict[str, Any] | None = None,
headless: bool = False,
) -> tuple[Any, type]:
"""Build engine args, then create VllmConfig and executor_class.
@@ -278,7 +309,10 @@ def build_vllm_config(
filtered_engine_args_dict = filter_dataclass_kwargs(OmniEngineArgs, engine_args_dict)
omni_engine_args = OmniEngineArgs(**filtered_engine_args_dict)
vllm_config = omni_engine_args.create_engine_config(usage_context=UsageContext.LLM_CLASS)
vllm_config = omni_engine_args.create_engine_config(
usage_context=UsageContext.LLM_CLASS,
headless=headless,
)
executor_class = Executor.get_class(vllm_config)
return vllm_config, executor_class
@@ -445,6 +479,37 @@ def get_stage_connector_spec(
return {}
def build_diffusion_config(
model: str,
stage_cfg: Any,
metadata: StageMetadata,
) -> Any:
"""Build diffusion config for a stage."""
from vllm_omni.diffusion.data import OmniDiffusionConfig
engine_args_dict = build_engine_args_dict(stage_cfg, model)
od_config = OmniDiffusionConfig.from_kwargs(**engine_args_dict)
num_devices_per_stage = od_config.parallel_config.world_size
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = os.environ.get(device_control_env) if device_control_env else None
if visible_devices_str:
physical_devices = [device.strip() for device in visible_devices_str.split(",") if device.strip()]
else:
physical_devices = list(range(current_omni_platform.get_device_count()))
if len(physical_devices) < num_devices_per_stage:
raise ValueError(
f"Stage {metadata.stage_id} requires {num_devices_per_stage} device(s) based on parallel_config, "
f"but {len(physical_devices)} device(s) are available: {physical_devices}"
)
od_config.num_gpus = num_devices_per_stage
if metadata.cfg_kv_collect_func is not None:
od_config.cfg_kv_collect_func = metadata.cfg_kv_collect_func
return od_config
def initialize_diffusion_stage(
model: str,
stage_cfg: Any,
@@ -461,30 +526,9 @@ def initialize_diffusion_stage(
diffusion engine. Passed through to ``StageDiffusionClient``
and ultimately to ``AsyncOmni``.
"""
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
od_config = OmniDiffusionConfig.from_kwargs(
model=model,
**_to_dict(stage_cfg.engine_args),
)
num_devices_per_stage = od_config.parallel_config.world_size
device_control_env = current_omni_platform.device_control_env_var
visible_devices_str = os.environ.get(device_control_env)
if visible_devices_str:
physical_devices = [device.strip() for device in visible_devices_str.split(",") if device.strip()]
else:
physical_devices = list(range(current_omni_platform.get_device_count()))
if len(physical_devices) < num_devices_per_stage:
raise ValueError(
f"Stage {metadata.stage_id} requires {num_devices_per_stage} device(s) based on parallel_config, "
f"but {len(physical_devices)} device(s) are available: {physical_devices}"
)
od_config.num_gpus = num_devices_per_stage
if metadata.cfg_kv_collect_func is not None:
od_config.cfg_kv_collect_func = metadata.cfg_kv_collect_func
od_config = build_diffusion_config(model, stage_cfg, metadata)
return StageDiffusionClient(model, od_config, metadata, batch_size=batch_size)
@@ -518,17 +562,18 @@ def _shutdown_or_close_resource(resource: Any, resource_name: str, stage_id: int
def close_started_llm_stage(started: StartedLlmStage) -> None:
"""Terminate the subprocess owned by a launched stage that never attached."""
if started.proc is None:
return
try:
terminate_alive_proc(started.proc)
except Exception as cleanup_error:
logger.warning(
"[stage_init] Failed to terminate process for stage %s: %s",
started.stage_id,
cleanup_error,
)
"""Release resources owned by a launched stage that never attached."""
if started.proc is not None:
try:
terminate_alive_proc(started.proc)
except Exception as cleanup_error:
logger.warning(
"[stage_init] Failed to terminate process for stage %s: %s",
started.stage_id,
cleanup_error,
)
_shutdown_or_close_resource(started.engine_manager, "engine manager", started.stage_id)
_shutdown_or_close_resource(started.coordinator, "coordinator", started.stage_id)
def finalize_initialized_stages(