[Entrypoint][Refactor]Stage CLI Refactor (#2020)

Signed-off-by: wuhang <wuhang6@huawei.com>
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
wuhangandDidan Deng authored and GitHub committed 2026-04-10 14:40:37 +08:00
1 parent f3f2dc590c
commit cb91cbe61e
19 files changed
+3773 -273

No files matched your search

+392 -134
View File
@@ -18,12 +18,10 @@ import time
import uuid
import weakref
from collections.abc import Mapping, Sequence
from contextlib import ExitStack
from dataclasses import asdict
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from vllm_omni.engine.arg_utils import OmniEngineArgs
import janus
import torch
from omegaconf import OmegaConf
@@ -34,26 +32,36 @@ from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.input_processor import InputProcessor
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
from vllm_omni.diffusion.stage_diffusion_proc import (
complete_diffusion_handshake,
spawn_diffusion_proc,
)
from vllm_omni.distributed.omni_connectors.utils.initialization import (
resolve_omni_kv_config_for_stage,
)
from vllm_omni.engine import (
OmniEngineCoreRequest,
)
from vllm_omni.engine import OmniEngineCoreRequest
from vllm_omni.engine.orchestrator import Orchestrator
from vllm_omni.engine.output_processor import MultimodalOutputProcessor
from vllm_omni.engine.serialization import (
deserialize_additional_information,
serialize_additional_information,
)
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
from vllm_omni.engine.stage_engine_core_proc import (
complete_stage_handshake,
spawn_stage_core,
)
from vllm_omni.engine.stage_engine_startup import (
OmniMasterServer,
connect_remote_engine_cores,
launch_omni_core_engines,
register_stage_with_omni_master,
)
from vllm_omni.engine.stage_init_utils import (
StartedLlmStage,
acquire_device_locks,
build_diffusion_config,
build_engine_args_dict,
build_vllm_config,
cleanup_failed_stage_initialization,
@@ -62,17 +70,20 @@ from vllm_omni.engine.stage_init_utils import (
finalize_initialized_stages,
get_stage_connector_spec,
initialize_diffusion_stage,
inject_kv_stage_info,
load_omni_transfer_config_for_model,
prepare_engine_environment,
release_device_locks,
setup_stage_devices,
terminate_alive_proc,
)
from vllm_omni.entrypoints.utils import (
load_and_resolve_stage_configs,
)
from vllm_omni.entrypoints.utils import load_and_resolve_stage_configs
from vllm_omni.inputs.preprocess import OmniInputPreprocessor
from vllm_omni.platforms import current_omni_platform
if TYPE_CHECKING:
from vllm_omni.engine.arg_utils import OmniEngineArgs
logger = init_logger(__name__)
@@ -86,39 +97,6 @@ def _patch_generation_config_if_needed(model_config: Any) -> None:
model_config.try_get_generation_config = lambda: {}
def _inject_kv_stage_info(stage_cfg: Any, stage_id: int) -> None:
"""Inject stage_id and engine_input_source into omni_kv_config.
OmniKVTransferManager needs stage_id to compute recv_stages for the
receiving side. In the old Omni architecture, OmniDiffusion.__init__
performed this injection; replicate it here for AsyncOmniEngine.
"""
try:
engine_args = stage_cfg.engine_args
if hasattr(engine_args, "get"):
omni_kv = engine_args.get("omni_kv_config", None)
else:
omni_kv = getattr(engine_args, "omni_kv_config", None)
if omni_kv is None:
return
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("stage_id", stage_id)
elif hasattr(omni_kv, "__setitem__"):
if "stage_id" not in omni_kv:
omni_kv["stage_id"] = stage_id
engine_input_source = getattr(stage_cfg, "engine_input_source", None)
if engine_input_source is not None:
if hasattr(omni_kv, "setdefault"):
omni_kv.setdefault("engine_input_source", list(engine_input_source))
elif hasattr(omni_kv, "__setitem__") and "engine_input_source" not in omni_kv:
omni_kv["engine_input_source"] = list(engine_input_source)
except Exception as e:
logger.debug("Failed to inject stage info into omni_kv_config: %s", e)
def _inject_global_id(target: Any, request_id: str) -> None:
"""Inject global_request_id into a prompt dict's additional_information."""
if isinstance(target, dict):
@@ -255,6 +233,7 @@ class AsyncOmniEngine:
stage_init_timeout: int = 300,
init_timeout: int = 600,
diffusion_batch_size: int = 1,
single_stage_mode: bool = False,
**kwargs: Any,
) -> None:
self.model = model
@@ -274,6 +253,31 @@ class AsyncOmniEngine:
ea_dict.pop("model", None)
kwargs = {**ea_dict, **kwargs}
# ------------------------------------------------------------------ #
# Single-stage mode detection #
# ------------------------------------------------------------------ #
# Single-stage mode is enabled when the caller explicitly passes #
# single_stage_mode=True, or when a stage_id is provided in the args. #
_stage_id_kwarg = kwargs.get("stage_id")
if isinstance(_stage_id_kwarg, int) and not single_stage_mode:
single_stage_mode = True
self.single_stage_mode: bool = single_stage_mode
self._single_stage_id_filter: int | None = (
int(_stage_id_kwarg) if single_stage_mode and isinstance(_stage_id_kwarg, int) else None
)
self._omni_master_address: str | None = kwargs.get("omni_master_address")
self._omni_master_port: int | None = kwargs.get("omni_master_port")
self._omni_master_server: OmniMasterServer | None = None
if single_stage_mode:
logger.info(
"[AsyncOmniEngine] Single-stage mode enabled (stage_id_filter=%s, master=%s:%s)",
self._single_stage_id_filter,
self._omni_master_address,
self._omni_master_port,
)
self.config_path, self.stage_configs = self._resolve_stage_configs(model, kwargs)
self.num_stages = len(self.stage_configs)
@@ -350,61 +354,89 @@ class AsyncOmniEngine:
started_stage: StartedLlmStage | None = None
lock_fds: list[int] = []
device_control_env = current_omni_platform.device_control_env_var
try:
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(metadata.stage_id, metadata.runtime_cfg)
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
)
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
if omni_conn_cfg:
omni_kv = engine_args_dict.get("omni_kv_config") or {}
if not isinstance(omni_kv, dict):
omni_kv = dict(omni_kv)
omni_kv["connector_config"] = omni_conn_cfg
omni_kv["omni_from_stage"] = omni_from
omni_kv["omni_to_stage"] = omni_to
omni_kv.setdefault("stage_id", metadata.stage_id)
engine_args_dict["omni_kv_config"] = omni_kv
vllm_config, executor_class = build_vllm_config(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
engine_args_dict=engine_args_dict,
)
lock_fds = acquire_device_locks(
metadata.stage_id,
engine_args_dict,
stage_init_timeout,
)
addresses, proc, handshake_address = spawn_stage_core(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
)
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
proc=proc,
addresses=addresses,
)
logger.info("[AsyncOmniEngine] Stage %s engine launch started", metadata.stage_id)
# Keep the stage-specific device visibility until vLLM
# finishes starting all child processes.
complete_stage_handshake(proc, handshake_address, addresses, vllm_config)
logger.info("[AsyncOmniEngine] Stage %s engine startup completed", metadata.stage_id)
finally:
if previous_visible_devices is None:
current_omni_platform.unset_device_control_env_var()
else:
current_omni_platform.set_device_control_env_var(previous_visible_devices)
proc = None
handshake_address = None
with ExitStack() as launch_stack:
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(metadata.stage_id, metadata.runtime_cfg)
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
)
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
if omni_conn_cfg:
omni_kv = engine_args_dict.get("omni_kv_config") or {}
if not isinstance(omni_kv, dict):
omni_kv = dict(omni_kv)
omni_kv["connector_config"] = omni_conn_cfg
omni_kv["omni_from_stage"] = omni_from
omni_kv["omni_to_stage"] = omni_to
omni_kv.setdefault("stage_id", metadata.stage_id)
engine_args_dict["omni_kv_config"] = omni_kv
vllm_config, executor_class = build_vllm_config(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
engine_args_dict=engine_args_dict,
)
lock_fds = acquire_device_locks(
metadata.stage_id,
engine_args_dict,
stage_init_timeout,
)
if self.single_stage_mode and self._omni_master_server is not None:
engine_manager, coordinator, addresses = launch_stack.enter_context(
launch_omni_core_engines(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
omni_master_server=self._omni_master_server,
stage_id=metadata.stage_id,
stage_config=stage_cfg,
)
)
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
addresses=addresses,
engine_manager=engine_manager,
coordinator=coordinator,
)
else:
addresses, proc, handshake_address = spawn_stage_core(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
)
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
addresses=addresses,
proc=proc,
)
logger.info("[AsyncOmniEngine] Stage %s engine launch started", metadata.stage_id)
# Keep the stage-specific device visibility until vLLM
# finishes starting all child processes.
if self.single_stage_mode and self._omni_master_server is not None:
launch_stack.close()
else:
assert proc is not None
assert handshake_address is not None
complete_stage_handshake(proc, handshake_address, addresses, vllm_config)
logger.info("[AsyncOmniEngine] Stage %s engine startup completed", metadata.stage_id)
finally:
if previous_visible_devices is None:
current_omni_platform.unset_device_control_env_var()
else:
current_omni_platform.set_device_control_env_var(previous_visible_devices)
assert started_stage is not None
return started_stage
@@ -416,13 +448,138 @@ class AsyncOmniEngine:
if lock_fds:
release_device_locks(lock_fds)
def _create_remote_llm_stage(
self,
stage_cfg: Any,
metadata: Any,
stage_connector_spec: dict[str, Any],
stage_init_timeout: int,
omni_master_server: OmniMasterServer,
) -> StartedLlmStage:
"""Attach to a remote engine core and wait for its startup handshake."""
started_stage: StartedLlmStage | None = None
try:
raw_stage_cfg = omni_master_server.get_stage_config(
metadata.stage_id,
timeout_s=stage_init_timeout,
)
if raw_stage_cfg is None:
raise ValueError(f"Remote stage {metadata.stage_id} registered without stage config")
stage_cfg = OmegaConf.create(raw_stage_cfg)
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
)
vllm_config, executor_class = build_vllm_config(
stage_cfg,
self.model,
stage_connector_spec=stage_connector_spec,
engine_args_dict=engine_args_dict,
)
vllm_config.parallel_config.data_parallel_size_local = 0
launch_cm = connect_remote_engine_cores(
vllm_config=vllm_config,
omni_master_server=omni_master_server,
stage_id=metadata.stage_id,
)
logger.info("[AsyncOmniEngine] Stage %s remote engine handshake started", metadata.stage_id)
with launch_cm as (engine_manager, coordinator, addresses):
started_stage = StartedLlmStage(
stage_id=metadata.stage_id,
metadata=metadata,
vllm_config=vllm_config,
executor_class=executor_class,
engine_manager=engine_manager,
coordinator=coordinator,
addresses=addresses,
)
logger.info("[AsyncOmniEngine] Stage %s remote engine startup completed", metadata.stage_id)
assert started_stage is not None
return started_stage
except Exception:
if started_stage is not None:
close_started_llm_stage(started_stage)
raise
def _launch_diffusion_stage(
self,
stage_cfg: Any,
metadata: Any,
omni_master_server: OmniMasterServer,
) -> StageDiffusionClient:
"""Launch a local diffusion stage on OmniMasterServer-allocated sockets."""
proc = None
try:
od_config = build_diffusion_config(self.model, stage_cfg, metadata)
handshake_address, request_address, response_address = register_stage_with_omni_master(
omni_master_address=omni_master_server.address,
omni_master_port=omni_master_server.port,
omni_stage_id=metadata.stage_id,
omni_stage_config=stage_cfg,
return_addresses=True,
)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion registration completed",
metadata.stage_id,
)
proc, _, _, _ = spawn_diffusion_proc(
self.model,
od_config,
handshake_address=handshake_address,
request_address=request_address,
response_address=response_address,
)
complete_diffusion_handshake(proc, handshake_address)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion startup completed",
metadata.stage_id,
)
return StageDiffusionClient.from_addresses(
metadata,
request_address=request_address,
response_address=response_address,
proc=proc,
batch_size=self.diffusion_batch_size,
)
except Exception:
if proc is not None:
terminate_alive_proc(proc)
raise
def _create_remote_diffusion_stage(
self,
metadata: Any,
stage_init_timeout: int,
omni_master_server: OmniMasterServer,
) -> StageDiffusionClient:
"""Attach to a remote diffusion stage registered with OmniMasterServer."""
remote_stage_cfg = OmegaConf.create(
omni_master_server.get_stage_config(
metadata.stage_id,
timeout_s=stage_init_timeout,
)
)
remote_metadata = extract_stage_metadata(remote_stage_cfg)
addresses = omni_master_server.get_zmq_addresses(metadata.stage_id)
logger.info(
"[AsyncOmniEngine] Stage %s remote diffusion startup completed",
metadata.stage_id,
)
return StageDiffusionClient.from_addresses(
remote_metadata,
request_address=addresses.inputs[0],
response_address=addresses.outputs[0],
batch_size=self.diffusion_batch_size,
)
def _attach_llm_stage(
self,
started: StartedLlmStage,
) -> tuple[Any, Any, Any, InputProcessor | None]:
"""Attach a READY LLM stage to the orchestrator event loop."""
client_addresses = {
client_addresses: dict[str, str] = {
"input_address": started.addresses.inputs[0],
"output_address": started.addresses.outputs[0],
}
@@ -430,14 +587,18 @@ class AsyncOmniEngine:
client_addresses["stats_update_address"] = started.addresses.frontend_stats_publish_address
try:
stage_client = StageEngineCoreClient(
stage_client = StageEngineCoreClientBase.make_async_mp_client(
vllm_config=started.vllm_config,
executor_class=started.executor_class,
metadata=started.metadata,
client_addresses=client_addresses,
proc=started.proc,
engine_manager=started.engine_manager,
coordinator=started.coordinator,
)
started.proc = None
started.engine_manager = None
started.coordinator = None
except Exception:
close_started_llm_stage(started)
raise
@@ -493,7 +654,7 @@ class AsyncOmniEngine:
output_processors: list[Any | None] = [None] * num_stages
stage_vllm_configs: list[Any | None] = [None] * num_stages
input_processor: InputProcessor | None = None
llm_stage_ids: list[int] = []
llm_stage_positions: list[int] = []
llm_launch_futures: dict[int, concurrent.futures.Future[StartedLlmStage]] = {}
started_llm_stages: dict[int, StartedLlmStage] = {}
llm_stage_launch_lock = threading.Lock()
@@ -507,45 +668,102 @@ class AsyncOmniEngine:
prepare_engine_environment()
omni_transfer_config = load_omni_transfer_config_for_model(self.model, self.config_path)
# ------------------------------------------------------------------ #
# Single-stage mode: start OmniMasterServer before launching stages. #
# ------------------------------------------------------------------ #
if self.single_stage_mode:
if not self._omni_master_address or not self._omni_master_port:
raise ValueError(
"AsyncOmniEngine single_stage_mode requires both "
"omni_master_address and omni_master_port to be set."
)
# Collect all configured stage IDs for pre-allocation.
all_stage_ids: list[int] = []
seen_stage_ids: set[int] = set()
for i, sc in enumerate(self.stage_configs):
stage_id = int(getattr(sc, "stage_id", i))
if stage_id in seen_stage_ids:
raise ValueError(
f"Duplicate stage_id {stage_id!r} detected among configured stages; stage_ids must be unique."
)
seen_stage_ids.add(stage_id)
all_stage_ids.append(stage_id)
self._omni_master_server = OmniMasterServer(
master_address=self._omni_master_address,
master_port=self._omni_master_port,
stage_ids=all_stage_ids,
)
self._omni_master_server.start()
logger.info(
"[AsyncOmniEngine] OmniMasterServer started for stages %s",
all_stage_ids,
)
try:
with concurrent.futures.ThreadPoolExecutor(
max_workers=max(1, llm_stage_count),
thread_name_prefix="llm-stage-launch",
) as launch_executor:
for stage_id, stage_cfg in enumerate(self.stage_configs):
logger.info("[AsyncOmniEngine] Initializing stage %s", stage_id)
for stage_idx, stage_cfg in enumerate(self.stage_configs):
metadata = extract_stage_metadata(stage_cfg)
configured_stage_id = metadata.stage_id
logger.info("[AsyncOmniEngine] Initializing stage %s", configured_stage_id)
if metadata.prompt_expand_func is not None:
prompt_expand_func = metadata.prompt_expand_func
if self.single_stage_mode:
metadata.runtime_cfg = None
stage_connector_spec = get_stage_connector_spec(
omni_transfer_config=omni_transfer_config,
stage_id=stage_id,
stage_id=configured_stage_id,
async_chunk=async_chunk,
)
omni_kv_connector = resolve_omni_kv_config_for_stage(omni_transfer_config, stage_id)
omni_kv_connector = resolve_omni_kv_config_for_stage(omni_transfer_config, configured_stage_id)
if metadata.stage_type == "diffusion":
is_remote_diffusion_stage = (
self.single_stage_mode
and self._single_stage_id_filter is not None
and configured_stage_id != self._single_stage_id_filter
)
if is_remote_diffusion_stage:
assert self._omni_master_server is not None
stage_clients[stage_idx] = self._create_remote_diffusion_stage(
metadata,
stage_init_timeout,
self._omni_master_server,
)
continue
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(stage_id, metadata.runtime_cfg)
setup_stage_devices(configured_stage_id, metadata.runtime_cfg)
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
if omni_conn_cfg:
from vllm_omni.entrypoints.utils import inject_omni_kv_config
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
_inject_kv_stage_info(stage_cfg, stage_id)
stage_clients[stage_id] = initialize_diffusion_stage(
self.model,
stage_cfg,
metadata,
batch_size=self.diffusion_batch_size,
)
inject_kv_stage_info(stage_cfg, configured_stage_id)
if self.single_stage_mode:
assert self._omni_master_server is not None
stage_clients[stage_idx] = self._launch_diffusion_stage(
stage_cfg,
metadata,
self._omni_master_server,
)
else:
stage_clients[stage_idx] = initialize_diffusion_stage(
self.model,
stage_cfg,
metadata,
batch_size=self.diffusion_batch_size,
)
logger.info(
"[AsyncOmniEngine] Stage %s initialized (diffusion, batch_size=%d)",
stage_id,
configured_stage_id,
self.diffusion_batch_size,
)
finally:
@@ -555,30 +773,58 @@ class AsyncOmniEngine:
current_omni_platform.set_device_control_env_var(previous_visible_devices)
continue
llm_stage_ids.append(stage_id)
llm_launch_futures[stage_id] = launch_executor.submit(
self._launch_llm_stage,
stage_cfg,
metadata,
stage_connector_spec,
stage_init_timeout,
llm_stage_launch_lock,
omni_kv_connector,
)
llm_stage_positions.append(stage_idx)
# In single-stage mode, stages that don't match the local
# stage_id filter are skipped.
if (
self.single_stage_mode
and self._single_stage_id_filter is not None
and configured_stage_id != self._single_stage_id_filter
):
assert self._omni_master_server is not None
llm_launch_futures[stage_idx] = launch_executor.submit(
self._create_remote_llm_stage,
stage_cfg,
metadata,
stage_connector_spec,
stage_init_timeout,
self._omni_master_server,
)
else:
llm_launch_futures[stage_idx] = launch_executor.submit(
self._launch_llm_stage,
stage_cfg,
metadata,
stage_connector_spec,
stage_init_timeout,
llm_stage_launch_lock,
omni_kv_connector,
)
concurrent.futures.wait(list(llm_launch_futures.values()))
for stage_id in llm_stage_ids:
started_llm_stages[stage_id] = llm_launch_futures[stage_id].result()
for stage_idx in llm_stage_positions:
started_llm_stages[stage_idx] = llm_launch_futures[stage_idx].result()
for stage_id in llm_stage_ids:
started = started_llm_stages[stage_id]
stage_client, output_processor, vllm_config, stage0_input_processor = self._attach_llm_stage(started)
stage_clients[stage_id] = stage_client
output_processors[stage_id] = output_processor
stage_vllm_configs[stage_id] = vllm_config
if stage0_input_processor is not None:
input_processor = stage0_input_processor
attach_futures: dict[concurrent.futures.Future[tuple[Any, Any, Any, InputProcessor | None]], int] = {}
with concurrent.futures.ThreadPoolExecutor(
max_workers=max(1, len(llm_stage_positions)),
thread_name_prefix="llm-stage-attach",
) as attach_executor:
for stage_idx in llm_stage_positions:
attach_futures[attach_executor.submit(self._attach_llm_stage, started_llm_stages[stage_idx])] = (
stage_idx
)
for future in concurrent.futures.as_completed(attach_futures):
stage_idx = attach_futures[future]
stage_client, output_processor, vllm_config, stage0_input_processor = future.result()
stage_clients[stage_idx] = stage_client
output_processors[stage_idx] = output_processor
stage_vllm_configs[stage_idx] = vllm_config
if stage0_input_processor is not None:
input_processor = stage0_input_processor
initialized_stage_clients, default_sampling_params_list, stage_metadata = finalize_initialized_stages(
stage_clients,
@@ -595,8 +841,13 @@ class AsyncOmniEngine:
)
cleanup_failed_stage_initialization(
stage_clients,
[started_llm_stages[stage_id] for stage_id in llm_stage_ids if stage_id in started_llm_stages],
[started_llm_stages[stage_idx] for stage_idx in llm_stage_positions if stage_idx in started_llm_stages],
)
if self._omni_master_server is not None:
try:
self._omni_master_server.stop()
except Exception:
logger.exception("[AsyncOmniEngine] Failed to stop OmniMasterServer during stage-init cleanup")
raise
self.stage_clients = initialized_stage_clients
@@ -1310,3 +1561,10 @@ class AsyncOmniEngine:
q.close()
except Exception:
pass
if self._omni_master_server is not None:
try:
self._omni_master_server.stop()
except Exception:
logger.exception("[AsyncOmniEngine] Failed to stop OmniMasterServer during shutdown")
self._omni_master_server = None