[Entrypoint][Refactor]Stage CLI Refactor (#2020)
Signed-off-by: wuhang <wuhang6@huawei.com> Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
f3f2dc590c
commit
cb91cbe61e
19 files changed
+3773
-273
No files matched your search
@@ -18,12 +18,10 @@ import time
|
||||
import uuid
|
||||
import weakref
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import asdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
|
||||
import janus
|
||||
import torch
|
||||
from omegaconf import OmegaConf
|
||||
@@ -34,26 +32,36 @@ from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.stage_diffusion_client import StageDiffusionClient
|
||||
from vllm_omni.diffusion.stage_diffusion_proc import (
|
||||
complete_diffusion_handshake,
|
||||
spawn_diffusion_proc,
|
||||
)
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import (
|
||||
resolve_omni_kv_config_for_stage,
|
||||
)
|
||||
from vllm_omni.engine import (
|
||||
OmniEngineCoreRequest,
|
||||
)
|
||||
from vllm_omni.engine import OmniEngineCoreRequest
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
from vllm_omni.engine.output_processor import MultimodalOutputProcessor
|
||||
from vllm_omni.engine.serialization import (
|
||||
deserialize_additional_information,
|
||||
serialize_additional_information,
|
||||
)
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
|
||||
from vllm_omni.engine.stage_engine_core_proc import (
|
||||
complete_stage_handshake,
|
||||
spawn_stage_core,
|
||||
)
|
||||
from vllm_omni.engine.stage_engine_startup import (
|
||||
OmniMasterServer,
|
||||
connect_remote_engine_cores,
|
||||
launch_omni_core_engines,
|
||||
register_stage_with_omni_master,
|
||||
)
|
||||
from vllm_omni.engine.stage_init_utils import (
|
||||
StartedLlmStage,
|
||||
acquire_device_locks,
|
||||
build_diffusion_config,
|
||||
build_engine_args_dict,
|
||||
build_vllm_config,
|
||||
cleanup_failed_stage_initialization,
|
||||
@@ -62,17 +70,20 @@ from vllm_omni.engine.stage_init_utils import (
|
||||
finalize_initialized_stages,
|
||||
get_stage_connector_spec,
|
||||
initialize_diffusion_stage,
|
||||
inject_kv_stage_info,
|
||||
load_omni_transfer_config_for_model,
|
||||
prepare_engine_environment,
|
||||
release_device_locks,
|
||||
setup_stage_devices,
|
||||
terminate_alive_proc,
|
||||
)
|
||||
from vllm_omni.entrypoints.utils import (
|
||||
load_and_resolve_stage_configs,
|
||||
)
|
||||
from vllm_omni.entrypoints.utils import load_and_resolve_stage_configs
|
||||
from vllm_omni.inputs.preprocess import OmniInputPreprocessor
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -86,39 +97,6 @@ def _patch_generation_config_if_needed(model_config: Any) -> None:
|
||||
model_config.try_get_generation_config = lambda: {}
|
||||
|
||||
|
||||
def _inject_kv_stage_info(stage_cfg: Any, stage_id: int) -> None:
|
||||
"""Inject stage_id and engine_input_source into omni_kv_config.
|
||||
|
||||
OmniKVTransferManager needs stage_id to compute recv_stages for the
|
||||
receiving side. In the old Omni architecture, OmniDiffusion.__init__
|
||||
performed this injection; replicate it here for AsyncOmniEngine.
|
||||
"""
|
||||
try:
|
||||
engine_args = stage_cfg.engine_args
|
||||
if hasattr(engine_args, "get"):
|
||||
omni_kv = engine_args.get("omni_kv_config", None)
|
||||
else:
|
||||
omni_kv = getattr(engine_args, "omni_kv_config", None)
|
||||
|
||||
if omni_kv is None:
|
||||
return
|
||||
|
||||
if hasattr(omni_kv, "setdefault"):
|
||||
omni_kv.setdefault("stage_id", stage_id)
|
||||
elif hasattr(omni_kv, "__setitem__"):
|
||||
if "stage_id" not in omni_kv:
|
||||
omni_kv["stage_id"] = stage_id
|
||||
|
||||
engine_input_source = getattr(stage_cfg, "engine_input_source", None)
|
||||
if engine_input_source is not None:
|
||||
if hasattr(omni_kv, "setdefault"):
|
||||
omni_kv.setdefault("engine_input_source", list(engine_input_source))
|
||||
elif hasattr(omni_kv, "__setitem__") and "engine_input_source" not in omni_kv:
|
||||
omni_kv["engine_input_source"] = list(engine_input_source)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to inject stage info into omni_kv_config: %s", e)
|
||||
|
||||
|
||||
def _inject_global_id(target: Any, request_id: str) -> None:
|
||||
"""Inject global_request_id into a prompt dict's additional_information."""
|
||||
if isinstance(target, dict):
|
||||
@@ -255,6 +233,7 @@ class AsyncOmniEngine:
|
||||
stage_init_timeout: int = 300,
|
||||
init_timeout: int = 600,
|
||||
diffusion_batch_size: int = 1,
|
||||
single_stage_mode: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self.model = model
|
||||
@@ -274,6 +253,31 @@ class AsyncOmniEngine:
|
||||
ea_dict.pop("model", None)
|
||||
kwargs = {**ea_dict, **kwargs}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Single-stage mode detection #
|
||||
# ------------------------------------------------------------------ #
|
||||
# Single-stage mode is enabled when the caller explicitly passes #
|
||||
# single_stage_mode=True, or when a stage_id is provided in the args. #
|
||||
_stage_id_kwarg = kwargs.get("stage_id")
|
||||
if isinstance(_stage_id_kwarg, int) and not single_stage_mode:
|
||||
single_stage_mode = True
|
||||
|
||||
self.single_stage_mode: bool = single_stage_mode
|
||||
self._single_stage_id_filter: int | None = (
|
||||
int(_stage_id_kwarg) if single_stage_mode and isinstance(_stage_id_kwarg, int) else None
|
||||
)
|
||||
self._omni_master_address: str | None = kwargs.get("omni_master_address")
|
||||
self._omni_master_port: int | None = kwargs.get("omni_master_port")
|
||||
self._omni_master_server: OmniMasterServer | None = None
|
||||
|
||||
if single_stage_mode:
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] Single-stage mode enabled (stage_id_filter=%s, master=%s:%s)",
|
||||
self._single_stage_id_filter,
|
||||
self._omni_master_address,
|
||||
self._omni_master_port,
|
||||
)
|
||||
|
||||
self.config_path, self.stage_configs = self._resolve_stage_configs(model, kwargs)
|
||||
|
||||
self.num_stages = len(self.stage_configs)
|
||||
@@ -350,61 +354,89 @@ class AsyncOmniEngine:
|
||||
started_stage: StartedLlmStage | None = None
|
||||
lock_fds: list[int] = []
|
||||
device_control_env = current_omni_platform.device_control_env_var
|
||||
|
||||
try:
|
||||
with llm_stage_launch_lock:
|
||||
previous_visible_devices = os.environ.get(device_control_env)
|
||||
try:
|
||||
setup_stage_devices(metadata.stage_id, metadata.runtime_cfg)
|
||||
engine_args_dict = build_engine_args_dict(
|
||||
stage_cfg,
|
||||
self.model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
)
|
||||
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
|
||||
if omni_conn_cfg:
|
||||
omni_kv = engine_args_dict.get("omni_kv_config") or {}
|
||||
if not isinstance(omni_kv, dict):
|
||||
omni_kv = dict(omni_kv)
|
||||
omni_kv["connector_config"] = omni_conn_cfg
|
||||
omni_kv["omni_from_stage"] = omni_from
|
||||
omni_kv["omni_to_stage"] = omni_to
|
||||
omni_kv.setdefault("stage_id", metadata.stage_id)
|
||||
engine_args_dict["omni_kv_config"] = omni_kv
|
||||
vllm_config, executor_class = build_vllm_config(
|
||||
stage_cfg,
|
||||
self.model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
engine_args_dict=engine_args_dict,
|
||||
)
|
||||
lock_fds = acquire_device_locks(
|
||||
metadata.stage_id,
|
||||
engine_args_dict,
|
||||
stage_init_timeout,
|
||||
)
|
||||
addresses, proc, handshake_address = spawn_stage_core(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
)
|
||||
started_stage = StartedLlmStage(
|
||||
stage_id=metadata.stage_id,
|
||||
metadata=metadata,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
proc=proc,
|
||||
addresses=addresses,
|
||||
)
|
||||
logger.info("[AsyncOmniEngine] Stage %s engine launch started", metadata.stage_id)
|
||||
# Keep the stage-specific device visibility until vLLM
|
||||
# finishes starting all child processes.
|
||||
complete_stage_handshake(proc, handshake_address, addresses, vllm_config)
|
||||
logger.info("[AsyncOmniEngine] Stage %s engine startup completed", metadata.stage_id)
|
||||
finally:
|
||||
if previous_visible_devices is None:
|
||||
current_omni_platform.unset_device_control_env_var()
|
||||
else:
|
||||
current_omni_platform.set_device_control_env_var(previous_visible_devices)
|
||||
proc = None
|
||||
handshake_address = None
|
||||
with ExitStack() as launch_stack:
|
||||
with llm_stage_launch_lock:
|
||||
previous_visible_devices = os.environ.get(device_control_env)
|
||||
try:
|
||||
setup_stage_devices(metadata.stage_id, metadata.runtime_cfg)
|
||||
engine_args_dict = build_engine_args_dict(
|
||||
stage_cfg,
|
||||
self.model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
)
|
||||
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
|
||||
if omni_conn_cfg:
|
||||
omni_kv = engine_args_dict.get("omni_kv_config") or {}
|
||||
if not isinstance(omni_kv, dict):
|
||||
omni_kv = dict(omni_kv)
|
||||
omni_kv["connector_config"] = omni_conn_cfg
|
||||
omni_kv["omni_from_stage"] = omni_from
|
||||
omni_kv["omni_to_stage"] = omni_to
|
||||
omni_kv.setdefault("stage_id", metadata.stage_id)
|
||||
engine_args_dict["omni_kv_config"] = omni_kv
|
||||
vllm_config, executor_class = build_vllm_config(
|
||||
stage_cfg,
|
||||
self.model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
engine_args_dict=engine_args_dict,
|
||||
)
|
||||
lock_fds = acquire_device_locks(
|
||||
metadata.stage_id,
|
||||
engine_args_dict,
|
||||
stage_init_timeout,
|
||||
)
|
||||
if self.single_stage_mode and self._omni_master_server is not None:
|
||||
engine_manager, coordinator, addresses = launch_stack.enter_context(
|
||||
launch_omni_core_engines(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
omni_master_server=self._omni_master_server,
|
||||
stage_id=metadata.stage_id,
|
||||
stage_config=stage_cfg,
|
||||
)
|
||||
)
|
||||
started_stage = StartedLlmStage(
|
||||
stage_id=metadata.stage_id,
|
||||
metadata=metadata,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
addresses=addresses,
|
||||
engine_manager=engine_manager,
|
||||
coordinator=coordinator,
|
||||
)
|
||||
else:
|
||||
addresses, proc, handshake_address = spawn_stage_core(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
)
|
||||
started_stage = StartedLlmStage(
|
||||
stage_id=metadata.stage_id,
|
||||
metadata=metadata,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
addresses=addresses,
|
||||
proc=proc,
|
||||
)
|
||||
logger.info("[AsyncOmniEngine] Stage %s engine launch started", metadata.stage_id)
|
||||
# Keep the stage-specific device visibility until vLLM
|
||||
# finishes starting all child processes.
|
||||
if self.single_stage_mode and self._omni_master_server is not None:
|
||||
launch_stack.close()
|
||||
else:
|
||||
assert proc is not None
|
||||
assert handshake_address is not None
|
||||
complete_stage_handshake(proc, handshake_address, addresses, vllm_config)
|
||||
logger.info("[AsyncOmniEngine] Stage %s engine startup completed", metadata.stage_id)
|
||||
finally:
|
||||
if previous_visible_devices is None:
|
||||
current_omni_platform.unset_device_control_env_var()
|
||||
else:
|
||||
current_omni_platform.set_device_control_env_var(previous_visible_devices)
|
||||
|
||||
assert started_stage is not None
|
||||
return started_stage
|
||||
@@ -416,13 +448,138 @@ class AsyncOmniEngine:
|
||||
if lock_fds:
|
||||
release_device_locks(lock_fds)
|
||||
|
||||
def _create_remote_llm_stage(
|
||||
self,
|
||||
stage_cfg: Any,
|
||||
metadata: Any,
|
||||
stage_connector_spec: dict[str, Any],
|
||||
stage_init_timeout: int,
|
||||
omni_master_server: OmniMasterServer,
|
||||
) -> StartedLlmStage:
|
||||
"""Attach to a remote engine core and wait for its startup handshake."""
|
||||
started_stage: StartedLlmStage | None = None
|
||||
try:
|
||||
raw_stage_cfg = omni_master_server.get_stage_config(
|
||||
metadata.stage_id,
|
||||
timeout_s=stage_init_timeout,
|
||||
)
|
||||
if raw_stage_cfg is None:
|
||||
raise ValueError(f"Remote stage {metadata.stage_id} registered without stage config")
|
||||
stage_cfg = OmegaConf.create(raw_stage_cfg)
|
||||
engine_args_dict = build_engine_args_dict(
|
||||
stage_cfg,
|
||||
self.model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
)
|
||||
vllm_config, executor_class = build_vllm_config(
|
||||
stage_cfg,
|
||||
self.model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
engine_args_dict=engine_args_dict,
|
||||
)
|
||||
vllm_config.parallel_config.data_parallel_size_local = 0
|
||||
launch_cm = connect_remote_engine_cores(
|
||||
vllm_config=vllm_config,
|
||||
omni_master_server=omni_master_server,
|
||||
stage_id=metadata.stage_id,
|
||||
)
|
||||
logger.info("[AsyncOmniEngine] Stage %s remote engine handshake started", metadata.stage_id)
|
||||
with launch_cm as (engine_manager, coordinator, addresses):
|
||||
started_stage = StartedLlmStage(
|
||||
stage_id=metadata.stage_id,
|
||||
metadata=metadata,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
engine_manager=engine_manager,
|
||||
coordinator=coordinator,
|
||||
addresses=addresses,
|
||||
)
|
||||
logger.info("[AsyncOmniEngine] Stage %s remote engine startup completed", metadata.stage_id)
|
||||
assert started_stage is not None
|
||||
return started_stage
|
||||
except Exception:
|
||||
if started_stage is not None:
|
||||
close_started_llm_stage(started_stage)
|
||||
raise
|
||||
|
||||
def _launch_diffusion_stage(
|
||||
self,
|
||||
stage_cfg: Any,
|
||||
metadata: Any,
|
||||
omni_master_server: OmniMasterServer,
|
||||
) -> StageDiffusionClient:
|
||||
"""Launch a local diffusion stage on OmniMasterServer-allocated sockets."""
|
||||
proc = None
|
||||
try:
|
||||
od_config = build_diffusion_config(self.model, stage_cfg, metadata)
|
||||
handshake_address, request_address, response_address = register_stage_with_omni_master(
|
||||
omni_master_address=omni_master_server.address,
|
||||
omni_master_port=omni_master_server.port,
|
||||
omni_stage_id=metadata.stage_id,
|
||||
omni_stage_config=stage_cfg,
|
||||
return_addresses=True,
|
||||
)
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] Stage %s diffusion registration completed",
|
||||
metadata.stage_id,
|
||||
)
|
||||
proc, _, _, _ = spawn_diffusion_proc(
|
||||
self.model,
|
||||
od_config,
|
||||
handshake_address=handshake_address,
|
||||
request_address=request_address,
|
||||
response_address=response_address,
|
||||
)
|
||||
complete_diffusion_handshake(proc, handshake_address)
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] Stage %s diffusion startup completed",
|
||||
metadata.stage_id,
|
||||
)
|
||||
return StageDiffusionClient.from_addresses(
|
||||
metadata,
|
||||
request_address=request_address,
|
||||
response_address=response_address,
|
||||
proc=proc,
|
||||
batch_size=self.diffusion_batch_size,
|
||||
)
|
||||
except Exception:
|
||||
if proc is not None:
|
||||
terminate_alive_proc(proc)
|
||||
raise
|
||||
|
||||
def _create_remote_diffusion_stage(
|
||||
self,
|
||||
metadata: Any,
|
||||
stage_init_timeout: int,
|
||||
omni_master_server: OmniMasterServer,
|
||||
) -> StageDiffusionClient:
|
||||
"""Attach to a remote diffusion stage registered with OmniMasterServer."""
|
||||
remote_stage_cfg = OmegaConf.create(
|
||||
omni_master_server.get_stage_config(
|
||||
metadata.stage_id,
|
||||
timeout_s=stage_init_timeout,
|
||||
)
|
||||
)
|
||||
remote_metadata = extract_stage_metadata(remote_stage_cfg)
|
||||
addresses = omni_master_server.get_zmq_addresses(metadata.stage_id)
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] Stage %s remote diffusion startup completed",
|
||||
metadata.stage_id,
|
||||
)
|
||||
return StageDiffusionClient.from_addresses(
|
||||
remote_metadata,
|
||||
request_address=addresses.inputs[0],
|
||||
response_address=addresses.outputs[0],
|
||||
batch_size=self.diffusion_batch_size,
|
||||
)
|
||||
|
||||
def _attach_llm_stage(
|
||||
self,
|
||||
started: StartedLlmStage,
|
||||
) -> tuple[Any, Any, Any, InputProcessor | None]:
|
||||
"""Attach a READY LLM stage to the orchestrator event loop."""
|
||||
|
||||
client_addresses = {
|
||||
client_addresses: dict[str, str] = {
|
||||
"input_address": started.addresses.inputs[0],
|
||||
"output_address": started.addresses.outputs[0],
|
||||
}
|
||||
@@ -430,14 +587,18 @@ class AsyncOmniEngine:
|
||||
client_addresses["stats_update_address"] = started.addresses.frontend_stats_publish_address
|
||||
|
||||
try:
|
||||
stage_client = StageEngineCoreClient(
|
||||
stage_client = StageEngineCoreClientBase.make_async_mp_client(
|
||||
vllm_config=started.vllm_config,
|
||||
executor_class=started.executor_class,
|
||||
metadata=started.metadata,
|
||||
client_addresses=client_addresses,
|
||||
proc=started.proc,
|
||||
engine_manager=started.engine_manager,
|
||||
coordinator=started.coordinator,
|
||||
)
|
||||
started.proc = None
|
||||
started.engine_manager = None
|
||||
started.coordinator = None
|
||||
except Exception:
|
||||
close_started_llm_stage(started)
|
||||
raise
|
||||
@@ -493,7 +654,7 @@ class AsyncOmniEngine:
|
||||
output_processors: list[Any | None] = [None] * num_stages
|
||||
stage_vllm_configs: list[Any | None] = [None] * num_stages
|
||||
input_processor: InputProcessor | None = None
|
||||
llm_stage_ids: list[int] = []
|
||||
llm_stage_positions: list[int] = []
|
||||
llm_launch_futures: dict[int, concurrent.futures.Future[StartedLlmStage]] = {}
|
||||
started_llm_stages: dict[int, StartedLlmStage] = {}
|
||||
llm_stage_launch_lock = threading.Lock()
|
||||
@@ -507,45 +668,102 @@ class AsyncOmniEngine:
|
||||
prepare_engine_environment()
|
||||
omni_transfer_config = load_omni_transfer_config_for_model(self.model, self.config_path)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Single-stage mode: start OmniMasterServer before launching stages. #
|
||||
# ------------------------------------------------------------------ #
|
||||
if self.single_stage_mode:
|
||||
if not self._omni_master_address or not self._omni_master_port:
|
||||
raise ValueError(
|
||||
"AsyncOmniEngine single_stage_mode requires both "
|
||||
"omni_master_address and omni_master_port to be set."
|
||||
)
|
||||
# Collect all configured stage IDs for pre-allocation.
|
||||
all_stage_ids: list[int] = []
|
||||
seen_stage_ids: set[int] = set()
|
||||
for i, sc in enumerate(self.stage_configs):
|
||||
stage_id = int(getattr(sc, "stage_id", i))
|
||||
if stage_id in seen_stage_ids:
|
||||
raise ValueError(
|
||||
f"Duplicate stage_id {stage_id!r} detected among configured stages; stage_ids must be unique."
|
||||
)
|
||||
seen_stage_ids.add(stage_id)
|
||||
all_stage_ids.append(stage_id)
|
||||
self._omni_master_server = OmniMasterServer(
|
||||
master_address=self._omni_master_address,
|
||||
master_port=self._omni_master_port,
|
||||
stage_ids=all_stage_ids,
|
||||
)
|
||||
self._omni_master_server.start()
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] OmniMasterServer started for stages %s",
|
||||
all_stage_ids,
|
||||
)
|
||||
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=max(1, llm_stage_count),
|
||||
thread_name_prefix="llm-stage-launch",
|
||||
) as launch_executor:
|
||||
for stage_id, stage_cfg in enumerate(self.stage_configs):
|
||||
logger.info("[AsyncOmniEngine] Initializing stage %s", stage_id)
|
||||
for stage_idx, stage_cfg in enumerate(self.stage_configs):
|
||||
metadata = extract_stage_metadata(stage_cfg)
|
||||
configured_stage_id = metadata.stage_id
|
||||
logger.info("[AsyncOmniEngine] Initializing stage %s", configured_stage_id)
|
||||
if metadata.prompt_expand_func is not None:
|
||||
prompt_expand_func = metadata.prompt_expand_func
|
||||
|
||||
if self.single_stage_mode:
|
||||
metadata.runtime_cfg = None
|
||||
|
||||
stage_connector_spec = get_stage_connector_spec(
|
||||
omni_transfer_config=omni_transfer_config,
|
||||
stage_id=stage_id,
|
||||
stage_id=configured_stage_id,
|
||||
async_chunk=async_chunk,
|
||||
)
|
||||
|
||||
omni_kv_connector = resolve_omni_kv_config_for_stage(omni_transfer_config, stage_id)
|
||||
omni_kv_connector = resolve_omni_kv_config_for_stage(omni_transfer_config, configured_stage_id)
|
||||
|
||||
if metadata.stage_type == "diffusion":
|
||||
is_remote_diffusion_stage = (
|
||||
self.single_stage_mode
|
||||
and self._single_stage_id_filter is not None
|
||||
and configured_stage_id != self._single_stage_id_filter
|
||||
)
|
||||
if is_remote_diffusion_stage:
|
||||
assert self._omni_master_server is not None
|
||||
stage_clients[stage_idx] = self._create_remote_diffusion_stage(
|
||||
metadata,
|
||||
stage_init_timeout,
|
||||
self._omni_master_server,
|
||||
)
|
||||
continue
|
||||
|
||||
with llm_stage_launch_lock:
|
||||
previous_visible_devices = os.environ.get(device_control_env)
|
||||
try:
|
||||
setup_stage_devices(stage_id, metadata.runtime_cfg)
|
||||
setup_stage_devices(configured_stage_id, metadata.runtime_cfg)
|
||||
omni_conn_cfg, omni_from, omni_to = omni_kv_connector
|
||||
if omni_conn_cfg:
|
||||
from vllm_omni.entrypoints.utils import inject_omni_kv_config
|
||||
|
||||
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
|
||||
_inject_kv_stage_info(stage_cfg, stage_id)
|
||||
stage_clients[stage_id] = initialize_diffusion_stage(
|
||||
self.model,
|
||||
stage_cfg,
|
||||
metadata,
|
||||
batch_size=self.diffusion_batch_size,
|
||||
)
|
||||
inject_kv_stage_info(stage_cfg, configured_stage_id)
|
||||
if self.single_stage_mode:
|
||||
assert self._omni_master_server is not None
|
||||
stage_clients[stage_idx] = self._launch_diffusion_stage(
|
||||
stage_cfg,
|
||||
metadata,
|
||||
self._omni_master_server,
|
||||
)
|
||||
else:
|
||||
stage_clients[stage_idx] = initialize_diffusion_stage(
|
||||
self.model,
|
||||
stage_cfg,
|
||||
metadata,
|
||||
batch_size=self.diffusion_batch_size,
|
||||
)
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] Stage %s initialized (diffusion, batch_size=%d)",
|
||||
stage_id,
|
||||
configured_stage_id,
|
||||
self.diffusion_batch_size,
|
||||
)
|
||||
finally:
|
||||
@@ -555,30 +773,58 @@ class AsyncOmniEngine:
|
||||
current_omni_platform.set_device_control_env_var(previous_visible_devices)
|
||||
continue
|
||||
|
||||
llm_stage_ids.append(stage_id)
|
||||
llm_launch_futures[stage_id] = launch_executor.submit(
|
||||
self._launch_llm_stage,
|
||||
stage_cfg,
|
||||
metadata,
|
||||
stage_connector_spec,
|
||||
stage_init_timeout,
|
||||
llm_stage_launch_lock,
|
||||
omni_kv_connector,
|
||||
)
|
||||
llm_stage_positions.append(stage_idx)
|
||||
|
||||
# In single-stage mode, stages that don't match the local
|
||||
# stage_id filter are skipped.
|
||||
if (
|
||||
self.single_stage_mode
|
||||
and self._single_stage_id_filter is not None
|
||||
and configured_stage_id != self._single_stage_id_filter
|
||||
):
|
||||
assert self._omni_master_server is not None
|
||||
llm_launch_futures[stage_idx] = launch_executor.submit(
|
||||
self._create_remote_llm_stage,
|
||||
stage_cfg,
|
||||
metadata,
|
||||
stage_connector_spec,
|
||||
stage_init_timeout,
|
||||
self._omni_master_server,
|
||||
)
|
||||
else:
|
||||
llm_launch_futures[stage_idx] = launch_executor.submit(
|
||||
self._launch_llm_stage,
|
||||
stage_cfg,
|
||||
metadata,
|
||||
stage_connector_spec,
|
||||
stage_init_timeout,
|
||||
llm_stage_launch_lock,
|
||||
omni_kv_connector,
|
||||
)
|
||||
|
||||
concurrent.futures.wait(list(llm_launch_futures.values()))
|
||||
|
||||
for stage_id in llm_stage_ids:
|
||||
started_llm_stages[stage_id] = llm_launch_futures[stage_id].result()
|
||||
for stage_idx in llm_stage_positions:
|
||||
started_llm_stages[stage_idx] = llm_launch_futures[stage_idx].result()
|
||||
|
||||
for stage_id in llm_stage_ids:
|
||||
started = started_llm_stages[stage_id]
|
||||
stage_client, output_processor, vllm_config, stage0_input_processor = self._attach_llm_stage(started)
|
||||
stage_clients[stage_id] = stage_client
|
||||
output_processors[stage_id] = output_processor
|
||||
stage_vllm_configs[stage_id] = vllm_config
|
||||
if stage0_input_processor is not None:
|
||||
input_processor = stage0_input_processor
|
||||
attach_futures: dict[concurrent.futures.Future[tuple[Any, Any, Any, InputProcessor | None]], int] = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
max_workers=max(1, len(llm_stage_positions)),
|
||||
thread_name_prefix="llm-stage-attach",
|
||||
) as attach_executor:
|
||||
for stage_idx in llm_stage_positions:
|
||||
attach_futures[attach_executor.submit(self._attach_llm_stage, started_llm_stages[stage_idx])] = (
|
||||
stage_idx
|
||||
)
|
||||
|
||||
for future in concurrent.futures.as_completed(attach_futures):
|
||||
stage_idx = attach_futures[future]
|
||||
stage_client, output_processor, vllm_config, stage0_input_processor = future.result()
|
||||
stage_clients[stage_idx] = stage_client
|
||||
output_processors[stage_idx] = output_processor
|
||||
stage_vllm_configs[stage_idx] = vllm_config
|
||||
if stage0_input_processor is not None:
|
||||
input_processor = stage0_input_processor
|
||||
|
||||
initialized_stage_clients, default_sampling_params_list, stage_metadata = finalize_initialized_stages(
|
||||
stage_clients,
|
||||
@@ -595,8 +841,13 @@ class AsyncOmniEngine:
|
||||
)
|
||||
cleanup_failed_stage_initialization(
|
||||
stage_clients,
|
||||
[started_llm_stages[stage_id] for stage_id in llm_stage_ids if stage_id in started_llm_stages],
|
||||
[started_llm_stages[stage_idx] for stage_idx in llm_stage_positions if stage_idx in started_llm_stages],
|
||||
)
|
||||
if self._omni_master_server is not None:
|
||||
try:
|
||||
self._omni_master_server.stop()
|
||||
except Exception:
|
||||
logger.exception("[AsyncOmniEngine] Failed to stop OmniMasterServer during stage-init cleanup")
|
||||
raise
|
||||
|
||||
self.stage_clients = initialized_stage_clients
|
||||
@@ -1310,3 +1561,10 @@ class AsyncOmniEngine:
|
||||
q.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if self._omni_master_server is not None:
|
||||
try:
|
||||
self._omni_master_server.stop()
|
||||
except Exception:
|
||||
logger.exception("[AsyncOmniEngine] Failed to stop OmniMasterServer during shutdown")
|
||||
self._omni_master_server = None
|
||||
Reference in new issue
Block a user