diff --git a/tests/engine/test_async_omni_engine_stage_init.py b/tests/engine/test_async_omni_engine_stage_init.py index b423aa407..71952e6cf 100644 --- a/tests/engine/test_async_omni_engine_stage_init.py +++ b/tests/engine/test_async_omni_engine_stage_init.py @@ -3,6 +3,7 @@ import os import threading import time import types +from contextlib import contextmanager import pytest @@ -189,10 +190,12 @@ def test_initialize_diffusion_replica_restores_device_visibility_after_local_ini old_env = os.environ.get(env_var) os.environ[env_var] = "0,1" - def _fake_setup_stage_devices(_stage_id, _runtime_cfg): + @contextmanager + def _fake_stage_runtime_setup(_stage_id, _runtime_cfg): current_omni_platform.set_device_control_env_var("1") + yield - monkeypatch.setattr(engine_mod, "setup_stage_devices", _fake_setup_stage_devices) + monkeypatch.setattr(engine_mod, "stage_runtime_setup", _fake_stage_runtime_setup) monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None) monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", lambda *_, **__: types.SimpleNamespace()) @@ -221,7 +224,11 @@ def test_initialize_diffusion_replica_passes_stage_init_timeout_and_inline_flag( captured: dict[str, object] = {} - monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None) + @contextmanager + def _noop_stage_runtime_setup(*_): + yield + + monkeypatch.setattr(engine_mod, "stage_runtime_setup", _noop_stage_runtime_setup) monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None) def _capture_initialize_diffusion_stage( @@ -527,7 +534,11 @@ def test_initialize_llm_replica_passes_stage_init_timeout_to_complete_stage_hand prev_device_env = os.environ.get(device_env_var) os.environ[device_env_var] = "0" - monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None) + @contextmanager + def _noop_stage_runtime_setup(*_): + yield + + monkeypatch.setattr(engine_mod, "stage_runtime_setup", _noop_stage_runtime_setup) monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {}) monkeypatch.setattr(engine_mod, "acquire_device_locks", lambda *_: []) monkeypatch.setattr(engine_mod, "spawn_stage_core", lambda **_: (fake_addresses, fake_proc, "ipc://handshake")) diff --git a/tests/engine/test_single_stage_mode.py b/tests/engine/test_single_stage_mode.py index 52187c341..04ddbe22d 100644 --- a/tests/engine/test_single_stage_mode.py +++ b/tests/engine/test_single_stage_mode.py @@ -4,7 +4,7 @@ from __future__ import annotations import os import threading -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from types import SimpleNamespace from typing import Any @@ -835,7 +835,7 @@ class TestSingleStageReplicaInitialization: prev_device_env = os.environ.get(device_env_var) os.environ[device_env_var] = "0" - mocker.patch.object(engine_mod, "setup_stage_devices") + mocker.patch.object(engine_mod, "stage_runtime_setup", return_value=nullcontext()) mocker.patch.object(engine_mod, "build_engine_args_dict", return_value={}) mocker.patch.object(engine_mod, "acquire_device_locks", return_value=[]) mocker.patch.object(engine_mod, "release_device_locks") @@ -909,7 +909,7 @@ class TestSingleStageReplicaInitialization: prev_device_env = os.environ.get(device_env_var) os.environ[device_env_var] = "0" - mocker.patch.object(engine_mod, "setup_stage_devices") + mocker.patch.object(engine_mod, "stage_runtime_setup", return_value=nullcontext()) mocker.patch.object(engine_mod, "inject_kv_stage_info") mocker.patch.object(engine_mod, "build_diffusion_config", return_value="diffusion-config") mock_register = mocker.patch.object( @@ -988,7 +988,7 @@ class TestSingleStageReplicaInitialization: prev_device_env = os.environ.get(device_env_var) os.environ[device_env_var] = "0" - mocker.patch.object(engine_mod, "setup_stage_devices") + mocker.patch.object(engine_mod, "stage_runtime_setup", return_value=nullcontext()) mocker.patch.object(engine_mod, "inject_kv_stage_info") mocker.patch.object(engine_mod, "build_diffusion_config", return_value="diffusion-config") mocker.patch.object( diff --git a/vllm_omni/config/stage_config.py b/vllm_omni/config/stage_config.py index 7c17cf7ce..17cb5cf20 100644 --- a/vllm_omni/config/stage_config.py +++ b/vllm_omni/config/stage_config.py @@ -10,7 +10,7 @@ import warnings from dataclasses import asdict, dataclass, field, fields from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from vllm.logger import init_logger from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler @@ -408,6 +408,7 @@ class StageDeployConfig: stage_id: int devices: str | None = None num_replicas: int = 1 + env: dict[str, Any] | None = None # Inter-stage connector wiring and request defaults. output_connectors: dict[str, str] | None = None @@ -478,6 +479,7 @@ _STAGE_RESERVED_KEYS = frozenset( "stage_id", "devices", "num_replicas", + "env", "output_connectors", "input_connectors", "default_sampling_params", @@ -498,6 +500,7 @@ def _parse_stage_deploy(stage_data: dict[str, Any]) -> StageDeployConfig: runtime_cfg = dict(stage_data.get("runtime", {})) devices = runtime_cfg.get("devices", stage_data.get("devices")) num_replicas = runtime_cfg.get("num_replicas", stage_data.get("num_replicas", 1)) + env = runtime_cfg.get("env", stage_data.get("env")) if "engine_args" in stage_data: for k, v in stage_data["engine_args"].items(): @@ -512,6 +515,7 @@ def _parse_stage_deploy(stage_data: dict[str, Any]) -> StageDeployConfig: "stage_id": stage_data["stage_id"], "devices": devices, "num_replicas": int(num_replicas), + "env": env, } for name, f in _STAGE_DEPLOY_FIELDS.items(): if name in flat_args: @@ -651,8 +655,14 @@ def load_deploy_config(path: str | Path) -> DeployConfig: return DeployConfig(**kwargs) -def _extract_platform_overrides(ps: dict[str, Any]) -> tuple[dict[str, Any], str | None]: - """Return ``(overrides, devices)`` from a platform stage entry. +class PlatformOverrides(NamedTuple): + overrides: dict[str, Any] + devices: str | None + env: dict[str, Any] | None + + +def _extract_platform_overrides(ps: dict[str, Any]) -> PlatformOverrides: + """Return overrides, devices, and env from a platform stage entry. Handles both the nested layout (``engine_args:`` / ``runtime.devices``) and the flat layout. ``devices`` is ``None`` when no override is set. @@ -662,9 +672,9 @@ def _extract_platform_overrides(ps: dict[str, Any]) -> tuple[dict[str, Any], str runtime_cfg = ps.get("runtime", {}) if "num_replicas" in runtime_cfg: overrides["num_replicas"] = runtime_cfg["num_replicas"] - return overrides, runtime_cfg.get("devices") - overrides = {k: v for k, v in ps.items() if k not in ("stage_id", "devices")} - return overrides, ps.get("devices") + return PlatformOverrides(overrides, runtime_cfg.get("devices"), runtime_cfg.get("env")) + overrides = {k: v for k, v in ps.items() if k not in ("stage_id", "devices", "env")} + return PlatformOverrides(overrides, ps.get("devices"), ps.get("env")) def _apply_platform_overrides( @@ -689,10 +699,21 @@ def _apply_platform_overrides( base = base_by_id.get(ps["stage_id"]) if base is None: continue - overrides, devices = _extract_platform_overrides(ps) - if devices is not None: - base.devices = devices - for key, val in overrides.items(): + po = _extract_platform_overrides(ps) + if po.devices is not None: + base.devices = po.devices + if po.env is not None: + if isinstance(base.env, dict) and isinstance(po.env, dict): + base.env = {**base.env, **po.env} + else: + logger.warning( + "Stage %s env override replaces base env entirely (base type=%s, override type=%s)", + ps["stage_id"], + type(base.env).__name__, + type(po.env).__name__, + ) + base.env = po.env + for key, val in po.overrides.items(): if hasattr(base, key): # Deep-merge dict-valued fields listed in _DEEP_MERGE_KEYS so # platform overlays don't silently clobber sibling keys (e.g. @@ -880,6 +901,8 @@ def merge_pipeline_deploy( if ds.devices is not None: runtime["devices"] = ds.devices runtime["num_replicas"] = ds.num_replicas + if ds.env is not None: + runtime["env"] = ds.env runtime["requires_multimodal_data"] = ps.requires_multimodal_data result.append( diff --git a/vllm_omni/deploy/glm_image.yaml b/vllm_omni/deploy/glm_image.yaml index 099df1b15..470bf2cd4 100644 --- a/vllm_omni/deploy/glm_image.yaml +++ b/vllm_omni/deploy/glm_image.yaml @@ -46,3 +46,40 @@ stages: guidance_scale: 1.5 height: 1024 width: 1024 + +platforms: + npu: + stages: + - stage_id: 0 + devices: "0,1,2,3" + env: + # When multiple business processes share a single NPU simultaneously, + # port conflict issues may occur. You can refer to the link below to set different parameters for different processes, + # replacing the default parameter configuration. + # https://www.hiascend.com/document/detail/zh/canncommercial/850/maintenref/envvar/envref_07_0143.html + HCCL_IF_BASE_PORT: 23000 + HCCL_HOST_SOCKET_PORT_RANGE: "23000-23199" + HCCL_NPU_SOCKET_PORT_RANGE: "23200-23399" + async_scheduling: true + enforce_eager: false + distributed_executor_backend: "mp" + enable_prefix_caching: true + tensor_parallel_size: 4 + sequence_parallel_size: 1 + compilation_config: + cudagraph_capture_sizes: [1, 2] + cudagraph_mode: "FULL_DECODE_ONLY" + worker_cls: vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + - stage_id: 1 + devices: "0,1,2,3" + env: + HCCL_IF_BASE_PORT: 24000 + HCCL_HOST_SOCKET_PORT_RANGE: "24000-24199" + HCCL_NPU_SOCKET_PORT_RANGE: "24200-24399" + distributed_executor_backend: "mp" + parallel_config: + tensor_parallel_size: 1 + sequence_parallel_size: 4 + ulysses_degree: 4 + ring_degree: 1 diff --git a/vllm_omni/engine/async_omni_engine.py b/vllm_omni/engine/async_omni_engine.py index 23ef9b855..2396f433a 100644 --- a/vllm_omni/engine/async_omni_engine.py +++ b/vllm_omni/engine/async_omni_engine.py @@ -100,7 +100,7 @@ from vllm_omni.engine.stage_init_utils import ( load_omni_transfer_config_for_model, prepare_engine_environment, release_device_locks, - setup_stage_devices, + stage_runtime_setup, terminate_alive_proc, ) from vllm_omni.engine.stage_pool import StagePool, StagePoolClient @@ -875,50 +875,53 @@ class AsyncOmniEngine: with llm_stage_launch_lock: previous_visible_devices = os.environ.get(device_control_env) try: - setup_stage_devices(plan.metadata.stage_id, plan.metadata.runtime_cfg) - vllm_config = plan.stage_vllm_config - executor_class = plan.executor_class - assert vllm_config is not None - assert executor_class is not None - engine_args_dict = build_engine_args_dict( - stage_cfg, - self.model, - stage_connector_spec=plan.stage_connector_spec, - cli_tokenizer=getattr(self, "tokenizer", None), - ) - lock_fds = acquire_device_locks( + with stage_runtime_setup( plan.metadata.stage_id, - engine_args_dict, - stage_init_timeout, - ) - if self.single_stage_mode and self._omni_master_server is not None: - coord_router_addr: str | None = ( - self._coordinator_runtime.router_address - if self._coordinator_runtime is not None - else None + plan.metadata.runtime_cfg, + ): + vllm_config = plan.stage_vllm_config + executor_class = plan.executor_class + assert vllm_config is not None + assert executor_class is not None + engine_args_dict = build_engine_args_dict( + stage_cfg, + self.model, + stage_connector_spec=plan.stage_connector_spec, + cli_tokenizer=getattr(self, "tokenizer", None), ) - engine_manager, coordinator, addresses = launch_stack.enter_context( - launch_omni_core_engines( + lock_fds = acquire_device_locks( + plan.metadata.stage_id, + engine_args_dict, + stage_init_timeout, + ) + if self.single_stage_mode and self._omni_master_server is not None: + coord_router_addr: str | None = ( + self._coordinator_runtime.router_address + if self._coordinator_runtime is not None + else None + ) + engine_manager, coordinator, addresses = launch_stack.enter_context( + launch_omni_core_engines( + vllm_config=vllm_config, + executor_class=executor_class, + log_stats=False, + omni_master_server=self._omni_master_server, + stage_id=plan.metadata.stage_id, + stage_config=stage_cfg, + replica_id=plan.replica_id, + omni_coordinator_address=coord_router_addr, + ) + ) + else: + addresses, proc, handshake_address = spawn_stage_core( vllm_config=vllm_config, executor_class=executor_class, log_stats=False, - omni_master_server=self._omni_master_server, - stage_id=plan.metadata.stage_id, - stage_config=stage_cfg, - replica_id=plan.replica_id, - omni_coordinator_address=coord_router_addr, ) + logger.info( + "[AsyncOmniEngine] Stage %s engine launch started", + plan.metadata.stage_id, ) - else: - addresses, proc, handshake_address = spawn_stage_core( - vllm_config=vllm_config, - executor_class=executor_class, - log_stats=False, - ) - logger.info( - "[AsyncOmniEngine] Stage %s engine launch started", - plan.metadata.stage_id, - ) finally: if previous_visible_devices is None: current_omni_platform.unset_device_control_env_var() @@ -1017,68 +1020,71 @@ class AsyncOmniEngine: with stage_launch_lock: previous_visible_devices = os.environ.get(device_control_env) try: - setup_stage_devices(plan.metadata.stage_id, plan.metadata.runtime_cfg) - omni_conn_cfg, omni_from, omni_to = plan.omni_kv_connector - if omni_conn_cfg: - inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to) - inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self.stage_configs) - if self.single_stage_mode: - assert self._omni_master_server is not None - od_config = build_diffusion_config(self.model, plan.stage_cfg, plan.metadata) - lock_fds = acquire_diffusion_device_locks( - plan.metadata.stage_id, - od_config, - stage_init_timeout, - ) - handshake_address, request_address, response_address = register_stage_with_omni_master( - omni_master_address=self._omni_master_server.address, - omni_master_port=self._omni_master_server.port, - omni_stage_id=plan.metadata.stage_id, - omni_stage_config=plan.stage_cfg, - return_addresses=True, - replica_id=plan.replica_id, - ) - logger.info( - "[AsyncOmniEngine] Stage %s diffusion registration completed", - plan.metadata.stage_id, - ) - coord_router_addr: str | None = ( - self._coordinator_runtime.router_address - if self._coordinator_runtime is not None - else None - ) - proc, _, _, _ = spawn_diffusion_proc( - self.model, - od_config, - handshake_address=handshake_address, - request_address=request_address, - response_address=response_address, - omni_coordinator_address=coord_router_addr, - omni_stage_id=plan.metadata.stage_id, - omni_replica_id=plan.replica_id, - ) - complete_diffusion_handshake(proc, handshake_address, stage_init_timeout) - logger.info( - "[AsyncOmniEngine] Stage %s diffusion startup completed", - plan.metadata.stage_id, - ) - client = StageDiffusionClient.from_addresses( - plan.metadata, - request_address=request_address, - response_address=response_address, - proc=proc, - batch_size=self.diffusion_batch_size, - ) - else: - client = initialize_diffusion_stage( - plan.metadata.stage_id, - self.model, - plan.stage_cfg, - plan.metadata, - stage_init_timeout=stage_init_timeout, - batch_size=self.diffusion_batch_size, - use_inline=self.num_stages == 1 and plan.num_replicas == 1, - ) + with stage_runtime_setup( + plan.metadata.stage_id, + plan.metadata.runtime_cfg, + ): + omni_conn_cfg, omni_from, omni_to = plan.omni_kv_connector + if omni_conn_cfg: + inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to) + inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self.stage_configs) + if self.single_stage_mode: + assert self._omni_master_server is not None + od_config = build_diffusion_config(self.model, plan.stage_cfg, plan.metadata) + lock_fds = acquire_diffusion_device_locks( + plan.metadata.stage_id, + od_config, + stage_init_timeout, + ) + handshake_address, request_address, response_address = register_stage_with_omni_master( + omni_master_address=self._omni_master_server.address, + omni_master_port=self._omni_master_server.port, + omni_stage_id=plan.metadata.stage_id, + omni_stage_config=plan.stage_cfg, + return_addresses=True, + replica_id=plan.replica_id, + ) + logger.info( + "[AsyncOmniEngine] Stage %s diffusion registration completed", + plan.metadata.stage_id, + ) + coord_router_addr: str | None = ( + self._coordinator_runtime.router_address + if self._coordinator_runtime is not None + else None + ) + proc, _, _, _ = spawn_diffusion_proc( + self.model, + od_config, + handshake_address=handshake_address, + request_address=request_address, + response_address=response_address, + omni_coordinator_address=coord_router_addr, + omni_stage_id=plan.metadata.stage_id, + omni_replica_id=plan.replica_id, + ) + complete_diffusion_handshake(proc, handshake_address, stage_init_timeout) + logger.info( + "[AsyncOmniEngine] Stage %s diffusion startup completed", + plan.metadata.stage_id, + ) + client = StageDiffusionClient.from_addresses( + plan.metadata, + request_address=request_address, + response_address=response_address, + proc=proc, + batch_size=self.diffusion_batch_size, + ) + else: + client = initialize_diffusion_stage( + plan.metadata.stage_id, + self.model, + plan.stage_cfg, + plan.metadata, + stage_init_timeout=stage_init_timeout, + batch_size=self.diffusion_batch_size, + use_inline=self.num_stages == 1 and plan.num_replicas == 1, + ) finally: if previous_visible_devices is None: current_omni_platform.unset_device_control_env_var() diff --git a/vllm_omni/engine/stage_init_utils.py b/vllm_omni/engine/stage_init_utils.py index 1a9f98fe5..bd727b930 100644 --- a/vllm_omni/engine/stage_init_utils.py +++ b/vllm_omni/engine/stage_init_utils.py @@ -13,9 +13,10 @@ import importlib import multiprocessing as mp import os import time -from collections.abc import Callable, Sequence +from collections.abc import Callable, Generator, Sequence +from contextlib import contextmanager from dataclasses import dataclass, replace -from typing import Any, Literal +from typing import Any, Literal, cast from vllm.logger import init_logger from vllm.sampling_params import SamplingParams @@ -635,6 +636,62 @@ def setup_stage_devices(stage_id: int, runtime_cfg: Any) -> None: ) +@contextmanager +def stage_runtime_setup(stage_id: int, runtime_cfg: Any) -> Generator[None, None, None]: + """Apply per-stage ``runtime.env`` and ``runtime.devices`` for the context. + + Restores ``runtime.env`` on exit. Device visibility restore remains the + caller's responsibility (e.g. ``AsyncOmniEngine`` saves/restores the + platform device-control env var around this block). + """ + with stage_runtime_env(stage_id, runtime_cfg): + setup_stage_devices(stage_id, runtime_cfg) + yield + + +@contextmanager +def stage_runtime_env(stage_id: int, runtime_cfg: Any) -> Generator[None, None, None]: + """Apply per-stage ``runtime.env`` for the duration of the context.""" + if runtime_cfg is None: + runtime_cfg = {} + elif not isinstance(runtime_cfg, dict): + runtime_cfg = cast(dict[str, Any], _to_dict(runtime_cfg)) + + raw_env = runtime_cfg.get("env") + if raw_env is None: + yield + return + if isinstance(raw_env, dict): + runtime_env = cast(dict[str, Any], raw_env) + else: + runtime_env = cast(dict[str, Any], _to_dict(raw_env)) + if not runtime_env: + logger.warning( + "[stage_init] Stage-%s ignored runtime.env with unsupported type %s", + stage_id, + type(raw_env).__name__, + ) + yield + return + + previous_env: dict[str, str | None] = {} + for key, value in runtime_env.items(): + env_key = str(key) + previous_env[env_key] = os.environ.get(env_key) + os.environ[env_key] = str(value) + + if previous_env: + logger.info("[stage_init] Stage-%s applied runtime env keys: %s", stage_id, sorted(previous_env)) + try: + yield + finally: + for key, old_value in previous_env.items(): + if old_value is None: + os.environ.pop(key, None) + else: + os.environ[key] = old_value + + def build_engine_args_dict( stage_config: Any, model: str,