glm-image: fix(npu)per-stage runtime env for HCCL ports + GLM-Image NPU stage config (#3235)

Signed-off-by: lyj-jjj <liuyingjun5@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-05-27 12:18:47 +12:00
1 parent 1f1c82ecdd
commit 283873411a
6 files changed
+254 -120

No files matched your search

@@ -3,6 +3,7 @@ import os
import threading
import time
import types
from contextlib import contextmanager
import pytest
@@ -189,10 +190,12 @@ def test_initialize_diffusion_replica_restores_device_visibility_after_local_ini
old_env = os.environ.get(env_var)
os.environ[env_var] = "0,1"
def _fake_setup_stage_devices(_stage_id, _runtime_cfg):
@contextmanager
def _fake_stage_runtime_setup(_stage_id, _runtime_cfg):
current_omni_platform.set_device_control_env_var("1")
yield
monkeypatch.setattr(engine_mod, "setup_stage_devices", _fake_setup_stage_devices)
monkeypatch.setattr(engine_mod, "stage_runtime_setup", _fake_stage_runtime_setup)
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
monkeypatch.setattr(engine_mod, "initialize_diffusion_stage", lambda *_, **__: types.SimpleNamespace())
@@ -221,7 +224,11 @@ def test_initialize_diffusion_replica_passes_stage_init_timeout_and_inline_flag(
captured: dict[str, object] = {}
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
@contextmanager
def _noop_stage_runtime_setup(*_):
yield
monkeypatch.setattr(engine_mod, "stage_runtime_setup", _noop_stage_runtime_setup)
monkeypatch.setattr(engine_mod, "inject_kv_stage_info", lambda *_: None)
def _capture_initialize_diffusion_stage(
@@ -527,7 +534,11 @@ def test_initialize_llm_replica_passes_stage_init_timeout_to_complete_stage_hand
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
monkeypatch.setattr(engine_mod, "setup_stage_devices", lambda *_: None)
@contextmanager
def _noop_stage_runtime_setup(*_):
yield
monkeypatch.setattr(engine_mod, "stage_runtime_setup", _noop_stage_runtime_setup)
monkeypatch.setattr(engine_mod, "build_engine_args_dict", lambda *_, **__: {})
monkeypatch.setattr(engine_mod, "acquire_device_locks", lambda *_: [])
monkeypatch.setattr(engine_mod, "spawn_stage_core", lambda **_: (fake_addresses, fake_proc, "ipc://handshake"))
+4 -4
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
import os
import threading
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from types import SimpleNamespace
from typing import Any
@@ -835,7 +835,7 @@ class TestSingleStageReplicaInitialization:
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
mocker.patch.object(engine_mod, "setup_stage_devices")
mocker.patch.object(engine_mod, "stage_runtime_setup", return_value=nullcontext())
mocker.patch.object(engine_mod, "build_engine_args_dict", return_value={})
mocker.patch.object(engine_mod, "acquire_device_locks", return_value=[])
mocker.patch.object(engine_mod, "release_device_locks")
@@ -909,7 +909,7 @@ class TestSingleStageReplicaInitialization:
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
mocker.patch.object(engine_mod, "setup_stage_devices")
mocker.patch.object(engine_mod, "stage_runtime_setup", return_value=nullcontext())
mocker.patch.object(engine_mod, "inject_kv_stage_info")
mocker.patch.object(engine_mod, "build_diffusion_config", return_value="diffusion-config")
mock_register = mocker.patch.object(
@@ -988,7 +988,7 @@ class TestSingleStageReplicaInitialization:
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
mocker.patch.object(engine_mod, "setup_stage_devices")
mocker.patch.object(engine_mod, "stage_runtime_setup", return_value=nullcontext())
mocker.patch.object(engine_mod, "inject_kv_stage_info")
mocker.patch.object(engine_mod, "build_diffusion_config", return_value="diffusion-config")
mocker.patch.object(
+33 -10
View File
@@ -10,7 +10,7 @@ import warnings
from dataclasses import asdict, dataclass, field, fields
from enum import Enum
from pathlib import Path
from typing import Any
from typing import Any, NamedTuple
from vllm.logger import init_logger
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
@@ -408,6 +408,7 @@ class StageDeployConfig:
stage_id: int
devices: str | None = None
num_replicas: int = 1
env: dict[str, Any] | None = None
# Inter-stage connector wiring and request defaults.
output_connectors: dict[str, str] | None = None
@@ -478,6 +479,7 @@ _STAGE_RESERVED_KEYS = frozenset(
"stage_id",
"devices",
"num_replicas",
"env",
"output_connectors",
"input_connectors",
"default_sampling_params",
@@ -498,6 +500,7 @@ def _parse_stage_deploy(stage_data: dict[str, Any]) -> StageDeployConfig:
runtime_cfg = dict(stage_data.get("runtime", {}))
devices = runtime_cfg.get("devices", stage_data.get("devices"))
num_replicas = runtime_cfg.get("num_replicas", stage_data.get("num_replicas", 1))
env = runtime_cfg.get("env", stage_data.get("env"))
if "engine_args" in stage_data:
for k, v in stage_data["engine_args"].items():
@@ -512,6 +515,7 @@ def _parse_stage_deploy(stage_data: dict[str, Any]) -> StageDeployConfig:
"stage_id": stage_data["stage_id"],
"devices": devices,
"num_replicas": int(num_replicas),
"env": env,
}
for name, f in _STAGE_DEPLOY_FIELDS.items():
if name in flat_args:
@@ -651,8 +655,14 @@ def load_deploy_config(path: str | Path) -> DeployConfig:
return DeployConfig(**kwargs)
def _extract_platform_overrides(ps: dict[str, Any]) -> tuple[dict[str, Any], str | None]:
"""Return ``(overrides, devices)`` from a platform stage entry.
class PlatformOverrides(NamedTuple):
overrides: dict[str, Any]
devices: str | None
env: dict[str, Any] | None
def _extract_platform_overrides(ps: dict[str, Any]) -> PlatformOverrides:
"""Return overrides, devices, and env from a platform stage entry.
Handles both the nested layout (``engine_args:`` / ``runtime.devices``) and
the flat layout. ``devices`` is ``None`` when no override is set.
@@ -662,9 +672,9 @@ def _extract_platform_overrides(ps: dict[str, Any]) -> tuple[dict[str, Any], str
runtime_cfg = ps.get("runtime", {})
if "num_replicas" in runtime_cfg:
overrides["num_replicas"] = runtime_cfg["num_replicas"]
return overrides, runtime_cfg.get("devices")
overrides = {k: v for k, v in ps.items() if k not in ("stage_id", "devices")}
return overrides, ps.get("devices")
return PlatformOverrides(overrides, runtime_cfg.get("devices"), runtime_cfg.get("env"))
overrides = {k: v for k, v in ps.items() if k not in ("stage_id", "devices", "env")}
return PlatformOverrides(overrides, ps.get("devices"), ps.get("env"))
def _apply_platform_overrides(
@@ -689,10 +699,21 @@ def _apply_platform_overrides(
base = base_by_id.get(ps["stage_id"])
if base is None:
continue
overrides, devices = _extract_platform_overrides(ps)
if devices is not None:
base.devices = devices
for key, val in overrides.items():
po = _extract_platform_overrides(ps)
if po.devices is not None:
base.devices = po.devices
if po.env is not None:
if isinstance(base.env, dict) and isinstance(po.env, dict):
base.env = {**base.env, **po.env}
else:
logger.warning(
"Stage %s env override replaces base env entirely (base type=%s, override type=%s)",
ps["stage_id"],
type(base.env).__name__,
type(po.env).__name__,
)
base.env = po.env
for key, val in po.overrides.items():
if hasattr(base, key):
# Deep-merge dict-valued fields listed in _DEEP_MERGE_KEYS so
# platform overlays don't silently clobber sibling keys (e.g.
@@ -880,6 +901,8 @@ def merge_pipeline_deploy(
if ds.devices is not None:
runtime["devices"] = ds.devices
runtime["num_replicas"] = ds.num_replicas
if ds.env is not None:
runtime["env"] = ds.env
runtime["requires_multimodal_data"] = ps.requires_multimodal_data
result.append(
+37
View File
@@ -46,3 +46,40 @@ stages:
guidance_scale: 1.5
height: 1024
width: 1024
platforms:
npu:
stages:
- stage_id: 0
devices: "0,1,2,3"
env:
# When multiple business processes share a single NPU simultaneously,
# port conflict issues may occur. You can refer to the link below to set different parameters for different processes,
# replacing the default parameter configuration.
# https://www.hiascend.com/document/detail/zh/canncommercial/850/maintenref/envvar/envref_07_0143.html
HCCL_IF_BASE_PORT: 23000
HCCL_HOST_SOCKET_PORT_RANGE: "23000-23199"
HCCL_NPU_SOCKET_PORT_RANGE: "23200-23399"
async_scheduling: true
enforce_eager: false
distributed_executor_backend: "mp"
enable_prefix_caching: true
tensor_parallel_size: 4
sequence_parallel_size: 1
compilation_config:
cudagraph_capture_sizes: [1, 2]
cudagraph_mode: "FULL_DECODE_ONLY"
worker_cls: vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
- stage_id: 1
devices: "0,1,2,3"
env:
HCCL_IF_BASE_PORT: 24000
HCCL_HOST_SOCKET_PORT_RANGE: "24000-24199"
HCCL_NPU_SOCKET_PORT_RANGE: "24200-24399"
distributed_executor_backend: "mp"
parallel_config:
tensor_parallel_size: 1
sequence_parallel_size: 4
ulysses_degree: 4
ring_degree: 1
+106 -100
View File
@@ -100,7 +100,7 @@ from vllm_omni.engine.stage_init_utils import (
load_omni_transfer_config_for_model,
prepare_engine_environment,
release_device_locks,
setup_stage_devices,
stage_runtime_setup,
terminate_alive_proc,
)
from vllm_omni.engine.stage_pool import StagePool, StagePoolClient
@@ -875,50 +875,53 @@ class AsyncOmniEngine:
with llm_stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(plan.metadata.stage_id, plan.metadata.runtime_cfg)
vllm_config = plan.stage_vllm_config
executor_class = plan.executor_class
assert vllm_config is not None
assert executor_class is not None
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=plan.stage_connector_spec,
cli_tokenizer=getattr(self, "tokenizer", None),
)
lock_fds = acquire_device_locks(
with stage_runtime_setup(
plan.metadata.stage_id,
engine_args_dict,
stage_init_timeout,
)
if self.single_stage_mode and self._omni_master_server is not None:
coord_router_addr: str | None = (
self._coordinator_runtime.router_address
if self._coordinator_runtime is not None
else None
plan.metadata.runtime_cfg,
):
vllm_config = plan.stage_vllm_config
executor_class = plan.executor_class
assert vllm_config is not None
assert executor_class is not None
engine_args_dict = build_engine_args_dict(
stage_cfg,
self.model,
stage_connector_spec=plan.stage_connector_spec,
cli_tokenizer=getattr(self, "tokenizer", None),
)
engine_manager, coordinator, addresses = launch_stack.enter_context(
launch_omni_core_engines(
lock_fds = acquire_device_locks(
plan.metadata.stage_id,
engine_args_dict,
stage_init_timeout,
)
if self.single_stage_mode and self._omni_master_server is not None:
coord_router_addr: str | None = (
self._coordinator_runtime.router_address
if self._coordinator_runtime is not None
else None
)
engine_manager, coordinator, addresses = launch_stack.enter_context(
launch_omni_core_engines(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
omni_master_server=self._omni_master_server,
stage_id=plan.metadata.stage_id,
stage_config=stage_cfg,
replica_id=plan.replica_id,
omni_coordinator_address=coord_router_addr,
)
)
else:
addresses, proc, handshake_address = spawn_stage_core(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
omni_master_server=self._omni_master_server,
stage_id=plan.metadata.stage_id,
stage_config=stage_cfg,
replica_id=plan.replica_id,
omni_coordinator_address=coord_router_addr,
)
logger.info(
"[AsyncOmniEngine] Stage %s engine launch started",
plan.metadata.stage_id,
)
else:
addresses, proc, handshake_address = spawn_stage_core(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
)
logger.info(
"[AsyncOmniEngine] Stage %s engine launch started",
plan.metadata.stage_id,
)
finally:
if previous_visible_devices is None:
current_omni_platform.unset_device_control_env_var()
@@ -1017,68 +1020,71 @@ class AsyncOmniEngine:
with stage_launch_lock:
previous_visible_devices = os.environ.get(device_control_env)
try:
setup_stage_devices(plan.metadata.stage_id, plan.metadata.runtime_cfg)
omni_conn_cfg, omni_from, omni_to = plan.omni_kv_connector
if omni_conn_cfg:
inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to)
inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self.stage_configs)
if self.single_stage_mode:
assert self._omni_master_server is not None
od_config = build_diffusion_config(self.model, plan.stage_cfg, plan.metadata)
lock_fds = acquire_diffusion_device_locks(
plan.metadata.stage_id,
od_config,
stage_init_timeout,
)
handshake_address, request_address, response_address = register_stage_with_omni_master(
omni_master_address=self._omni_master_server.address,
omni_master_port=self._omni_master_server.port,
omni_stage_id=plan.metadata.stage_id,
omni_stage_config=plan.stage_cfg,
return_addresses=True,
replica_id=plan.replica_id,
)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion registration completed",
plan.metadata.stage_id,
)
coord_router_addr: str | None = (
self._coordinator_runtime.router_address
if self._coordinator_runtime is not None
else None
)
proc, _, _, _ = spawn_diffusion_proc(
self.model,
od_config,
handshake_address=handshake_address,
request_address=request_address,
response_address=response_address,
omni_coordinator_address=coord_router_addr,
omni_stage_id=plan.metadata.stage_id,
omni_replica_id=plan.replica_id,
)
complete_diffusion_handshake(proc, handshake_address, stage_init_timeout)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion startup completed",
plan.metadata.stage_id,
)
client = StageDiffusionClient.from_addresses(
plan.metadata,
request_address=request_address,
response_address=response_address,
proc=proc,
batch_size=self.diffusion_batch_size,
)
else:
client = initialize_diffusion_stage(
plan.metadata.stage_id,
self.model,
plan.stage_cfg,
plan.metadata,
stage_init_timeout=stage_init_timeout,
batch_size=self.diffusion_batch_size,
use_inline=self.num_stages == 1 and plan.num_replicas == 1,
)
with stage_runtime_setup(
plan.metadata.stage_id,
plan.metadata.runtime_cfg,
):
omni_conn_cfg, omni_from, omni_to = plan.omni_kv_connector
if omni_conn_cfg:
inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to)
inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self.stage_configs)
if self.single_stage_mode:
assert self._omni_master_server is not None
od_config = build_diffusion_config(self.model, plan.stage_cfg, plan.metadata)
lock_fds = acquire_diffusion_device_locks(
plan.metadata.stage_id,
od_config,
stage_init_timeout,
)
handshake_address, request_address, response_address = register_stage_with_omni_master(
omni_master_address=self._omni_master_server.address,
omni_master_port=self._omni_master_server.port,
omni_stage_id=plan.metadata.stage_id,
omni_stage_config=plan.stage_cfg,
return_addresses=True,
replica_id=plan.replica_id,
)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion registration completed",
plan.metadata.stage_id,
)
coord_router_addr: str | None = (
self._coordinator_runtime.router_address
if self._coordinator_runtime is not None
else None
)
proc, _, _, _ = spawn_diffusion_proc(
self.model,
od_config,
handshake_address=handshake_address,
request_address=request_address,
response_address=response_address,
omni_coordinator_address=coord_router_addr,
omni_stage_id=plan.metadata.stage_id,
omni_replica_id=plan.replica_id,
)
complete_diffusion_handshake(proc, handshake_address, stage_init_timeout)
logger.info(
"[AsyncOmniEngine] Stage %s diffusion startup completed",
plan.metadata.stage_id,
)
client = StageDiffusionClient.from_addresses(
plan.metadata,
request_address=request_address,
response_address=response_address,
proc=proc,
batch_size=self.diffusion_batch_size,
)
else:
client = initialize_diffusion_stage(
plan.metadata.stage_id,
self.model,
plan.stage_cfg,
plan.metadata,
stage_init_timeout=stage_init_timeout,
batch_size=self.diffusion_batch_size,
use_inline=self.num_stages == 1 and plan.num_replicas == 1,
)
finally:
if previous_visible_devices is None:
current_omni_platform.unset_device_control_env_var()
+59 -2
View File
@@ -13,9 +13,10 @@ import importlib
import multiprocessing as mp
import os
import time
from collections.abc import Callable, Sequence
from collections.abc import Callable, Generator, Sequence
from contextlib import contextmanager
from dataclasses import dataclass, replace
from typing import Any, Literal
from typing import Any, Literal, cast
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
@@ -635,6 +636,62 @@ def setup_stage_devices(stage_id: int, runtime_cfg: Any) -> None:
)
@contextmanager
def stage_runtime_setup(stage_id: int, runtime_cfg: Any) -> Generator[None, None, None]:
"""Apply per-stage ``runtime.env`` and ``runtime.devices`` for the context.
Restores ``runtime.env`` on exit. Device visibility restore remains the
caller's responsibility (e.g. ``AsyncOmniEngine`` saves/restores the
platform device-control env var around this block).
"""
with stage_runtime_env(stage_id, runtime_cfg):
setup_stage_devices(stage_id, runtime_cfg)
yield
@contextmanager
def stage_runtime_env(stage_id: int, runtime_cfg: Any) -> Generator[None, None, None]:
"""Apply per-stage ``runtime.env`` for the duration of the context."""
if runtime_cfg is None:
runtime_cfg = {}
elif not isinstance(runtime_cfg, dict):
runtime_cfg = cast(dict[str, Any], _to_dict(runtime_cfg))
raw_env = runtime_cfg.get("env")
if raw_env is None:
yield
return
if isinstance(raw_env, dict):
runtime_env = cast(dict[str, Any], raw_env)
else:
runtime_env = cast(dict[str, Any], _to_dict(raw_env))
if not runtime_env:
logger.warning(
"[stage_init] Stage-%s ignored runtime.env with unsupported type %s",
stage_id,
type(raw_env).__name__,
)
yield
return
previous_env: dict[str, str | None] = {}
for key, value in runtime_env.items():
env_key = str(key)
previous_env[env_key] = os.environ.get(env_key)
os.environ[env_key] = str(value)
if previous_env:
logger.info("[stage_init] Stage-%s applied runtime env keys: %s", stage_id, sorted(previous_env))
try:
yield
finally:
for key, old_value in previous_env.items():
if old_value is None:
os.environ.pop(key, None)
else:
os.environ[key] = old_value
def build_engine_args_dict(
stage_config: Any,
model: str,