[feature] Enable multi-node inference through Ray backend (#7938)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Co-authored-by: Bartosz Stefaniak <bstefaniak@nvidia.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
20 files changed
+1721
-37
No files matched your search
@@ -71,6 +71,34 @@ def test_stage_runtime_env_accepts_typed_runtime_config(monkeypatch):
|
||||
assert env_key not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize("typed", [False, True])
|
||||
def test_build_ray_diffusion_config_preserves_explicit_stage_env(monkeypatch, typed):
|
||||
from vllm_omni.engine import stage_init_utils as init_mod
|
||||
|
||||
env = {"CUSTOM_PLUGIN_SETTING": "stage", "OMP_NUM_THREADS": 2}
|
||||
runtime_cfg = OmniStageRuntimeConfig(env=env) if typed else {"env": env}
|
||||
metadata = types.SimpleNamespace(
|
||||
runtime_cfg=runtime_cfg, stage_id=1, cfg_kv_collect_func=None, default_sampling_params=None
|
||||
)
|
||||
config = types.SimpleNamespace(
|
||||
distributed_executor_backend="ray", parallel_config=types.SimpleNamespace(world_size=2)
|
||||
)
|
||||
monkeypatch.setattr(init_mod, "build_engine_args_dict", lambda *args: {})
|
||||
monkeypatch.setattr(init_mod, "OmniDiffusionConfig", lambda **kwargs: config)
|
||||
monkeypatch.setattr(
|
||||
init_mod,
|
||||
"current_omni_platform",
|
||||
types.SimpleNamespace(device_control_env_var=None, get_device_count=lambda: 0),
|
||||
)
|
||||
|
||||
result = init_mod.build_diffusion_config("model", {}, metadata)
|
||||
|
||||
assert result is config
|
||||
assert result.ray_worker_env == {"CUSTOM_PLUGIN_SETTING": "stage", "OMP_NUM_THREADS": "2"}
|
||||
assert result.num_gpus == 2
|
||||
assert env["OMP_NUM_THREADS"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_key,invalid_value",
|
||||
[("INVALID=ENV", "value"), ("INVALID\0ENV", "value"), ("INVALID_ENV", "value\0")],
|
||||
@@ -702,7 +730,11 @@ def test_launch_diffusion_stage_replica_preserves_configured_max_num_seqs(monkey
|
||||
assert kwargs["visible_devices"] == "1"
|
||||
return [42]
|
||||
|
||||
od_config = types.SimpleNamespace(max_num_seqs=4, parallel_config=types.SimpleNamespace(world_size=1))
|
||||
od_config = types.SimpleNamespace(
|
||||
max_num_seqs=4,
|
||||
distributed_executor_backend="mp",
|
||||
parallel_config=types.SimpleNamespace(world_size=1),
|
||||
)
|
||||
monkeypatch.setattr(startup_mod, "build_diffusion_config", lambda *args: od_config)
|
||||
monkeypatch.setattr(startup_mod, "acquire_device_locks", acquire)
|
||||
monkeypatch.setattr(
|
||||
@@ -796,6 +828,7 @@ def test_launch_diffusion_stage_replica_preserves_step_execution_max_num_seqs(mo
|
||||
od_config = types.SimpleNamespace(
|
||||
max_num_seqs=8,
|
||||
step_execution=True,
|
||||
distributed_executor_backend="mp",
|
||||
parallel_config=types.SimpleNamespace(world_size=1),
|
||||
)
|
||||
monkeypatch.setattr(startup_mod, "build_diffusion_config", lambda *args: od_config)
|
||||
|
||||
@@ -1219,7 +1219,11 @@ class TestSingleStageReplicaInitialization:
|
||||
runtime._init_visible_devices_baseline = "0"
|
||||
|
||||
mocker.patch.object(runtime_mod, "inject_kv_stage_info")
|
||||
od_config = SimpleNamespace(max_num_seqs=4, parallel_config=SimpleNamespace(world_size=1))
|
||||
od_config = SimpleNamespace(
|
||||
max_num_seqs=4,
|
||||
parallel_config=SimpleNamespace(world_size=1),
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.build_diffusion_config", return_value=od_config)
|
||||
mock_register = mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
@@ -1352,7 +1356,11 @@ class TestSingleStageReplicaInitialization:
|
||||
runtime._init_visible_devices_baseline = "0"
|
||||
|
||||
mocker.patch.object(runtime_mod, "inject_kv_stage_info")
|
||||
od_config = SimpleNamespace(max_num_seqs=None, parallel_config=SimpleNamespace(world_size=1))
|
||||
od_config = SimpleNamespace(
|
||||
max_num_seqs=None,
|
||||
parallel_config=SimpleNamespace(world_size=1),
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_engine_startup.build_diffusion_config", return_value=od_config)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_engine_startup.register_stage_with_omni_master",
|
||||
|
||||
Reference in new issue
Block a user