[Core] Retain structured configs through runtime startup (#6849)
Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
This commit is contained in:
1 parent
becfefd440
commit
f1a6e7ce2a
49 files changed
+1856
-331
No files matched your search
@@ -323,7 +323,7 @@ inputs, fields, and ownership rules are described below.
|
|||||||
flowchart TB
|
flowchart TB
|
||||||
layer1["Layer 1 · Authoring inputs<br/>PipelineConfig + DeployConfig"]
|
layer1["Layer 1 · Authoring inputs<br/>PipelineConfig + DeployConfig"]
|
||||||
layer2["Layer 2 · Production resolution boundary<br/>resolve_omni_config()<br/>(StageConfigFactory + VllmOmniConfig internally)"]
|
layer2["Layer 2 · Production resolution boundary<br/>resolve_omni_config()<br/>(StageConfigFactory + VllmOmniConfig internally)"]
|
||||||
layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + temporary OmegaConf stage bridge"]
|
layer3["Layer 3 · Startup hand-off<br/>OmniConfigResolution<br/>PipelineConfig + typed stage configs"]
|
||||||
layer4["Layer 4 · Runtime launch planning<br/>StageRuntime"]
|
layer4["Layer 4 · Runtime launch planning<br/>StageRuntime"]
|
||||||
layer5["Layer 5 · Engine materialization<br/>VllmConfig / OmniDiffusionConfig"]
|
layer5["Layer 5 · Engine materialization<br/>VllmConfig / OmniDiffusionConfig"]
|
||||||
|
|
||||||
@@ -335,10 +335,10 @@ The production resolution boundary is `resolve_omni_config()` in
|
|||||||
It delegates typed construction to `StageConfigFactory.create_from_model()` and
|
It delegates typed construction to `StageConfigFactory.create_from_model()` and
|
||||||
`VllmOmniConfig.from_pipeline_config()`, then returns an
|
`VllmOmniConfig.from_pipeline_config()`, then returns an
|
||||||
`OmniConfigResolution` consumed by both `AsyncOmniEngine` and headless startup.
|
`OmniConfigResolution` consumed by both `AsyncOmniEngine` and headless startup.
|
||||||
Until `StageRuntime` consumes typed stage configs directly, this envelope carries
|
This envelope carries the effective `PipelineConfig` alongside typed
|
||||||
the effective `PipelineConfig` alongside OmegaConf-compatible `stage_configs` as
|
`stage_configs`, which `StageRuntime` consumes directly. Both views describe the
|
||||||
a temporary runtime bridge; both views describe the same resolved topology,
|
same resolved topology, including injected stages. Backend arguments are
|
||||||
including injected stages.
|
projected from these typed stages when each engine is initialized.
|
||||||
|
|
||||||
The legacy `stage_args` YAML path has been removed. Model topology now resolves
|
The legacy `stage_args` YAML path has been removed. Model topology now resolves
|
||||||
through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`.
|
through `PipelineConfig`, with runtime overrides supplied by `DeployConfig`.
|
||||||
@@ -360,7 +360,7 @@ The important ownership rules are:
|
|||||||
3. CLI and Python overrides are applied at the resolution boundary, with
|
3. CLI and Python overrides are applied at the resolution boundary, with
|
||||||
per-stage overrides taking precedence over global values where supported.
|
per-stage overrides taking precedence over global values where supported.
|
||||||
4. `OmniConfigResolution` is the sole startup hand-off. Its `pipeline_config`
|
4. `OmniConfigResolution` is the sole startup hand-off. Its `pipeline_config`
|
||||||
and temporary `stage_configs` compatibility view must describe the same
|
and typed `stage_configs` must describe the same
|
||||||
topology.
|
topology.
|
||||||
5. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
|
5. `StageRuntime` owns launch planning and replica lifecycle. `ReplicaInitPlan`
|
||||||
is runtime-private state, not a user configuration object.
|
is runtime-private state, not a user configuration object.
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
"""Integration tests: strategy overlay on the real merge_pipeline_deploy seam."""
|
"""Integration tests: strategy overlay on the real merge_pipeline_deploy seam."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -20,6 +20,7 @@ from vllm_omni.config.composable_parallel import (
|
|||||||
apply_strategy_specs,
|
apply_strategy_specs,
|
||||||
)
|
)
|
||||||
from vllm_omni.config.config_factory import StageConfigFactory
|
from vllm_omni.config.config_factory import StageConfigFactory
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||||
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
|
from vllm_omni.config.stage_config import load_deploy_config, merge_pipeline_deploy
|
||||||
|
|
||||||
@@ -137,3 +138,31 @@ def test_cli_overrides_strategy_with_warning():
|
|||||||
assert _resolved(stages, "talker").runtime.num_replicas == 3
|
assert _resolved(stages, "talker").runtime.num_replicas == 3
|
||||||
# ...and the override was warned about, naming the conflicting field.
|
# ...and the override was warned about, naming the conflicting field.
|
||||||
assert any("num_replicas" in m and "overrides the strategy-derived" in m for m in messages)
|
assert any("num_replicas" in m and "overrides the strategy-derived" in m for m in messages)
|
||||||
|
|
||||||
|
|
||||||
|
def test_typed_cli_overrides_strategy():
|
||||||
|
pipeline = OMNI_PIPELINES["qwen2_5_omni"]
|
||||||
|
config = VllmOmniConfig.from_pipeline_config(
|
||||||
|
pipeline,
|
||||||
|
user_deploy_config=load_deploy_config(_DEPLOY),
|
||||||
|
cli_overrides={"stage_1_num_replicas": 3},
|
||||||
|
strategy_specs={"talker": [_stage_replica(2, "round_robin")]},
|
||||||
|
)
|
||||||
|
|
||||||
|
talker = next(stage for stage in config.stage_configs if stage.model_stage == "talker")
|
||||||
|
assert talker.runtime_config.num_replicas == 3
|
||||||
|
assert config.orchestrator_config.omni_lb_policy == "round-robin"
|
||||||
|
assert config.strategy_omni_lb_policy == "round-robin"
|
||||||
|
|
||||||
|
|
||||||
|
def test_typed_tp_only_strategy_has_no_derived_lb_policy():
|
||||||
|
pipeline = OMNI_PIPELINES["qwen2_5_omni"]
|
||||||
|
config = VllmOmniConfig.from_pipeline_config(
|
||||||
|
pipeline,
|
||||||
|
user_deploy_config=load_deploy_config(_DEPLOY),
|
||||||
|
cli_overrides={"omni_lb_policy": "round-robin"},
|
||||||
|
strategy_specs={"thinker": [_tp(1)]},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert config.orchestrator_config.omni_lb_policy == "round-robin"
|
||||||
|
assert config.strategy_omni_lb_policy is None
|
||||||
@@ -2,13 +2,19 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from vllm.pooling_params import PoolingParams
|
||||||
|
|
||||||
import vllm_omni.config.pipeline_registry # noqa: F401 (populate registry)
|
import vllm_omni.config.pipeline_registry # noqa: F401 (populate registry)
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES as _PIPELINE_REGISTRY
|
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES as _PIPELINE_REGISTRY
|
||||||
from vllm_omni.config.stage_config import (
|
from vllm_omni.config.stage_config import (
|
||||||
DeployConfig,
|
DeployConfig,
|
||||||
StageExecutionType,
|
StageExecutionType,
|
||||||
)
|
)
|
||||||
|
from vllm_omni.engine.stage_init_utils import (
|
||||||
|
build_engine_args_dict_from_omni_stage_config,
|
||||||
|
extract_stage_metadata_from_omni_stage_config,
|
||||||
|
)
|
||||||
from vllm_omni.model_executor.stage_input_processors.forced_aligner import (
|
from vllm_omni.model_executor.stage_input_processors.forced_aligner import (
|
||||||
POOLING_OUTPUT_DECODER_PATH,
|
POOLING_OUTPUT_DECODER_PATH,
|
||||||
TIMESTAMPS_MODALITY,
|
TIMESTAMPS_MODALITY,
|
||||||
@@ -94,3 +100,33 @@ def test_aligner_uses_completed_audio_without_disabling_upstream_chunks(async_ch
|
|||||||
assert stages[2].engine_args.runner == "pooling"
|
assert stages[2].engine_args.runner == "pooling"
|
||||||
assert len(pipeline.stages) == 2
|
assert len(pipeline.stages) == 2
|
||||||
assert deploy.stages == []
|
assert deploy.stages == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_injected_aligner_survives_typed_config_and_runtime_projections():
|
||||||
|
pipeline, deploy = inject_forced_aligner_stage(
|
||||||
|
_PIPELINE_REGISTRY["qwen3_tts"],
|
||||||
|
DeployConfig(),
|
||||||
|
{"forced_aligner": "/models/Qwen3-ForcedAligner-0.6B"},
|
||||||
|
)
|
||||||
|
config = VllmOmniConfig.from_pipeline_config(
|
||||||
|
pipeline,
|
||||||
|
user_deploy_config=deploy,
|
||||||
|
cli_overrides={"model": "/models/Qwen3-TTS"},
|
||||||
|
)
|
||||||
|
stage = config.stage_configs[-1]
|
||||||
|
|
||||||
|
assert stage.pooling_config.runner == "pooling"
|
||||||
|
assert stage.pooling_config.pooling_output_decoder == POOLING_OUTPUT_DECODER_PATH
|
||||||
|
assert isinstance(stage.pooling_config.default_pooling_params, PoolingParams)
|
||||||
|
assert stage.pooling_config.default_pooling_params.task == deploy.stages[-1].default_pooling_params["task"]
|
||||||
|
|
||||||
|
metadata = extract_stage_metadata_from_omni_stage_config(stage)
|
||||||
|
assert isinstance(metadata.default_sampling_params, PoolingParams)
|
||||||
|
assert metadata.default_sampling_params.task == deploy.stages[-1].default_pooling_params["task"]
|
||||||
|
|
||||||
|
engine_args = build_engine_args_dict_from_omni_stage_config(
|
||||||
|
stage,
|
||||||
|
stage.model_config.model or "/models/Qwen3-ForcedAligner-0.6B",
|
||||||
|
)
|
||||||
|
assert engine_args["runner"] == "pooling"
|
||||||
|
assert engine_args["pooling_output_decoder"] == POOLING_OUTPUT_DECODER_PATH
|
||||||
@@ -11,6 +11,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import pytest
|
import pytest
|
||||||
|
import torch
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from pydantic.fields import FieldInfo
|
from pydantic.fields import FieldInfo
|
||||||
from transformers import Qwen3OmniMoeConfig
|
from transformers import Qwen3OmniMoeConfig
|
||||||
@@ -123,6 +124,18 @@ def test_non_duplex_deploy_keeps_model_session_capacity_at_one(tmp_path: Path) -
|
|||||||
assert [stage.model_config.session_mode for stage in omni_config.stage_configs] == ["turn", "turn"]
|
assert [stage.model_config.session_mode for stage in omni_config.stage_configs] == ["turn", "turn"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_nested_stage_override_deep_merges_structured_model_config() -> None:
|
||||||
|
config = _from_pipeline_key(
|
||||||
|
"cosmos3_policy",
|
||||||
|
deploy_config_path=get_deploy_config_path("cosmos3_policy_droid.yaml"),
|
||||||
|
cli_overrides={"stage_0_model_config": {"guardrails": False}},
|
||||||
|
)
|
||||||
|
|
||||||
|
model_config = config.stage_by_id(0).diffusion_config.model_config
|
||||||
|
assert model_config["guardrails"] is False
|
||||||
|
assert model_config["policy_server_config"]["action_space"] == "joint_position"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))
|
@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))
|
||||||
def test_vllm_omni_config_from_pipeline_config_matches_merge_pipeline_deploy(model_type: str):
|
def test_vllm_omni_config_from_pipeline_config_matches_merge_pipeline_deploy(model_type: str):
|
||||||
pipeline = _resolve_pipeline_or_skip(model_type)
|
pipeline = _resolve_pipeline_or_skip(model_type)
|
||||||
@@ -654,6 +667,7 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
|
|||||||
"load_config",
|
"load_config",
|
||||||
"cache_config",
|
"cache_config",
|
||||||
"scheduler_config",
|
"scheduler_config",
|
||||||
|
"pooling_config",
|
||||||
"connector_config",
|
"connector_config",
|
||||||
"runtime_config",
|
"runtime_config",
|
||||||
"parallel_config",
|
"parallel_config",
|
||||||
@@ -889,6 +903,17 @@ def test_structured_llm_stage_registration_payloads_remain_msgpack_transport_saf
|
|||||||
msgspec.msgpack.encode(payload)
|
msgspec.msgpack.encode(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_diffusion_torch_dtype_is_msgpack_transport_safe():
|
||||||
|
stage_config = _from_pipeline_key(
|
||||||
|
"hunyuan_image3_dit",
|
||||||
|
cli_overrides={"dtype": torch.bfloat16},
|
||||||
|
).stage_by_id(0)
|
||||||
|
|
||||||
|
serialized = _serialize_stage_config(stage_config)
|
||||||
|
assert serialized["diffusion_config"]["dtype"] == "bfloat16"
|
||||||
|
msgspec.msgpack.encode(serialized)
|
||||||
|
|
||||||
|
|
||||||
def test_diffusion_parallel_config_fields_cover_legacy_surface():
|
def test_diffusion_parallel_config_fields_cover_legacy_surface():
|
||||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import torch.distributed as dist
|
|||||||
from transformers import AutoTokenizer
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig
|
||||||
from vllm_omni.diffusion.executor.uniproc_executor import UniProcDiffusionExecutor
|
from vllm_omni.diffusion.executor.uniproc_executor import UniProcDiffusionExecutor
|
||||||
from vllm_omni.diffusion.inline_stage_diffusion_client import InlineStageDiffusionClient
|
from vllm_omni.diffusion.inline_stage_diffusion_client import InlineStageDiffusionClient
|
||||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||||
@@ -136,6 +137,15 @@ async def _generate_once(
|
|||||||
|
|
||||||
def _assert_live_uniproc_worker(engine: AsyncOmni) -> None:
|
def _assert_live_uniproc_worker(engine: AsyncOmni) -> None:
|
||||||
"""Default single-GPU path must use the in-process executor, not mp IPC."""
|
"""Default single-GPU path must use the in-process executor, not mp IPC."""
|
||||||
|
stage_configs = engine.engine.stage_configs
|
||||||
|
assert len(stage_configs) == 1
|
||||||
|
stage_config = stage_configs[0]
|
||||||
|
assert isinstance(stage_config, VllmOmniDiffusionStageConfig)
|
||||||
|
assert stage_config.model_config.model == MODEL
|
||||||
|
assert stage_config.model_config.enforce_eager is True
|
||||||
|
assert stage_config.scheduler_config.max_num_seqs == 1
|
||||||
|
assert stage_config.diffusion_config.custom_pipeline_args == {"pipeline_class": CUSTOM_PIPELINE_CLASS}
|
||||||
|
assert stage_config.diffusion_config.worker_extension_cls == WORKER_EXTENSION_CLASS
|
||||||
clients = engine.engine.stage_clients
|
clients = engine.engine.stage_clients
|
||||||
assert len(clients) == 1
|
assert len(clients) == 1
|
||||||
client = clients[0]
|
client = clients[0]
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
# ruff: noqa: E501
|
# ruff: noqa: E501
|
||||||
|
import copy
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import os
|
||||||
import sys
|
import sys
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -10,20 +15,37 @@ import torch.nn.functional as F
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from transformers import CLIPModel, CLIPProcessor
|
from transformers import CLIPModel, CLIPProcessor
|
||||||
|
|
||||||
|
from tests.helpers.mark import hardware_test
|
||||||
from tests.helpers.runtime import OmniRunner
|
from tests.helpers.runtime import OmniRunner
|
||||||
from tests.helpers.stage_config import get_deploy_config_path
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
from vllm_omni import Omni
|
from vllm_omni import Omni
|
||||||
|
from vllm_omni.config.omni_config import (
|
||||||
|
VllmOmniARStageConfig,
|
||||||
|
VllmOmniDiffusionStageConfig,
|
||||||
|
)
|
||||||
|
from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import build_prompt_tokens, resolve_stop_token_ids
|
||||||
from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
|
from vllm_omni.entrypoints.openai.stage_params import clone_sampling_params
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
from vllm_omni.model_extras import get_ar_input_builder, get_ar_tokenizer_validator, get_model_class_name
|
from vllm_omni.model_extras import get_ar_input_builder, get_ar_tokenizer_validator, get_model_class_name
|
||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
PROMPT = "A brown and white dog is running on the grass"
|
PROMPT = "A brown and white dog is running on the grass"
|
||||||
MODEL_NAME = "tencent/HunyuanImage-3.0"
|
MODEL_NAME = os.environ.get("HUNYUAN_IMAGE3_MODEL", "tencent/HunyuanImage-3.0-Instruct")
|
||||||
LOCAL_CLIP_PATH = "openai/clip-vit-base-patch32"
|
LOCAL_CLIP_PATH = "openai/clip-vit-base-patch32"
|
||||||
DEPLOY_CONFIG_PATH = get_deploy_config_path("hunyuan_image_3_moe.yaml")
|
_PRODUCTION_DEPLOY_CONFIG_PATH = get_deploy_config_path("hunyuan_image_3_moe.yaml")
|
||||||
|
# Keep the production Mooncake connector, but use its TCP transport for this
|
||||||
|
# single-host test. The shared server's RDMA device is not usable reliably;
|
||||||
|
# TCP still exercises the Mooncake AR→DiT data plane without changing deploy
|
||||||
|
# semantics in production.
|
||||||
|
DEPLOY_CONFIG_PATH = modify_stage_config(
|
||||||
|
_PRODUCTION_DEPLOY_CONFIG_PATH,
|
||||||
|
updates={"connectors.rdma_connector.extra.protocol": "tcp"},
|
||||||
|
)
|
||||||
|
|
||||||
pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion]
|
pytestmark = [
|
||||||
|
pytest.mark.advanced_model,
|
||||||
|
pytest.mark.diffusion,
|
||||||
|
]
|
||||||
|
|
||||||
# System prompt type. Options: None, dynamic, en_vanilla, en_recaption, en_think_recaption, en_unified
|
# System prompt type. Options: None, dynamic, en_vanilla, en_recaption, en_think_recaption, en_unified
|
||||||
# Below are the CLIP embedding tensors from the official HunyuanImage model (seed=1234, prompt: "A brown and white dog is running on the grass").
|
# Below are the CLIP embedding tensors from the official HunyuanImage model (seed=1234, prompt: "A brown and white dog is running on the grass").
|
||||||
@@ -285,25 +307,97 @@ def omni() -> Generator[Omni, None, None]:
|
|||||||
yield runner.omni
|
yield runner.omni
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(torch.accelerator.device_count() < 4, reason="Need at least 4 CUDA GPUs for this test.")
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||||
|
def test_structured_mixed_pipeline_config_reaches_runtime(omni: Omni) -> None:
|
||||||
|
"""Mixed-pipeline deploy settings reach both terminal engine configs."""
|
||||||
|
engine = omni.engine
|
||||||
|
|
||||||
|
# Topology, connector selection, and placement remain Omni-owned.
|
||||||
|
resolved_stages = engine.stage_configs
|
||||||
|
assert len(resolved_stages) == 2
|
||||||
|
ar_stage, diffusion_stage = resolved_stages
|
||||||
|
assert isinstance(ar_stage, VllmOmniARStageConfig)
|
||||||
|
assert ar_stage.stage_id == 0
|
||||||
|
assert ar_stage.model_stage == "AR"
|
||||||
|
assert ar_stage.final_output_type == "text"
|
||||||
|
assert ar_stage.runtime_config.devices == "0,1"
|
||||||
|
assert ar_stage.connector_config.output_connectors == {"to_stage_1": "rdma_connector"}
|
||||||
|
assert isinstance(diffusion_stage, VllmOmniDiffusionStageConfig)
|
||||||
|
assert diffusion_stage.stage_id == 1
|
||||||
|
assert diffusion_stage.model_stage == "dit"
|
||||||
|
assert diffusion_stage.final_output_type == "image"
|
||||||
|
assert diffusion_stage.input_sources == [0]
|
||||||
|
assert diffusion_stage.runtime_config.devices == "2,3"
|
||||||
|
assert diffusion_stage.connector_config.input_connectors == {"from_stage_0": "rdma_connector"}
|
||||||
|
|
||||||
|
# Stage 0 is materialized as the VllmConfig actually passed to its engine.
|
||||||
|
ar_vllm_config, diffusion_vllm_config = engine.stage_vllm_configs
|
||||||
|
assert ar_vllm_config is not None
|
||||||
|
assert diffusion_vllm_config is None
|
||||||
|
assert ar_vllm_config.model_config.enforce_eager is True
|
||||||
|
assert ar_vllm_config.model_config.trust_remote_code is True
|
||||||
|
assert ar_vllm_config.parallel_config.tensor_parallel_size == 2
|
||||||
|
assert ar_vllm_config.scheduler_config.max_num_seqs == 1
|
||||||
|
assert ar_vllm_config.scheduler_config.max_num_batched_tokens == 32768
|
||||||
|
assert ar_vllm_config.cache_config.gpu_memory_utilization == 0.95
|
||||||
|
ar_omni_config = ar_vllm_config.model_config.omni_config
|
||||||
|
assert ar_omni_config.omni_kv_config["need_send_cache"] is True
|
||||||
|
|
||||||
|
expected_ar_sampling = {
|
||||||
|
"temperature": 0.0,
|
||||||
|
"top_p": 1,
|
||||||
|
"top_k": -1,
|
||||||
|
"max_tokens": 8192,
|
||||||
|
"detokenize": True,
|
||||||
|
"skip_special_tokens": False,
|
||||||
|
"include_stop_str_in_output": True,
|
||||||
|
}
|
||||||
|
ar_sampling, diffusion_sampling = engine.default_sampling_params_list
|
||||||
|
assert all(getattr(ar_sampling, name) == value for name, value in expected_ar_sampling.items())
|
||||||
|
assert diffusion_sampling.num_inference_steps == 50
|
||||||
|
assert diffusion_sampling.guidance_scale == 0
|
||||||
|
|
||||||
|
# Stage 1 is a diffusion engine, so its terminal owner is the launched
|
||||||
|
# OmniDiffusionConfig rather than a VllmConfig.
|
||||||
|
od_config = omni.get_diffusion_od_config()
|
||||||
|
assert od_config is not None
|
||||||
|
assert od_config.model == MODEL_NAME
|
||||||
|
assert od_config.enforce_eager is True
|
||||||
|
assert od_config.distributed_executor_backend == "mp"
|
||||||
|
assert od_config.parallel_config.tensor_parallel_size == 2
|
||||||
|
assert od_config.parallel_config.enable_expert_parallel is True
|
||||||
|
assert od_config.max_num_seqs == 1
|
||||||
|
diffusion_kv = od_config.omni_kv_config
|
||||||
|
assert diffusion_kv["need_recv_cache"] is True
|
||||||
|
assert diffusion_kv["enable_kv_async_prefetch"] is True
|
||||||
|
assert diffusion_kv["kv_prefetch_min_free_mem_ratio"] == 0.2
|
||||||
|
|
||||||
|
|
||||||
def _extract_generated_image(outputs: list[object]) -> Image.Image:
|
def _extract_generated_image(outputs: list[object]) -> Image.Image:
|
||||||
if not outputs:
|
if not outputs:
|
||||||
raise AssertionError("No outputs were returned from Omni.generate()")
|
raise AssertionError("No outputs were returned from Omni.generate()")
|
||||||
|
|
||||||
first_output = outputs[0]
|
for output in outputs:
|
||||||
if images := getattr(first_output, "images", None):
|
if images := getattr(output, "images", None):
|
||||||
return images[0]
|
return images[0]
|
||||||
|
multimodal_output = getattr(output, "multimodal_output", None)
|
||||||
request_output = first_output
|
if isinstance(multimodal_output, dict) and (images := multimodal_output.get("images")):
|
||||||
if request_output is not None and (images := getattr(request_output, "images", None)):
|
return images[0]
|
||||||
return images[0]
|
|
||||||
|
|
||||||
raise AssertionError("No generated image found in Omni output")
|
raise AssertionError("No generated image found in Omni output")
|
||||||
|
|
||||||
|
|
||||||
def extract_embedding(image: Image.Image, clip_model: CLIPModel, clip_processor: CLIPProcessor) -> torch.Tensor:
|
def extract_embedding(image: Image.Image, clip_model: CLIPModel, clip_processor: CLIPProcessor) -> torch.Tensor:
|
||||||
inputs = clip_processor(images=image.convert("RGB"), return_tensors="pt")
|
# The full CLIP forward API is stable across supported Transformers
|
||||||
|
# versions, but it expects text inputs as well as pixels. An empty text
|
||||||
|
# prompt is sufficient when only the projected image embedding is needed.
|
||||||
|
inputs = clip_processor(text=[""], images=[image.convert("RGB")], return_tensors="pt", padding=True)
|
||||||
with torch.inference_mode():
|
with torch.inference_mode():
|
||||||
features = clip_model.get_image_features(**inputs)
|
# Use the public CLIP forward output, matching the shared CLIPScorer
|
||||||
|
# helper. ``get_image_features`` changed from Tensor to
|
||||||
|
# BaseModelOutputWithPooling in Transformers 5.x.
|
||||||
|
features = clip_model(**inputs).image_embeds
|
||||||
features = F.normalize(features, p=2, dim=-1)
|
features = F.normalize(features, p=2, dim=-1)
|
||||||
return features.squeeze(0)
|
return features.squeeze(0)
|
||||||
|
|
||||||
@@ -320,16 +414,36 @@ def compare_semantic(
|
|||||||
|
|
||||||
|
|
||||||
def _generate_image(omni: Omni, use_system_prompt: str | None) -> Image.Image:
|
def _generate_image(omni: Omni, use_system_prompt: str | None) -> Image.Image:
|
||||||
generator_device = current_omni_platform.device_type or "cuda"
|
from transformers import AutoTokenizer
|
||||||
sampling_params = OmniDiffusionSamplingParams(
|
|
||||||
seed=1234,
|
|
||||||
generator=torch.Generator(device=generator_device).manual_seed(1234),
|
|
||||||
num_outputs_per_prompt=1,
|
|
||||||
)
|
|
||||||
if use_system_prompt is not None:
|
|
||||||
sampling_params.extra_args = {"use_system_prompt": use_system_prompt}
|
|
||||||
|
|
||||||
outputs = omni.generate({"prompt": PROMPT}, sampling_params)
|
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||||
|
prompt_tokens = build_prompt_tokens(
|
||||||
|
PROMPT,
|
||||||
|
tokenizer,
|
||||||
|
task="t2i",
|
||||||
|
bot_task="think",
|
||||||
|
sys_type=use_system_prompt or "None",
|
||||||
|
)
|
||||||
|
stop_token_ids = resolve_stop_token_ids(task="t2i", bot_task="think", tokenizer=tokenizer, image_size="1024x1024")
|
||||||
|
params_list = copy.deepcopy(omni.default_sampling_params_list)
|
||||||
|
generator_device = current_omni_platform.device_type or "cuda"
|
||||||
|
for params in params_list:
|
||||||
|
if isinstance(params, OmniDiffusionSamplingParams):
|
||||||
|
params.seed = 1234
|
||||||
|
params.generator = torch.Generator(device=generator_device).manual_seed(1234)
|
||||||
|
params.num_outputs_per_prompt = 1
|
||||||
|
elif hasattr(params, "stop_token_ids"):
|
||||||
|
params.stop_token_ids = stop_token_ids
|
||||||
|
|
||||||
|
prompt = {
|
||||||
|
"prompt": PROMPT,
|
||||||
|
"prompt_token_ids": prompt_tokens.token_ids,
|
||||||
|
"use_system_prompt": prompt_tokens.system_prompt_type,
|
||||||
|
"modalities": ["image"],
|
||||||
|
"height": 1024,
|
||||||
|
"width": 1024,
|
||||||
|
}
|
||||||
|
outputs = omni.generate(prompt, params_list)
|
||||||
return _extract_generated_image(outputs)
|
return _extract_generated_image(outputs)
|
||||||
|
|
||||||
|
|
||||||
@@ -342,7 +456,7 @@ def _generate_image(omni: Omni, use_system_prompt: str | None) -> Image.Image:
|
|||||||
MIN_SEMANTIC_SIMILARITY = 0.5
|
MIN_SEMANTIC_SIMILARITY = 0.5
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(torch.accelerator.device_count() < 8, reason="Need at least 8 CUDA GPUs for this test.")
|
@pytest.mark.skipif(torch.accelerator.device_count() < 4, reason="Need at least 4 CUDA GPUs for this test.")
|
||||||
@pytest.mark.parametrize("system_prompt_name,use_system_prompt,expected_embedding", SYSTEM_PROMPT_CASES)
|
@pytest.mark.parametrize("system_prompt_name,use_system_prompt,expected_embedding", SYSTEM_PROMPT_CASES)
|
||||||
def test_system_prompt_scores(
|
def test_system_prompt_scores(
|
||||||
omni: Omni,
|
omni: Omni,
|
||||||
@@ -378,7 +492,7 @@ def _load_text_to_image_module():
|
|||||||
return module
|
return module
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(torch.accelerator.device_count() < 8, reason="Need at least 8 CUDA GPUs for this test.")
|
@pytest.mark.skipif(torch.accelerator.device_count() < 4, reason="Need at least 4 CUDA GPUs for this test.")
|
||||||
def test_shared_script_ar_path_reaches_generation(
|
def test_shared_script_ar_path_reaches_generation(
|
||||||
omni: Omni,
|
omni: Omni,
|
||||||
clip_bundle: tuple[CLIPModel, CLIPProcessor],
|
clip_bundle: tuple[CLIPModel, CLIPProcessor],
|
||||||
|
|||||||
@@ -17,6 +17,10 @@ import pytest
|
|||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from tests.helpers.media import generate_synthetic_video
|
from tests.helpers.media import generate_synthetic_video
|
||||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||||
|
from vllm_omni.config.omni_config import (
|
||||||
|
VllmOmniARStageConfig,
|
||||||
|
VllmOmniGenerationStageConfig,
|
||||||
|
)
|
||||||
from vllm_omni.platforms import current_omni_platform
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||||
@@ -25,6 +29,7 @@ thinker_only_models = ["Qwen/Qwen3-Omni-30B-A3B-Captioner"]
|
|||||||
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
# Single CI deploy YAML; rocm/xpu deltas are picked automatically via the
|
||||||
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
|
# platforms: section. Only CUDA needs an extra enforce_eager tweak.
|
||||||
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
_CI_DEPLOY = get_deploy_config_path("ci/qwen3_omni_moe.yaml")
|
||||||
|
_PRODUCTION_DEPLOY = get_deploy_config_path("qwen3_omni_moe.yaml")
|
||||||
|
|
||||||
|
|
||||||
def get_cuda_graph_config():
|
def get_cuda_graph_config():
|
||||||
@@ -46,6 +51,7 @@ else:
|
|||||||
|
|
||||||
# Create parameter combinations for model and stage config
|
# Create parameter combinations for model and stage config
|
||||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||||
|
production_test_params = [(model, _PRODUCTION_DEPLOY) for model in models]
|
||||||
# we can use the same config for a model that only has thinker (i.e., does not
|
# we can use the same config for a model that only has thinker (i.e., does not
|
||||||
# enable audio output) because the resolver should figure out that it doesn't
|
# enable audio output) because the resolver should figure out that it doesn't
|
||||||
# need the full pipeline based on the HF config.
|
# need the full pipeline based on the HF config.
|
||||||
@@ -59,6 +65,85 @@ def get_question(prompt_type="video"):
|
|||||||
return prompts.get(prompt_type, prompts["video"])
|
return prompts.get(prompt_type, prompts["video"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.omni
|
||||||
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
|
@pytest.mark.parametrize("omni_runner", production_test_params, indirect=True)
|
||||||
|
def test_structured_multistage_config_reaches_runtime(omni_runner, offline_client) -> None:
|
||||||
|
"""Deploy settings reach the materialized configs used by live stages."""
|
||||||
|
engine = omni_runner.omni.engine
|
||||||
|
|
||||||
|
# The resolved typed stages remain the Omni-owned source for topology and
|
||||||
|
# placement, which are not fields of the terminal vLLM config.
|
||||||
|
resolved_stages = engine.stage_configs
|
||||||
|
assert len(resolved_stages) == 3
|
||||||
|
thinker, talker, code2wav = resolved_stages
|
||||||
|
assert isinstance(thinker, VllmOmniARStageConfig)
|
||||||
|
assert isinstance(talker, VllmOmniARStageConfig)
|
||||||
|
assert isinstance(code2wav, VllmOmniGenerationStageConfig)
|
||||||
|
assert [stage.stage_id for stage in resolved_stages] == [0, 1, 2]
|
||||||
|
assert [stage.model_stage for stage in resolved_stages] == ["thinker", "talker", "code2wav"]
|
||||||
|
assert code2wav.final_output_type == "audio"
|
||||||
|
assert thinker.runtime_config.devices == "0"
|
||||||
|
assert talker.runtime_config.devices == code2wav.runtime_config.devices == "1"
|
||||||
|
assert talker.connector_config.input_connectors == {"from_stage_0": "connector_of_shared_memory"}
|
||||||
|
assert code2wav.connector_config.input_connectors == {"from_stage_1": "connector_of_shared_memory"}
|
||||||
|
|
||||||
|
# Engine-owned deploy settings must survive the startup projection into
|
||||||
|
# the actual VllmConfig objects retained by the stage pools.
|
||||||
|
assert len(engine.stage_vllm_configs) == 3
|
||||||
|
thinker_vllm, talker_vllm, code2wav_vllm = engine.stage_vllm_configs
|
||||||
|
assert thinker_vllm is not None
|
||||||
|
assert talker_vllm is not None
|
||||||
|
assert code2wav_vllm is not None
|
||||||
|
assert thinker_vllm.scheduler_config.max_num_seqs == 64
|
||||||
|
assert talker_vllm.scheduler_config.max_num_seqs == 64
|
||||||
|
assert code2wav_vllm.scheduler_config.max_num_seqs == 64
|
||||||
|
assert thinker_vllm.scheduler_config.max_num_batched_tokens == 32768
|
||||||
|
assert talker_vllm.scheduler_config.max_num_batched_tokens == 32768
|
||||||
|
assert code2wav_vllm.scheduler_config.max_num_batched_tokens == 65536
|
||||||
|
assert thinker_vllm.cache_config.gpu_memory_utilization == 0.9
|
||||||
|
assert talker_vllm.cache_config.gpu_memory_utilization == 0.6
|
||||||
|
assert code2wav_vllm.cache_config.gpu_memory_utilization == 0.1
|
||||||
|
assert all(config.cache_config.enable_prefix_caching is False for config in engine.stage_vllm_configs)
|
||||||
|
assert all(config.model_config.trust_remote_code is True for config in engine.stage_vllm_configs)
|
||||||
|
assert code2wav_vllm.scheduler_config.enable_chunked_prefill is False
|
||||||
|
assert code2wav_vllm.scheduler_config.async_scheduling is False
|
||||||
|
|
||||||
|
# Sampling defaults are consumed from StageClient metadata, rather than
|
||||||
|
# from either the resolver output or VllmConfig.
|
||||||
|
expected_sampling = (
|
||||||
|
{"temperature": 0.0, "max_tokens": 2048},
|
||||||
|
{"temperature": 0.9, "top_k": 50, "max_tokens": 4096, "repetition_penalty": 1.05},
|
||||||
|
{
|
||||||
|
"temperature": 0.0,
|
||||||
|
"top_p": 1.0,
|
||||||
|
"top_k": -1,
|
||||||
|
"max_tokens": 65536,
|
||||||
|
"repetition_penalty": 1.1,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert len(engine.default_sampling_params_list) == len(expected_sampling)
|
||||||
|
for runtime_params, expected in zip(engine.default_sampling_params_list, expected_sampling, strict=True):
|
||||||
|
# SamplingParams normalization may add backend defaults (for example
|
||||||
|
# ``detokenize=True``). Verify deploy-owned values without rejecting
|
||||||
|
# those non-lossy normalized fields.
|
||||||
|
assert all(getattr(runtime_params, name) == value for name, value in expected.items())
|
||||||
|
|
||||||
|
if current_omni_platform.is_cuda():
|
||||||
|
assert thinker_vllm.model_config.enforce_eager is False
|
||||||
|
assert talker_vllm.model_config.enforce_eager is False
|
||||||
|
|
||||||
|
# Exercise the materialized stages and their connector path with a real
|
||||||
|
# request; the assertions above are not merely resolver-shape checks.
|
||||||
|
offline_client.send_omni_request(
|
||||||
|
{
|
||||||
|
"prompts": "Answer with one short sentence: what is the capital of China?",
|
||||||
|
"modalities": ["text"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.omni
|
@pytest.mark.omni
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import pytest
|
|||||||
|
|
||||||
from tests.helpers.mark import hardware_test
|
from tests.helpers.mark import hardware_test
|
||||||
from tests.helpers.runtime import OfflineOmniClient
|
from tests.helpers.runtime import OfflineOmniClient
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
|
|
||||||
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||||
@@ -12,6 +13,31 @@ PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
|
|||||||
NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
|
NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.advanced_model
|
||||||
|
@pytest.mark.diffusion
|
||||||
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 1})
|
||||||
|
@pytest.mark.parametrize("omni_runner", [(MODEL, None)], indirect=True)
|
||||||
|
def test_structured_diffusion_config_reaches_runtime(omni_runner) -> None:
|
||||||
|
"""The default WAN resolver launches the typed diffusion config itself."""
|
||||||
|
omni = omni_runner.omni
|
||||||
|
resolved_stages = omni.engine.stage_configs
|
||||||
|
assert len(resolved_stages) == 1
|
||||||
|
stage_config = resolved_stages[0]
|
||||||
|
|
||||||
|
assert isinstance(stage_config, VllmOmniDiffusionStageConfig)
|
||||||
|
assert stage_config.stage_id == 0
|
||||||
|
assert stage_config.model_stage == "dit"
|
||||||
|
assert stage_config.final_output_type == "video"
|
||||||
|
|
||||||
|
assert omni.engine.stage_vllm_configs == [None]
|
||||||
|
od_config = omni.get_diffusion_od_config()
|
||||||
|
assert od_config is not None
|
||||||
|
assert od_config.stage_id == 0
|
||||||
|
assert od_config.model == MODEL
|
||||||
|
assert od_config.model_class_name == "WanPipeline"
|
||||||
|
assert od_config.output_type == "pil"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.advanced_model
|
@pytest.mark.advanced_model
|
||||||
@pytest.mark.diffusion
|
@pytest.mark.diffusion
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 1})
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 1})
|
||||||
|
|||||||
@@ -66,9 +66,10 @@ def test_abort_async_waits_for_ack(mocker: MockerFixture):
|
|||||||
assert msg.rpc_id == "abort-rpc-1"
|
assert msg.rpc_id == "abort-rpc-1"
|
||||||
assert not task.done()
|
assert not task.done()
|
||||||
|
|
||||||
rpc_q.put(AbortResultMessage(rpc_id="abort-rpc-1", success=True))
|
partial_output = SimpleNamespace(request_id="req-a", finished=True)
|
||||||
|
rpc_q.put(AbortResultMessage(rpc_id="abort-rpc-1", success=True, abort_outputs=[partial_output]))
|
||||||
result = await task
|
result = await task
|
||||||
assert result == []
|
assert result == [partial_output]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
asyncio.run(_run())
|
asyncio.run(_run())
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import types
|
|||||||
import pytest
|
import pytest
|
||||||
from omegaconf import OmegaConf
|
from omegaconf import OmegaConf
|
||||||
|
|
||||||
|
from vllm_omni.config.omni_config import OmniStageRuntimeConfig
|
||||||
from vllm_omni.diffusion.data import AttentionConfig
|
from vllm_omni.diffusion.data import AttentionConfig
|
||||||
from vllm_omni.engine import omni_engine_base as async_omni_engine_module
|
from vllm_omni.engine import omni_engine_base as async_omni_engine_module
|
||||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||||
@@ -21,12 +22,23 @@ from vllm_omni.engine.stage_init_utils import (
|
|||||||
build_stage0_input_processor,
|
build_stage0_input_processor,
|
||||||
compute_replica_layout,
|
compute_replica_layout,
|
||||||
split_devices_for_replicas,
|
split_devices_for_replicas,
|
||||||
|
stage_runtime_env,
|
||||||
)
|
)
|
||||||
from vllm_omni.engine.stage_runtime import StageRuntime
|
from vllm_omni.engine.stage_runtime import StageRuntime
|
||||||
|
|
||||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stage_runtime_env_accepts_typed_runtime_config(monkeypatch):
|
||||||
|
env_key = "VLLM_OMNI_TEST_TYPED_STAGE_ENV"
|
||||||
|
monkeypatch.delenv(env_key, raising=False)
|
||||||
|
|
||||||
|
with stage_runtime_env(0, OmniStageRuntimeConfig(env={env_key: "typed-value"})):
|
||||||
|
assert os.environ[env_key] == "typed-value"
|
||||||
|
|
||||||
|
assert env_key not in os.environ
|
||||||
|
|
||||||
|
|
||||||
def test_orchestrator_startup_timeout_warns_how_to_raise_limits(monkeypatch):
|
def test_orchestrator_startup_timeout_warns_how_to_raise_limits(monkeypatch):
|
||||||
engine = object.__new__(AsyncOmniEngine)
|
engine = object.__new__(AsyncOmniEngine)
|
||||||
engine.orchestrator_thread = types.SimpleNamespace(is_alive=lambda: True)
|
engine.orchestrator_thread = types.SimpleNamespace(is_alive=lambda: True)
|
||||||
@@ -223,6 +235,50 @@ def test_async_omni_engine_initialize_stages_passes_log_stats_to_runtime(monkeyp
|
|||||||
assert captured["log_stats"] is True
|
assert captured["log_stats"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_omni_engine_initialize_stages_retains_stage0_prompt_transform(monkeypatch):
|
||||||
|
import vllm_omni.engine.omni_engine_base as engine_mod
|
||||||
|
|
||||||
|
engine = object.__new__(AsyncOmniEngine)
|
||||||
|
engine.stage_configs = [types.SimpleNamespace()]
|
||||||
|
engine.model = "dummy-model"
|
||||||
|
engine.config_path = "dummy-config"
|
||||||
|
engine.single_stage_mode = False
|
||||||
|
engine.async_chunk = False
|
||||||
|
engine.tokenizer = None
|
||||||
|
engine._single_stage_id_filter = None
|
||||||
|
engine._omni_master_address = None
|
||||||
|
engine._omni_master_port = None
|
||||||
|
engine._omni_dp_size_local = 1
|
||||||
|
engine._omni_heartbeat_timeout = 30.0
|
||||||
|
engine._omni_lb_policy = "random"
|
||||||
|
engine.request_queue = types.SimpleNamespace()
|
||||||
|
engine._log_stats = False
|
||||||
|
engine._parallel_stage_init = False
|
||||||
|
|
||||||
|
prompt_transform = object()
|
||||||
|
client = types.SimpleNamespace(
|
||||||
|
prompt_transform_func=prompt_transform,
|
||||||
|
prompt_expand_func=None,
|
||||||
|
default_sampling_params=types.SimpleNamespace(),
|
||||||
|
final_output=True,
|
||||||
|
final_output_type="text",
|
||||||
|
stage_type="llm",
|
||||||
|
model_stage="text_encoder",
|
||||||
|
is_comprehension=True,
|
||||||
|
)
|
||||||
|
pool = types.SimpleNamespace(
|
||||||
|
stage_client=client,
|
||||||
|
stage_vllm_config=None,
|
||||||
|
output_processor=None,
|
||||||
|
)
|
||||||
|
runtime = types.SimpleNamespace(stage_pools=[pool], initialize=lambda: None)
|
||||||
|
monkeypatch.setattr(engine_mod, "create_stage_runtime", lambda **_kwargs: runtime)
|
||||||
|
|
||||||
|
engine._initialize_stages(stage_init_timeout=7)
|
||||||
|
|
||||||
|
assert engine.prompt_transform_func is prompt_transform
|
||||||
|
|
||||||
|
|
||||||
def test_compute_replica_layout_splits_diffusion_devices_by_world_size():
|
def test_compute_replica_layout_splits_diffusion_devices_by_world_size():
|
||||||
stage_cfg = types.SimpleNamespace(
|
stage_cfg = types.SimpleNamespace(
|
||||||
stage_id=0,
|
stage_id=0,
|
||||||
@@ -1715,6 +1771,51 @@ def test_inject_kv_stage_info_infers_receiver_tp_topology():
|
|||||||
assert stage1.engine_args["omni_kv_config"]["rank_mapping"] == {"from_tp": 4, "to_tp": 2}
|
assert stage1.engine_args["omni_kv_config"]["rank_mapping"] == {"from_tp": 4, "to_tp": 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_inject_kv_stage_info_updates_typed_connector_config():
|
||||||
|
from vllm_omni.config.omni_config import (
|
||||||
|
OmniStageConnectorConfig,
|
||||||
|
OmniStageDiffusionParallelConfig,
|
||||||
|
VllmOmniDiffusionStageConfig,
|
||||||
|
)
|
||||||
|
from vllm_omni.config.stage_config import StageExecutionType, StagePipelineConfig
|
||||||
|
from vllm_omni.engine.stage_init_utils import inject_kv_stage_info
|
||||||
|
from vllm_omni.entrypoints.utils import inject_omni_kv_config
|
||||||
|
|
||||||
|
stage0 = VllmOmniDiffusionStageConfig(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(
|
||||||
|
stage_id=0,
|
||||||
|
model_stage="diffusion",
|
||||||
|
execution_type=StageExecutionType.DIFFUSION,
|
||||||
|
),
|
||||||
|
connector_config=OmniStageConnectorConfig(
|
||||||
|
omni_kv_config={
|
||||||
|
"need_send_cache": True,
|
||||||
|
"omni_from_stage": "0",
|
||||||
|
"omni_to_stage": "1",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
parallel_config=OmniStageDiffusionParallelConfig(tensor_parallel_size=4),
|
||||||
|
)
|
||||||
|
stage1 = VllmOmniDiffusionStageConfig(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(
|
||||||
|
stage_id=1,
|
||||||
|
model_stage="diffusion",
|
||||||
|
execution_type=StageExecutionType.DIFFUSION,
|
||||||
|
input_sources=(0,),
|
||||||
|
),
|
||||||
|
connector_config=OmniStageConnectorConfig(omni_kv_config={"need_recv_cache": True}),
|
||||||
|
parallel_config=OmniStageDiffusionParallelConfig(tensor_parallel_size=2),
|
||||||
|
)
|
||||||
|
|
||||||
|
inject_omni_kv_config(stage0, {"kv_connector": "P2pNcclConnector"}, "0", "1")
|
||||||
|
inject_kv_stage_info(stage0, 0, [stage0, stage1])
|
||||||
|
|
||||||
|
assert stage0.connector_config.omni_kv_config["stage_id"] == 0
|
||||||
|
assert stage0.connector_config.omni_kv_config["connector_config"] == {"kv_connector": "P2pNcclConnector"}
|
||||||
|
assert stage0.connector_config.omni_kv_config["engine_input_source"] == []
|
||||||
|
assert stage0.connector_config.omni_kv_config["rank_mapping"] == {"from_tp": 4, "to_tp": 2}
|
||||||
|
|
||||||
|
|
||||||
def test_extract_legacy_stage_metadata_rocm_does_not_inject_diffusion_attention(monkeypatch):
|
def test_extract_legacy_stage_metadata_rocm_does_not_inject_diffusion_attention(monkeypatch):
|
||||||
"""ROCm default attention logic only applies to LLM stages, not diffusion."""
|
"""ROCm default attention logic only applies to LLM stages, not diffusion."""
|
||||||
from vllm_omni.engine.stage_init_utils import extract_legacy_stage_metadata
|
from vllm_omni.engine.stage_init_utils import extract_legacy_stage_metadata
|
||||||
@@ -1852,3 +1953,80 @@ def test_engine_async_chunk_includes_downstream_stages(monkeypatch, stage_modes,
|
|||||||
with pytest.raises(ConfigResolvedError):
|
with pytest.raises(ConfigResolvedError):
|
||||||
engine.__init__("test-model")
|
engine.__init__("test-model")
|
||||||
assert engine.async_chunk is expected
|
assert engine.async_chunk is expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||||
|
def test_dist_stage_runtime_applies_local_dp_to_stage_config(typed):
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniARStageConfig
|
||||||
|
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||||
|
from vllm_omni.engine.stage_runtime import DistStageRuntime
|
||||||
|
|
||||||
|
if typed:
|
||||||
|
stage = VllmOmniARStageConfig(stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="ar"))
|
||||||
|
runtime_cfg = stage.runtime_config
|
||||||
|
else:
|
||||||
|
runtime_cfg = types.SimpleNamespace(num_replicas=1)
|
||||||
|
stage = types.SimpleNamespace(stage_id=0, runtime=runtime_cfg)
|
||||||
|
runtime = DistStageRuntime(
|
||||||
|
stage_configs=[stage],
|
||||||
|
model="dummy-model",
|
||||||
|
config_path="dummy-config",
|
||||||
|
stage_init_timeout=1,
|
||||||
|
async_chunk=False,
|
||||||
|
single_stage_id_filter=0,
|
||||||
|
omni_master_address="127.0.0.1",
|
||||||
|
omni_master_port=12345,
|
||||||
|
omni_dp_size_local=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
runtime._validate_single_stage_mode_replica_constraints()
|
||||||
|
|
||||||
|
assert runtime_cfg.num_replicas == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("num_replicas", [1, 2, 3])
|
||||||
|
def test_typed_diffusion_replicas_share_one_config_between_planning_and_launch(mocker, num_replicas):
|
||||||
|
from vllm_omni.config.config_factory import StageConfigFactory
|
||||||
|
from vllm_omni.engine import stage_runtime as runtime_module
|
||||||
|
|
||||||
|
stage = StageConfigFactory.create_typed_default_diffusion(
|
||||||
|
"generic-diffusion", {"model_class_name": "QwenImagePipeline"}
|
||||||
|
).stage_configs[0]
|
||||||
|
devices = ",".join(str(i) for i in range(num_replicas))
|
||||||
|
stage.runtime_config.devices = devices
|
||||||
|
stage.runtime_config.num_replicas = num_replicas
|
||||||
|
runtime = StageRuntime(
|
||||||
|
stage_configs=[stage],
|
||||||
|
model="dummy-model",
|
||||||
|
config_path="dummy-config",
|
||||||
|
stage_init_timeout=1,
|
||||||
|
async_chunk=False,
|
||||||
|
)
|
||||||
|
mocker.patch.object(runtime_module, "get_stage_connector_spec", return_value={})
|
||||||
|
mocker.patch.object(runtime_module, "resolve_omni_kv_config_for_stage", return_value=(None, None, None))
|
||||||
|
mocker.patch.object(runtime, "_stage_device_scope", side_effect=lambda *_: contextlib.nullcontext())
|
||||||
|
client = mocker.Mock()
|
||||||
|
launch = mocker.patch.object(runtime_module, "launch_diffusion_stage_replica", return_value=(client, None))
|
||||||
|
|
||||||
|
# Replanning must not inherit the first replica's narrowed device slice.
|
||||||
|
for _ in range(2):
|
||||||
|
counts, device_map = compute_replica_layout([stage])
|
||||||
|
plans = runtime._build_logical_stage_init_plans(
|
||||||
|
omni_transfer_config=None,
|
||||||
|
replicas_per_stage=counts,
|
||||||
|
replica_devices_map=device_map,
|
||||||
|
)
|
||||||
|
replicas = plans[0].replicas
|
||||||
|
assert stage.runtime_config.devices == devices
|
||||||
|
assert len({id(plan.stage_cfg) for plan in replicas}) == num_replicas
|
||||||
|
for i, plan in enumerate(replicas):
|
||||||
|
assert plan.metadata.runtime_cfg is plan.stage_cfg.runtime_config
|
||||||
|
assert plan.stage_cfg.runtime_config.devices == str(i)
|
||||||
|
if num_replicas > 1:
|
||||||
|
assert plan.stage_cfg is not stage
|
||||||
|
assert runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=1) is client
|
||||||
|
assert launch.call_args.kwargs["stage_config"] is plan.stage_cfg
|
||||||
|
assert launch.call_args.kwargs["metadata"] is plan.metadata
|
||||||
|
assert launch.call_args.kwargs["use_inline"] is (num_replicas == 1)
|
||||||
|
|
||||||
|
assert launch.call_count == 2 * num_replicas
|
||||||
@@ -306,6 +306,23 @@ def test_check_admission_admits_diffusion_with_explicit_util():
|
|||||||
assert ledgers[0].kv_budget_bytes == int(cap * 0.3)
|
assert ledgers[0].kv_budget_bytes == int(cap * 0.3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_admission_reads_typed_diffusion_cache_config():
|
||||||
|
cap = 80 * 1024**3
|
||||||
|
diff = _local_diffusion_replica(0, 0, "0", util=None)
|
||||||
|
diff.stage_cfg.cache_config = types.SimpleNamespace(gpu_memory_utilization=0.35)
|
||||||
|
plan = LogicalStageInitPlan(stage_idx=0, stage_id=0, replicas=[diff])
|
||||||
|
|
||||||
|
ledgers = check_admission(
|
||||||
|
[plan],
|
||||||
|
resolve_physical_devices=lambda _r: [0],
|
||||||
|
device_total_memory=lambda _d: cap,
|
||||||
|
external_reserve_bytes=0,
|
||||||
|
safety_margin_bytes=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert ledgers[0].kv_budget_bytes == int(cap * 0.35)
|
||||||
|
|
||||||
|
|
||||||
def test_check_admission_unresolved_local_raises():
|
def test_check_admission_unresolved_local_raises():
|
||||||
"""A local replica invisible to the ledger must fail admission, not skip it:
|
"""A local replica invisible to the ledger must fail admission, not skip it:
|
||||||
it would otherwise initialize concurrently with unbounded demand."""
|
it would otherwise initialize concurrently with unbounded demand."""
|
||||||
|
|||||||
@@ -16,7 +16,8 @@ from pytest_mock import MockerFixture
|
|||||||
from vllm.v1.engine.utils import EngineZmqAddresses
|
from vllm.v1.engine.utils import EngineZmqAddresses
|
||||||
|
|
||||||
from vllm_omni.config.config_factory import StageConfigFactory
|
from vllm_omni.config.config_factory import StageConfigFactory
|
||||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
from vllm_omni.config.omni_config import VllmOmniARStageConfig, VllmOmniDiffusionStageConfig
|
||||||
|
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig, StageExecutionType, StagePipelineConfig
|
||||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
|
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
|
||||||
from vllm_omni.engine.stage_engine_startup import (
|
from vllm_omni.engine.stage_engine_startup import (
|
||||||
@@ -57,6 +58,10 @@ def _make_llm_plan(
|
|||||||
vllm_config: Any | None = None,
|
vllm_config: Any | None = None,
|
||||||
) -> LogicalStageInitPlan:
|
) -> LogicalStageInitPlan:
|
||||||
stage_cfg = _make_stage_cfg(stage_id)
|
stage_cfg = _make_stage_cfg(stage_id)
|
||||||
|
if launch_mode == "remote":
|
||||||
|
stage_cfg = VllmOmniARStageConfig(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(stage_id=stage_id, model_stage="thinker"),
|
||||||
|
)
|
||||||
metadata = SimpleNamespace(
|
metadata = SimpleNamespace(
|
||||||
stage_id=stage_id,
|
stage_id=stage_id,
|
||||||
stage_type="llm",
|
stage_type="llm",
|
||||||
@@ -98,6 +103,16 @@ def _make_diffusion_plan(
|
|||||||
launch_mode: str,
|
launch_mode: str,
|
||||||
) -> LogicalStageInitPlan:
|
) -> LogicalStageInitPlan:
|
||||||
stage_cfg = _make_stage_cfg(stage_id, stage_type="diffusion")
|
stage_cfg = _make_stage_cfg(stage_id, stage_type="diffusion")
|
||||||
|
if launch_mode == "remote":
|
||||||
|
stage_cfg = VllmOmniDiffusionStageConfig(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(
|
||||||
|
stage_id=stage_id,
|
||||||
|
model_stage="diffusion",
|
||||||
|
execution_type=StageExecutionType.DIFFUSION,
|
||||||
|
final_output=True,
|
||||||
|
final_output_type="image",
|
||||||
|
),
|
||||||
|
)
|
||||||
metadata = SimpleNamespace(
|
metadata = SimpleNamespace(
|
||||||
stage_id=stage_id,
|
stage_id=stage_id,
|
||||||
stage_type="diffusion",
|
stage_type="diffusion",
|
||||||
@@ -960,6 +975,7 @@ class TestSingleStageReplicaInitialization:
|
|||||||
assert mock_connect.call_args.kwargs["stage_id"] == 1
|
assert mock_connect.call_args.kwargs["stage_id"] == 1
|
||||||
assert mock_connect.call_args.kwargs["replica_id"] == 0
|
assert mock_connect.call_args.kwargs["replica_id"] == 0
|
||||||
assert client_kwargs["log_stats"] is True
|
assert client_kwargs["log_stats"] is True
|
||||||
|
assert client_kwargs["metadata"].model_stage == "thinker"
|
||||||
assert events == ["enter", "exit", "attach"]
|
assert events == ["enter", "exit", "attach"]
|
||||||
|
|
||||||
def test_initialize_llm_replica_remote_missing_registered_stage_config_raises(self, mocker: MockerFixture):
|
def test_initialize_llm_replica_remote_missing_registered_stage_config_raises(self, mocker: MockerFixture):
|
||||||
@@ -1125,11 +1141,9 @@ class TestSingleStageReplicaInitialization:
|
|||||||
|
|
||||||
return _ctx()
|
return _ctx()
|
||||||
|
|
||||||
remote_metadata = _make_diffusion_plan(1, stage_id=1, launch_mode="remote").replicas[0].metadata
|
|
||||||
plan = _make_diffusion_plan(1, stage_id=1, launch_mode="remote").replicas[0]
|
plan = _make_diffusion_plan(1, stage_id=1, launch_mode="remote").replicas[0]
|
||||||
sentinel_client = SimpleNamespace()
|
sentinel_client = SimpleNamespace()
|
||||||
|
|
||||||
mocker.patch.object(runtime_mod, "extract_legacy_stage_metadata", return_value=remote_metadata)
|
|
||||||
mock_connect = mocker.patch.object(runtime_mod, "connect_remote_diffusion_proc", side_effect=_fake_connect)
|
mock_connect = mocker.patch.object(runtime_mod, "connect_remote_diffusion_proc", side_effect=_fake_connect)
|
||||||
mock_from_addresses = mocker.patch(
|
mock_from_addresses = mocker.patch(
|
||||||
"vllm_omni.diffusion.stage_diffusion_client.StageDiffusionClient.from_addresses",
|
"vllm_omni.diffusion.stage_diffusion_client.StageDiffusionClient.from_addresses",
|
||||||
@@ -1146,6 +1160,7 @@ class TestSingleStageReplicaInitialization:
|
|||||||
replica_id=0,
|
replica_id=0,
|
||||||
)
|
)
|
||||||
mock_from_addresses.assert_called_once()
|
mock_from_addresses.assert_called_once()
|
||||||
|
assert mock_from_addresses.call_args.args[0].final_output_type == "image"
|
||||||
|
|
||||||
def test_initialize_local_diffusion_replica_registers_with_master(self, mocker: MockerFixture):
|
def test_initialize_local_diffusion_replica_registers_with_master(self, mocker: MockerFixture):
|
||||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||||
|
|||||||
@@ -156,6 +156,43 @@ def test_replica_guard_and_splitter_share_local_world_size(devices, expected):
|
|||||||
assert replica_devices_map == {0: expected}
|
assert replica_devices_map == {0: expected}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||||
|
@pytest.mark.parametrize("local_dp", [None, 0, 1, 2])
|
||||||
|
@pytest.mark.parametrize("pool_devices", [False, True], ids=["template", "pool"])
|
||||||
|
def test_replica_layout_preserves_local_dp_semantics(typed, local_dp, pool_devices):
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||||
|
from vllm_omni.config.stage_config import PipelineConfig, StagePipelineConfig
|
||||||
|
from vllm_omni.engine.stage_engine_startup import get_headless_replica_devices
|
||||||
|
|
||||||
|
engine_args = {
|
||||||
|
"tensor_parallel_size": 2,
|
||||||
|
"data_parallel_size": 4,
|
||||||
|
"data_parallel_size_local": local_dp,
|
||||||
|
"pipeline_parallel_size": 2,
|
||||||
|
}
|
||||||
|
width = 2 * max(1, local_dp if local_dp is not None else 4) * 2
|
||||||
|
devices = ",".join(str(i) for i in range(width * (2 if pool_devices else 1)))
|
||||||
|
if typed:
|
||||||
|
pipeline = PipelineConfig(
|
||||||
|
model_type="test",
|
||||||
|
model_arch="TestModel",
|
||||||
|
stages=(StagePipelineConfig(stage_id=0, model_stage="ar", final_output=True),),
|
||||||
|
)
|
||||||
|
stage = VllmOmniConfig.from_pipeline_config(
|
||||||
|
pipeline,
|
||||||
|
cli_overrides={
|
||||||
|
f"stage_0_{key}": value for key, value in {**engine_args, "devices": devices, "num_replicas": 2}.items()
|
||||||
|
},
|
||||||
|
).stage_configs[0]
|
||||||
|
else:
|
||||||
|
stage = _stage(0, devices=devices, num_replicas=2, engine_args=engine_args)
|
||||||
|
expected = [",".join(str(i) for i in range(r * width, (r + 1) * width)) for r in range(2)]
|
||||||
|
|
||||||
|
assert get_stage_devices_per_replica(stage) == width
|
||||||
|
assert compute_replica_layout([stage]) == ([2], {0: expected})
|
||||||
|
assert get_headless_replica_devices(stage, stage_id=0, omni_dp_size_local=2) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_non_tp_layout_error_uses_generic_guidance():
|
def test_non_tp_layout_error_uses_generic_guidance():
|
||||||
"""A PP mismatch must not be reported as a top-level TP broadcast."""
|
"""A PP mismatch must not be reported as a top-level TP broadcast."""
|
||||||
stage = _stage(0, devices="0")
|
stage = _stage(0, devices="0")
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ _OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
|
|||||||
"num_weight_load_threads",
|
"num_weight_load_threads",
|
||||||
"omni_kv_config",
|
"omni_kv_config",
|
||||||
"parallel_config",
|
"parallel_config",
|
||||||
|
"pooling_output_decoder",
|
||||||
"silence_ban_frames",
|
"silence_ban_frames",
|
||||||
"subtalker_sampling_params",
|
"subtalker_sampling_params",
|
||||||
"task_type",
|
"task_type",
|
||||||
@@ -366,6 +367,21 @@ def test_typed_llm_projection_does_not_emit_inherited_upstream_defaults():
|
|||||||
assert inherited_defaults.isdisjoint(engine_args)
|
assert inherited_defaults.isdisjoint(engine_args)
|
||||||
|
|
||||||
|
|
||||||
|
def test_typed_llm_projection_omits_diffusion_only_and_process_only_defaults():
|
||||||
|
stage_config = VllmOmniARStageConfig(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="test"),
|
||||||
|
)
|
||||||
|
|
||||||
|
engine_args = stage_init_utils._project_omni_stage_engine_args(stage_config)
|
||||||
|
|
||||||
|
assert {
|
||||||
|
"enable_multithread_weight_load",
|
||||||
|
"num_weight_load_threads",
|
||||||
|
"disable_autocast",
|
||||||
|
"log_level",
|
||||||
|
}.isdisjoint(engine_args)
|
||||||
|
|
||||||
|
|
||||||
def test_mammoth_fp8_kv_deploy_projects_only_ar_stage(monkeypatch):
|
def test_mammoth_fp8_kv_deploy_projects_only_ar_stage(monkeypatch):
|
||||||
monkeypatch.setattr(stage_init_utils, "resolve_worker_cls", lambda _engine_args: None)
|
monkeypatch.setattr(stage_init_utils, "resolve_worker_cls", lambda _engine_args: None)
|
||||||
|
|
||||||
|
|||||||
@@ -64,6 +64,13 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|||||||
# TODO: extend this set and the expected census if @router.head / @router.options appear.
|
# TODO: extend this set and the expected census if @router.head / @router.options appear.
|
||||||
_HTTP_METHODS = {"GET", "POST", "DELETE", "PUT", "PATCH"}
|
_HTTP_METHODS = {"GET", "POST", "DELETE", "PUT", "PATCH"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_profiler_endpoints_detect_typed_stage_config():
|
||||||
|
stage = SimpleNamespace(profiler_config=SimpleNamespace(profiler="torch"))
|
||||||
|
|
||||||
|
assert api_server._should_enable_profiler_endpoints([stage])
|
||||||
|
|
||||||
|
|
||||||
# Exact Omni router census on main. Update intentionally when routes change.
|
# Exact Omni router census on main. Update intentionally when routes change.
|
||||||
_EXPECTED_ROUTER_ROUTES = {
|
_EXPECTED_ROUTER_ROUTES = {
|
||||||
("POST", "/v1/chat/completions"),
|
("POST", "/v1/chat/completions"),
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from types import SimpleNamespace
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||||
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES,
|
_AUDEX_NO_AUDIO_GUARD_MODEL_TYPES,
|
||||||
OmniOpenAIServingSpeech,
|
OmniOpenAIServingSpeech,
|
||||||
@@ -251,9 +252,28 @@ class TestAudexStageDetection:
|
|||||||
stage = serving._find_tts_stage()
|
stage = serving._find_tts_stage()
|
||||||
assert stage is not None and stage.engine_args.model_stage == "audex_thinker"
|
assert stage is not None and stage.engine_args.model_stage == "audex_thinker"
|
||||||
|
|
||||||
|
def test_typed_pipeline_detected_without_legacy_engine_args(self):
|
||||||
|
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||||
|
stages = [
|
||||||
|
SimpleNamespace(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(stage_id=index, model_stage=model_stage),
|
||||||
|
model_config=SimpleNamespace(model_arch=None),
|
||||||
|
worker_type="generation",
|
||||||
|
)
|
||||||
|
for index, model_stage in enumerate(("audex_thinker", "audex_code2wav"))
|
||||||
|
]
|
||||||
|
serving.engine_client = SimpleNamespace(stage_configs=stages)
|
||||||
|
|
||||||
|
stage = serving._find_tts_stage()
|
||||||
|
serving._tts_stage = stage
|
||||||
|
|
||||||
|
assert stage is stages[0]
|
||||||
|
assert serving._detect_tts_model_type() == "audex"
|
||||||
|
|
||||||
|
|
||||||
def test_guided_injection_sets_guided_temperature():
|
def test_guided_injection_sets_guided_temperature():
|
||||||
"""Guided requests decode at the measured CFG temperature (0.05)."""
|
"""Guided requests decode at the measured CFG temperature (0.05)."""
|
||||||
|
|
||||||
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
|
serving = TestAudexCfgInjection()._serving_with_fake_tokenizer()
|
||||||
cond = TestAudexCfgInjection()._cond_prompt()
|
cond = TestAudexCfgInjection()._cond_prompt()
|
||||||
params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
|
params = SimpleNamespace(extra_args={"cfg_scale": 1.5}, temperature=0.1)
|
||||||
@@ -265,6 +285,31 @@ def test_guided_injection_sets_guided_temperature():
|
|||||||
assert out.temperature == 0.1
|
assert out.temperature == 0.1
|
||||||
|
|
||||||
|
|
||||||
|
def test_typed_full_pipeline_uses_full_checkpoint_tokenizer(mocker):
|
||||||
|
from vllm_omni.entrypoints.openai.tts_adapters.base import SpeechServingContext
|
||||||
|
|
||||||
|
serving = OmniOpenAIServingSpeech.__new__(OmniOpenAIServingSpeech)
|
||||||
|
serving._tts_stage = SimpleNamespace(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(stage_id=0, model_stage="audex_omni"),
|
||||||
|
model_config=SimpleNamespace(model_arch=None),
|
||||||
|
worker_type="ar",
|
||||||
|
)
|
||||||
|
serving.engine_client = SimpleNamespace(
|
||||||
|
model_config=SimpleNamespace(model="/models/audex"),
|
||||||
|
)
|
||||||
|
ensure_snapshot = mocker.patch(
|
||||||
|
"vllm_omni.model_executor.models.audex.checkpoint.ensure_audex_snapshot",
|
||||||
|
return_value="/snapshots/audex",
|
||||||
|
)
|
||||||
|
from_pretrained = mocker.patch("transformers.AutoTokenizer.from_pretrained", return_value=object())
|
||||||
|
|
||||||
|
adapter = AudexAdapter(SpeechServingContext(server=serving, engine_client=serving.engine_client))
|
||||||
|
adapter._get_tokenizer()
|
||||||
|
|
||||||
|
ensure_snapshot.assert_called_once_with("/models/audex", profile="full")
|
||||||
|
from_pretrained.assert_called_once_with("/snapshots/audex/checkpoint_folder_full")
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- TTA serving surface
|
# ---------------------------------------------------------------- TTA serving surface
|
||||||
|
|
||||||
from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter # noqa: E402
|
from vllm_omni.entrypoints.openai.tts_adapters import resolve_adapter # noqa: E402
|
||||||
|
|||||||
@@ -138,6 +138,36 @@ def test_policy_server_config_reads_stage_config_model_config():
|
|||||||
assert serving.policy_server_config.to_dict() == policy_config
|
assert serving.policy_server_config.to_dict() == policy_config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("inline", [False, True], ids=["out_of_process", "inline"])
|
||||||
|
def test_policy_server_config_reads_typed_runtime(inline):
|
||||||
|
from tests.helpers.stage_config import get_deploy_config_path
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||||
|
from vllm_omni.diffusion.models.pi0_pipeline_config import PI0_PIPELINE
|
||||||
|
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||||
|
|
||||||
|
typed_stage = VllmOmniConfig.from_pipeline_config(
|
||||||
|
PI0_PIPELINE, deploy_config_path=get_deploy_config_path("pi0.yaml")
|
||||||
|
).stage_configs[0]
|
||||||
|
policy_config = typed_stage.diffusion_config.model_config["policy_server_config"]
|
||||||
|
stage_client = SimpleNamespace(stage_type="diffusion")
|
||||||
|
if inline:
|
||||||
|
# Worker-enriched inline metadata remains authoritative over the
|
||||||
|
# startup stage config; only the process client needs the fallback.
|
||||||
|
policy_config = {**policy_config, "worker_metadata": "loaded"}
|
||||||
|
stage_client.od_config = SimpleNamespace(model_config={"policy_server_config": policy_config})
|
||||||
|
engine_client = AsyncOmni.__new__(AsyncOmni)
|
||||||
|
engine_client.engine = SimpleNamespace(
|
||||||
|
stage_clients=[stage_client],
|
||||||
|
stage_configs=[typed_stage],
|
||||||
|
stage_vllm_configs=[None],
|
||||||
|
get_diffusion_od_config=lambda: SimpleNamespace(model_class_name="Pi0Pipeline"),
|
||||||
|
)
|
||||||
|
|
||||||
|
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=engine_client)
|
||||||
|
|
||||||
|
assert serving.policy_server_config.to_dict() == policy_config
|
||||||
|
|
||||||
|
|
||||||
def test_policy_server_config_reads_omegaconf_stage_config():
|
def test_policy_server_config_reads_omegaconf_stage_config():
|
||||||
engine_client = SimpleNamespace(
|
engine_client = SimpleNamespace(
|
||||||
get_diffusion_od_config=lambda: None,
|
get_diffusion_od_config=lambda: None,
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class _EngineClientStub:
|
class _EngineClientStub:
|
||||||
stage_configs: list[dict[str, object]]
|
stage_configs: list[object]
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -149,6 +149,37 @@ async def test_chat_reference_audio_reaches_code2wav_transport(
|
|||||||
assert code2wav_payload.meta.ref_audio_sr == 16000
|
assert code2wav_payload.meta.ref_audio_sr == 16000
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_typed_stage_enables_reference_audio(
|
||||||
|
serving_chat: OmniOpenAIServingChat,
|
||||||
|
mocker: MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
reference_audio_url = "data:audio/wav;base64,AAAA"
|
||||||
|
reference_waveform = np.array([0.25, -0.5], dtype=np.float32)
|
||||||
|
serving_chat.engine_client = _EngineClientStub(
|
||||||
|
[
|
||||||
|
SimpleNamespace(
|
||||||
|
model_config=SimpleNamespace(model_arch="MiniCPMO45OmniForConditionalGeneration"),
|
||||||
|
model_stage="llm",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
serving_chat.model_config = _ModelConfigStub()
|
||||||
|
engine_prompt: TokensPrompt = {"type": "tokens", "prompt_token_ids": [1]}
|
||||||
|
media_connector = mocker.patch(
|
||||||
|
"vllm_omni.entrypoints.openai.serving_chat.MediaConnector",
|
||||||
|
).return_value
|
||||||
|
media_connector.fetch_audio_async = mocker.AsyncMock(return_value=(reference_waveform, 16000))
|
||||||
|
|
||||||
|
await serving_chat._attach_minicpmo45_reference_audio(
|
||||||
|
engine_prompt,
|
||||||
|
_chat_request(reference_audio_url),
|
||||||
|
)
|
||||||
|
|
||||||
|
media_connector.fetch_audio_async.assert_awaited_once_with(reference_audio_url)
|
||||||
|
assert MINICPMO45_REFERENCE_AUDIO_KEY in cast(dict[str, object], engine_prompt)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("request_shape", ["extra_body", "model_extra"])
|
@pytest.mark.parametrize("request_shape", ["extra_body", "model_extra"])
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_reference_audio_accepts_request_extra_mappings(
|
async def test_reference_audio_accepts_request_extra_mappings(
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
"""Regression tests for MiniCPM-o 4.5 audio-streaming output-kind coercion.
|
"""Regression tests for MiniCPM-o 4.5 audio-streaming output-kind coercion.
|
||||||
|
|
||||||
Chat ``stream=true`` coerces every AR stage to DELTA. For MiniCPM-o 4.5 that
|
Chat ``stream=true`` coerces every AR stage to DELTA. For MiniCPM-o 4.5 that
|
||||||
@@ -33,6 +34,13 @@ def _stage(model_arch: str, model_stage: str) -> SimpleNamespace:
|
|||||||
return SimpleNamespace(engine_args=SimpleNamespace(model_arch=model_arch, model_stage=model_stage))
|
return SimpleNamespace(engine_args=SimpleNamespace(model_arch=model_arch, model_stage=model_stage))
|
||||||
|
|
||||||
|
|
||||||
|
def _typed_stage(model_arch: str, model_stage: str) -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
model_config=SimpleNamespace(model_arch=model_arch),
|
||||||
|
model_stage=model_stage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _minicpmo45_stages():
|
def _minicpmo45_stages():
|
||||||
return [_stage(MINICPMO45_ARCH, "llm"), _stage(MINICPMO45_ARCH, "tts")]
|
return [_stage(MINICPMO45_ARCH, "llm"), _stage(MINICPMO45_ARCH, "tts")]
|
||||||
|
|
||||||
@@ -50,6 +58,20 @@ def test_thinker_forced_final_only_talker_stays_delta(serving_chat):
|
|||||||
assert out[1].output_kind is RequestOutputKind.DELTA
|
assert out[1].output_kind is RequestOutputKind.DELTA
|
||||||
|
|
||||||
|
|
||||||
|
def test_typed_stages_force_thinker_final_only(serving_chat):
|
||||||
|
serving_chat.engine_client = SimpleNamespace(
|
||||||
|
stage_configs=[
|
||||||
|
_typed_stage(MINICPMO45_ARCH, "llm"),
|
||||||
|
_typed_stage(MINICPMO45_ARCH, "tts"),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
out = serving_chat._fix_minicpmo45_audio_stream_output_kinds(_delta_params(), ["text", "audio"])
|
||||||
|
|
||||||
|
assert out[0].output_kind is RequestOutputKind.FINAL_ONLY
|
||||||
|
assert out[1].output_kind is RequestOutputKind.DELTA
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("modalities", [None, [], ["text"]])
|
@pytest.mark.parametrize("modalities", [None, [], ["text"]])
|
||||||
def test_non_audio_request_is_untouched(serving_chat, modalities):
|
def test_non_audio_request_is_untouched(serving_chat, modalities):
|
||||||
serving_chat.engine_client = SimpleNamespace(stage_configs=_minicpmo45_stages())
|
serving_chat.engine_client = SimpleNamespace(stage_configs=_minicpmo45_stages())
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from pydantic import ValidationError
|
|||||||
from pytest_mock import MockerFixture
|
from pytest_mock import MockerFixture
|
||||||
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse
|
||||||
|
|
||||||
|
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||||
from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler
|
from vllm_omni.diffusion.sched.request_scheduler import RequestScheduler
|
||||||
from vllm_omni.diffusion.sched.step_scheduler import StepScheduler
|
from vllm_omni.diffusion.sched.step_scheduler import StepScheduler
|
||||||
@@ -1039,6 +1040,38 @@ class TestTTSMethods:
|
|||||||
asyncio.run(server._prepare_speech_generation(request))
|
asyncio.run(server._prepare_speech_generation(request))
|
||||||
server.shutdown()
|
server.shutdown()
|
||||||
|
|
||||||
|
def test_prepare_speech_rejects_typed_non_tts_omni_model(self, mocker: MockerFixture):
|
||||||
|
"""Typed Qwen3-Omni stages must hit the same talker guard as legacy stages."""
|
||||||
|
mock_engine_client = mocker.MagicMock()
|
||||||
|
mock_engine_client.errored = False
|
||||||
|
mock_engine_client.tts_max_instructions_length = None
|
||||||
|
mock_engine_client.stage_configs = [
|
||||||
|
SimpleNamespace(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(stage_id=index, model_stage=model_stage),
|
||||||
|
model_config=SimpleNamespace(model_arch=None),
|
||||||
|
worker_type=worker_type,
|
||||||
|
)
|
||||||
|
for index, (model_stage, worker_type) in enumerate(
|
||||||
|
(
|
||||||
|
("thinker", "ar"),
|
||||||
|
("talker", "ar"),
|
||||||
|
("code2wav", "generation"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
mock_models = mocker.MagicMock()
|
||||||
|
mock_models.is_base_model.return_value = True
|
||||||
|
server = OmniOpenAIServingSpeech(
|
||||||
|
engine_client=mock_engine_client,
|
||||||
|
models=mock_models,
|
||||||
|
request_logger=mocker.MagicMock(),
|
||||||
|
)
|
||||||
|
request = OpenAICreateSpeechRequest(input="Hello world")
|
||||||
|
with pytest.raises(ValueError, match="only supported for dedicated TTS models"):
|
||||||
|
asyncio.run(server._prepare_speech_generation(request))
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
def test_estimate_prompt_len_fallback(self, speech_server):
|
def test_estimate_prompt_len_fallback(self, speech_server):
|
||||||
"""Test prompt length estimation falls back to 2048 when model is unavailable."""
|
"""Test prompt length estimation falls back to 2048 when model is unavailable."""
|
||||||
tts_params = {"text": ["Hello"], "task_type": ["CustomVoice"]}
|
tts_params = {"text": ["Hello"], "task_type": ["CustomVoice"]}
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from vllm_omni.entrypoints.openai.tts_adapters import (
|
|||||||
detect_tts_model_type,
|
detect_tts_model_type,
|
||||||
resolve_adapter,
|
resolve_adapter,
|
||||||
)
|
)
|
||||||
|
from vllm_omni.entrypoints.openai.tts_adapters.base import resolve_stage_model_path
|
||||||
from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter
|
from vllm_omni.entrypoints.openai.tts_adapters.covo_audio import CovoAudioAdapter
|
||||||
from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter
|
from vllm_omni.entrypoints.openai.tts_adapters.higgs_audio_v2 import HiggsAudioV2Adapter
|
||||||
from vllm_omni.entrypoints.openai.tts_adapters.indextts2 import (
|
from vllm_omni.entrypoints.openai.tts_adapters.indextts2 import (
|
||||||
@@ -99,6 +100,31 @@ def test_resolve_unknown_returns_none():
|
|||||||
assert resolve_adapter(None) is None
|
assert resolve_adapter(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_stage_model_path_prefers_typed_override():
|
||||||
|
engine_client = SimpleNamespace(
|
||||||
|
stage_configs=[
|
||||||
|
SimpleNamespace(engine_args=SimpleNamespace(model="legacy-stage-model")),
|
||||||
|
SimpleNamespace(
|
||||||
|
model_config=SimpleNamespace(model="typed-stage-model"),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
model="served-model",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resolve_stage_model_path(engine_client) == "typed-stage-model"
|
||||||
|
|
||||||
|
|
||||||
|
def test_stage_model_path_falls_back_to_legacy_then_served_model():
|
||||||
|
legacy_client = SimpleNamespace(
|
||||||
|
stage_configs=[SimpleNamespace(engine_args=SimpleNamespace(model="legacy-stage-model"))],
|
||||||
|
model="served-model",
|
||||||
|
)
|
||||||
|
served_client = SimpleNamespace(stage_configs=[SimpleNamespace()], model="served-model")
|
||||||
|
|
||||||
|
assert resolve_stage_model_path(legacy_client) == "legacy-stage-model"
|
||||||
|
assert resolve_stage_model_path(served_client) == "served-model"
|
||||||
|
|
||||||
|
|
||||||
def test_voxcpm2_resolves():
|
def test_voxcpm2_resolves():
|
||||||
"""VoxCPM2 (the served ``latent_generator`` model) resolves cleanly.
|
"""VoxCPM2 (the served ``latent_generator`` model) resolves cleanly.
|
||||||
|
|
||||||
|
|||||||
@@ -1134,6 +1134,32 @@ def test_mixed_reference_capability_uses_model_metadata_when_config_defaults_fal
|
|||||||
assert handler.supports_mixed_reference_inputs
|
assert handler.supports_mixed_reference_inputs
|
||||||
|
|
||||||
|
|
||||||
|
def test_typed_stage_drives_video_capability_checks():
|
||||||
|
from vllm_omni.config.config_factory import StageConfigFactory
|
||||||
|
|
||||||
|
minimax_stage = StageConfigFactory.create_typed_default_diffusion(
|
||||||
|
"minimax-h3",
|
||||||
|
{"model_class_name": "MiniMaxH3Pipeline"},
|
||||||
|
).stage_configs[0]
|
||||||
|
minimax_handler = OmniOpenAIServingVideo.for_diffusion(
|
||||||
|
SimpleNamespace(od_config=SimpleNamespace(model_class_name=None)),
|
||||||
|
model_name="minimax-h3",
|
||||||
|
stage_configs=[minimax_stage],
|
||||||
|
)
|
||||||
|
assert minimax_handler.supports_mixed_reference_inputs
|
||||||
|
|
||||||
|
cosmos_stage = StageConfigFactory.create_typed_default_diffusion(
|
||||||
|
"cosmos3",
|
||||||
|
{"model_class_name": "Cosmos3OmniDiffusersPipeline"},
|
||||||
|
).stage_configs[0]
|
||||||
|
cosmos_handler = OmniOpenAIServingVideo.for_diffusion(
|
||||||
|
SimpleNamespace(od_config=SimpleNamespace(model_class_name=None)),
|
||||||
|
model_name="cosmos3",
|
||||||
|
stage_configs=[cosmos_stage],
|
||||||
|
)
|
||||||
|
assert cosmos_handler.supported_control_upload_types == frozenset({"edge", "blur", "depth", "seg", "wsm"})
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("model_class_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
|
@pytest.mark.parametrize("model_class_name", ["Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"])
|
||||||
def test_control_upload_capability_is_declared_only_by_cosmos3(test_client, model_class_name):
|
def test_control_upload_capability_is_declared_only_by_cosmos3(test_client, model_class_name):
|
||||||
handler = test_client.app.state.openai_serving_video
|
handler = test_client.app.state.openai_serving_video
|
||||||
@@ -1191,6 +1217,35 @@ def test_cosmos3_reference_video_limit_uses_v2v_condition_frames():
|
|||||||
assert spec.keep == "first"
|
assert spec.keep == "first"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("num_frames", "extra_params", "expected"),
|
||||||
|
[
|
||||||
|
(189, {"condition_frame_indexes_vision": [0, 2], "condition_video_keep": "last"}, (9, "last")),
|
||||||
|
(189, {"condition_frame_indexes_vision": [0, 2]}, (9, "first")),
|
||||||
|
(5, {"condition_frame_indexes_vision": [0, 20]}, (5, "first")),
|
||||||
|
(None, {"action_mode": "inverse_dynamics", "action_chunk_size": 16}, (17, "first")),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_cosmos3_reference_video_decode_policy_with_runtime_configs(typed, num_frames, extra_params, expected):
|
||||||
|
from vllm_omni.config.config_factory import StageConfigFactory
|
||||||
|
|
||||||
|
if typed:
|
||||||
|
stages = list(
|
||||||
|
StageConfigFactory.create_typed_default_diffusion(
|
||||||
|
"cosmos3",
|
||||||
|
{"model_class_name": "Cosmos3OmniDiffusersPipeline"},
|
||||||
|
).stage_configs
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
stages = _cosmos3_stage_configs()
|
||||||
|
request = VideoGenerationRequest(prompt="Continue this motion.", num_frames=num_frames, extra_params=extra_params)
|
||||||
|
|
||||||
|
spec = _reference_video_decode_spec(request, stages)
|
||||||
|
|
||||||
|
assert (spec.max_frames, spec.keep) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_cosmos3_reference_video_limit_preserves_action_frames():
|
def test_cosmos3_reference_video_limit_preserves_action_frames():
|
||||||
request = VideoGenerationRequest(
|
request = VideoGenerationRequest(
|
||||||
prompt="Predict the action.",
|
prompt="Predict the action.",
|
||||||
|
|||||||
@@ -2,13 +2,15 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
from vllm_omni.config.config_factory import StageConfigFactory
|
from vllm_omni.config.config_factory import StageConfigFactory
|
||||||
from vllm_omni.config.resolver import OmniConfigResolution
|
from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig
|
||||||
|
from vllm_omni.config.resolver import OmniConfigResolution, resolve_omni_config
|
||||||
from vllm_omni.diffusion.data import AttentionConfig, OmniDiffusionConfig
|
from vllm_omni.diffusion.data import AttentionConfig, OmniDiffusionConfig
|
||||||
from vllm_omni.engine import stage_init_utils
|
from vllm_omni.engine import stage_init_utils
|
||||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||||
@@ -117,7 +119,7 @@ def test_stage_override_preserves_model_extras_for_default_diffusion_stage(mocke
|
|||||||
trust_remote_code=False,
|
trust_remote_code=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert stage_configs[0]["engine_args"]["extras"]["ltx2_use_conv_vae"] is True
|
assert stage_configs[0].diffusion_config.extras["ltx2_use_conv_vae"] is True
|
||||||
|
|
||||||
|
|
||||||
def test_default_stage_rejects_unknown_nested_parallel_config_key():
|
def test_default_stage_rejects_unknown_nested_parallel_config_key():
|
||||||
@@ -239,6 +241,33 @@ def test_default_stage_config_includes_default_sampling_params():
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||||
|
@pytest.mark.parametrize("sampling_defaults", [{"0": {"guidance_scale": 7.5}}, '{"0":{"guidance_scale":7.5}}'])
|
||||||
|
def test_generic_diffusion_sampling_defaults_remain_overridable(typed, sampling_defaults):
|
||||||
|
from vllm_omni.config.yaml_util import create_config
|
||||||
|
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||||
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
|
|
||||||
|
kwargs = {"model_class_name": "QwenImagePipeline", "default_sampling_params": sampling_defaults}
|
||||||
|
if typed:
|
||||||
|
stage = StageConfigFactory.create_typed_default_diffusion("generic-diffusion", kwargs).stage_configs[0]
|
||||||
|
metadata = stage_init_utils.extract_stage_metadata_from_omni_stage_config(stage)
|
||||||
|
else:
|
||||||
|
stage = create_config(StageConfigFactory.create_default_diffusion(kwargs))[0]
|
||||||
|
metadata = stage_init_utils.extract_legacy_stage_metadata(stage)
|
||||||
|
base = OmniBase.__new__(OmniBase)
|
||||||
|
base.engine = SimpleNamespace(num_stages=1, stage_configs=[stage])
|
||||||
|
base.default_sampling_params_list = [metadata.default_sampling_params]
|
||||||
|
base.sampling_constraints_list = base._get_sampling_constraints_list([stage])
|
||||||
|
|
||||||
|
assert base.resolve_sampling_params_list(None)[0].guidance_scale == 7.5
|
||||||
|
requested = OmniDiffusionSamplingParams(guidance_scale=2.0)
|
||||||
|
assert base.resolve_sampling_params_list(requested)[0].guidance_scale == 2.0
|
||||||
|
assert requested.guidance_scale == 2.0
|
||||||
|
assert base.default_sampling_params_list[0].guidance_scale == 7.5
|
||||||
|
assert base.sampling_constraints_list == [{}]
|
||||||
|
|
||||||
|
|
||||||
def test_default_stage_config_includes_diffusion_attention_backend():
|
def test_default_stage_config_includes_diffusion_attention_backend():
|
||||||
"""Ensure diffusion attention shorthand lands in engine_args.diffusion_attention_config."""
|
"""Ensure diffusion attention shorthand lands in engine_args.diffusion_attention_config."""
|
||||||
stage_cfg = StageConfigFactory.create_default_diffusion(
|
stage_cfg = StageConfigFactory.create_default_diffusion(
|
||||||
@@ -726,3 +755,106 @@ def test_default_stage_config_includes_quantization_config():
|
|||||||
stage_cfg = StageConfigFactory.create_default_diffusion({"quantization_config": quantization_config})[0]
|
stage_cfg = StageConfigFactory.create_default_diffusion({"quantization_config": quantization_config})[0]
|
||||||
|
|
||||||
assert stage_cfg["engine_args"]["quantization_config"] == quantization_config
|
assert stage_cfg["engine_args"]["quantization_config"] == quantization_config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model_class_name", ["HeliosPipeline", "HunyuanVideo15Pipeline"])
|
||||||
|
def test_generic_diffusion_uses_canonical_video_output_type(model_class_name):
|
||||||
|
config = StageConfigFactory.create_typed_default_diffusion(
|
||||||
|
"generic-video",
|
||||||
|
{"model_class_name": model_class_name},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert config.stage_configs[0].final_output_type == "video"
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_diffusion_resolves_structured_stage_without_legacy_conversion(mocker):
|
||||||
|
"""Generic diffusion reaches runtime as the structured stage itself."""
|
||||||
|
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||||
|
return_value=(True, "FakeDiffusionPipeline"),
|
||||||
|
)
|
||||||
|
|
||||||
|
resolved = resolve_omni_config(
|
||||||
|
"generic-diffusion",
|
||||||
|
trust_remote_code=False,
|
||||||
|
deploy_config_path=None,
|
||||||
|
cli_overrides={
|
||||||
|
"num_gpus": 4,
|
||||||
|
"tensor_parallel_size": 2,
|
||||||
|
"default_sampling_params": '{"0": {"guidance_scale": 7.5}}',
|
||||||
|
},
|
||||||
|
stage_overrides=None,
|
||||||
|
strategy_config_path=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resolved.pipeline_config is not None
|
||||||
|
assert resolved.pipeline_config.model_type == "generic_diffusion"
|
||||||
|
assert len(resolved.stage_configs) == 1
|
||||||
|
stage = resolved.stage_configs[0]
|
||||||
|
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||||
|
assert stage.model_config.model == "generic-diffusion"
|
||||||
|
assert stage.model_config.default_sampling_params == {"guidance_scale": 7.5}
|
||||||
|
assert stage.parallel_config.tensor_parallel_size == 2
|
||||||
|
assert stage.parallel_config.data_parallel_size == 2
|
||||||
|
assert stage.parallel_config.world_size == 4
|
||||||
|
assert stage.runtime_config.devices == "0,1,2,3"
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_diffusion_structured_stage_reaches_standard_startup(mocker):
|
||||||
|
"""Standard runtime resolves and starts the typed stage through the real launcher."""
|
||||||
|
from vllm_omni.engine import stage_engine_startup as startup_module
|
||||||
|
from vllm_omni.engine import stage_runtime as runtime_module
|
||||||
|
from vllm_omni.engine.stage_runtime import StageRuntime
|
||||||
|
|
||||||
|
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||||
|
return_value=(True, "FakeDiffusionPipeline"),
|
||||||
|
)
|
||||||
|
resolved = resolve_omni_config(
|
||||||
|
"generic-diffusion",
|
||||||
|
trust_remote_code=False,
|
||||||
|
deploy_config_path=None,
|
||||||
|
cli_overrides={"num_gpus": 1},
|
||||||
|
stage_overrides=None,
|
||||||
|
strategy_config_path=None,
|
||||||
|
)
|
||||||
|
stage = resolved.stage_configs[0]
|
||||||
|
launched: dict[str, Any] = {}
|
||||||
|
client = SimpleNamespace(input_address=None, shutdown=mocker.Mock())
|
||||||
|
|
||||||
|
mocker.patch.object(runtime_module, "prepare_engine_environment")
|
||||||
|
mocker.patch.object(runtime_module, "load_omni_transfer_config_for_model", return_value=None)
|
||||||
|
mocker.patch.object(runtime_module, "get_stage_connector_spec", return_value={})
|
||||||
|
mocker.patch.object(runtime_module, "resolve_omni_kv_config_for_stage", return_value=(None, None, None))
|
||||||
|
|
||||||
|
def _initialize_typed_stage(stage_id, model, stage_config, metadata, **kwargs):
|
||||||
|
launched.update(
|
||||||
|
stage_id=stage_id,
|
||||||
|
model=model,
|
||||||
|
stage_config=stage_config,
|
||||||
|
metadata=metadata,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
return client
|
||||||
|
|
||||||
|
mocker.patch.object(startup_module, "initialize_diffusion_stage", side_effect=_initialize_typed_stage)
|
||||||
|
|
||||||
|
runtime = StageRuntime(
|
||||||
|
stage_configs=list(resolved.stage_configs),
|
||||||
|
model="generic-diffusion",
|
||||||
|
config_path="",
|
||||||
|
stage_init_timeout=10,
|
||||||
|
async_chunk=False,
|
||||||
|
)
|
||||||
|
runtime.initialize()
|
||||||
|
|
||||||
|
assert launched["stage_config"] is stage
|
||||||
|
assert isinstance(launched["stage_config"], VllmOmniDiffusionStageConfig)
|
||||||
|
assert launched["metadata"].stage_type == "diffusion"
|
||||||
|
assert launched["metadata"].model_stage == "diffusion"
|
||||||
|
assert launched["use_inline"] is True
|
||||||
|
assert launched["model"] == "generic-diffusion"
|
||||||
|
assert launched["stage_id"] == 0
|
||||||
|
assert runtime.stage_pools[0].clients == [client]
|
||||||
@@ -17,6 +17,8 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|||||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||||
|
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||||
|
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||||
from vllm_omni.config.stage_config import StageConfig
|
from vllm_omni.config.stage_config import StageConfig
|
||||||
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
|
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
|
||||||
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
|
from vllm_omni.engine.messages import ErrorMessage, OutputMessage
|
||||||
@@ -202,6 +204,49 @@ def _make_base():
|
|||||||
return obj
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("typed", [False, True], ids=["legacy", "typed"])
|
||||||
|
def test_resolve_sampling_params_list_preserves_stage_constraints(typed):
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniARStageConfig
|
||||||
|
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||||
|
|
||||||
|
base = _make_base()
|
||||||
|
base.engine.num_stages = 1
|
||||||
|
base.default_sampling_params_list = [SamplingParams(max_tokens=1000, detokenize=False, stop_token_ids=[42])]
|
||||||
|
if typed:
|
||||||
|
stage = VllmOmniARStageConfig(
|
||||||
|
stage_pipeline_config=StagePipelineConfig(
|
||||||
|
stage_id=0,
|
||||||
|
model_stage="dummy-model",
|
||||||
|
sampling_constraints={"detokenize": False, "stop_token_ids": [42]},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
stage = StageConfig(
|
||||||
|
stage_id=0,
|
||||||
|
model_stage="dummy-model",
|
||||||
|
sampling_constraints={"detokenize": False, "stop_token_ids": [42]},
|
||||||
|
).to_omegaconf()
|
||||||
|
base.engine.stage_configs = [stage]
|
||||||
|
base.sampling_constraints_list = base._get_sampling_constraints_list(base.engine.stage_configs)
|
||||||
|
assert base.sampling_constraints_list == [{"detokenize": False, "stop_token_ids": [42]}]
|
||||||
|
|
||||||
|
resolved_defaults = base.resolve_sampling_params_list(None)
|
||||||
|
assert resolved_defaults[0].stop_token_ids == [42]
|
||||||
|
|
||||||
|
caller_params = SamplingParams(seed=1234, max_tokens=7, detokenize=True, stop_token_ids=[7])
|
||||||
|
resolved = base.resolve_sampling_params_list(caller_params)
|
||||||
|
|
||||||
|
assert resolved[0] is not caller_params
|
||||||
|
assert resolved[0].seed == 1234
|
||||||
|
assert resolved[0].max_tokens == 7
|
||||||
|
assert resolved[0].detokenize is False
|
||||||
|
assert resolved[0].stop_token_ids == [7, 42]
|
||||||
|
assert 7 in resolved[0]._all_stop_token_ids
|
||||||
|
assert 42 in resolved[0]._all_stop_token_ids
|
||||||
|
assert caller_params.detokenize is True
|
||||||
|
assert caller_params.stop_token_ids == [7]
|
||||||
|
|
||||||
|
|
||||||
def test_resolve_sampling_params_list_merges_required_stop_tokens():
|
def test_resolve_sampling_params_list_merges_required_stop_tokens():
|
||||||
base = _make_base()
|
base = _make_base()
|
||||||
base.engine.num_stages = 1
|
base.engine.num_stages = 1
|
||||||
@@ -219,22 +264,13 @@ def test_resolve_sampling_params_list_merges_required_stop_tokens():
|
|||||||
base.sampling_constraints_list = base._get_sampling_constraints_list(base.engine.stage_configs)
|
base.sampling_constraints_list = base._get_sampling_constraints_list(base.engine.stage_configs)
|
||||||
assert base.sampling_constraints_list == [{"detokenize": False, "stop_token_ids": required_stop_ids}]
|
assert base.sampling_constraints_list == [{"detokenize": False, "stop_token_ids": required_stop_ids}]
|
||||||
|
|
||||||
resolved_defaults = base.resolve_sampling_params_list(None)
|
|
||||||
|
|
||||||
assert resolved_defaults[0].stop_token_ids == [99, 151704, 151645]
|
def test_sampling_constraints_are_forwarded_by_typed_stage_configs():
|
||||||
|
config = VllmOmniConfig.from_pipeline_config(OMNI_PIPELINES["qwen3_tts"])
|
||||||
|
|
||||||
caller_params = SamplingParams(seed=1234, max_tokens=7, detokenize=True, stop_token_ids=[100, 151704])
|
constraints = OmniBase._get_sampling_constraints_list(config.stage_configs)
|
||||||
|
|
||||||
resolved = base.resolve_sampling_params_list(caller_params)
|
assert constraints == [dict(stage.stage_pipeline_config.sampling_constraints) for stage in config.stage_configs]
|
||||||
|
|
||||||
assert resolved[0] is not caller_params
|
|
||||||
assert resolved[0].seed == 1234
|
|
||||||
assert resolved[0].max_tokens == 7
|
|
||||||
assert resolved[0].detokenize is False
|
|
||||||
assert resolved[0].stop_token_ids == [100, 151704, 151645]
|
|
||||||
assert {100, 151704, 151645}.issubset(resolved[0]._all_stop_token_ids)
|
|
||||||
assert caller_params.detokenize is True
|
|
||||||
assert caller_params.stop_token_ids == [100, 151704]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("use_defaults", [False, True])
|
@pytest.mark.parametrize("use_defaults", [False, True])
|
||||||
|
|||||||
@@ -32,15 +32,18 @@ def test_dreamzero_resolves_through_registry_with_model_defaults(monkeypatch):
|
|||||||
stage_overrides=None,
|
stage_overrides=None,
|
||||||
strategy_config_path=None,
|
strategy_config_path=None,
|
||||||
)
|
)
|
||||||
engine_args = resolved.stage_configs[0].engine_args
|
stage = resolved.stage_configs[0]
|
||||||
|
|
||||||
assert resolved.config_path is not None
|
assert resolved.config_path is not None
|
||||||
assert resolved.config_path.endswith("vllm_omni/deploy/dreamzero.yaml")
|
assert resolved.config_path.endswith("vllm_omni/deploy/dreamzero.yaml")
|
||||||
assert engine_args.model_class_name == "DreamZeroPipeline"
|
assert stage.diffusion_config.model_class_name == "DreamZeroPipeline"
|
||||||
assert engine_args.model_config.policy_server_config.action_space == "joint_position"
|
assert stage.diffusion_config.model_config["policy_server_config"]["action_space"] == "joint_position"
|
||||||
|
|
||||||
|
|
||||||
def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch):
|
def test_dreamzero_enrich_config_preserves_explicit_model_class_name(monkeypatch):
|
||||||
|
def _get_hf_config(path, _model):
|
||||||
|
return None if path == "model_index.json" else {"model_type": "vla", "architectures": ["VLA"]}
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"vllm.transformers_utils.config.get_hf_file_to_dict",
|
"vllm.transformers_utils.config.get_hf_file_to_dict",
|
||||||
lambda path, _model, **_kwargs: (
|
lambda path, _model, **_kwargs: (
|
||||||
|
|||||||
@@ -7,10 +7,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from pytest_mock import MockerFixture
|
from pytest_mock import MockerFixture
|
||||||
|
|
||||||
|
from vllm_omni.config.omni_config import VllmOmniDiffusionStageConfig
|
||||||
from vllm_omni.config.resolver import OmniConfigResolution
|
from vllm_omni.config.resolver import OmniConfigResolution
|
||||||
from vllm_omni.entrypoints.cli.serve import (
|
from vllm_omni.entrypoints.cli.serve import (
|
||||||
OmniServeCommand,
|
OmniServeCommand,
|
||||||
@@ -334,6 +336,26 @@ def test_parse_stage_overrides_invalid_json_raises() -> None:
|
|||||||
assert f"Got: {bad!r}" in message
|
assert f"Got: {bad!r}" in message
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"payload",
|
||||||
|
['{"abc": {}}', '{"-1": {}}', '{"1.5": {}}', '{"\uff10": {}}'],
|
||||||
|
)
|
||||||
|
def test_parse_stage_overrides_rejects_invalid_stage_ids(payload: str) -> None:
|
||||||
|
with pytest.raises(argparse.ArgumentTypeError, match="non-negative integer stage ids"):
|
||||||
|
_parse_stage_overrides(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_stage_overrides_preserves_arbitrary_override_fields() -> None:
|
||||||
|
parsed = _parse_stage_overrides(
|
||||||
|
'{"0": {"extras": {"ltx2_use_conv_vae": true}, "kv_cache_dtype": "fp8", "typo_field_xyz": 1}}'
|
||||||
|
)
|
||||||
|
assert parsed["0"] == {
|
||||||
|
"extras": {"ltx2_use_conv_vae": True},
|
||||||
|
"kv_cache_dtype": "fp8",
|
||||||
|
"typo_field_xyz": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_parse_stage_overrides_rejects_non_dict_top_level() -> None:
|
def test_parse_stage_overrides_rejects_non_dict_top_level() -> None:
|
||||||
"""Top-level must be a JSON object (dict). A list, scalar, or non-dict
|
"""Top-level must be a JSON object (dict). A list, scalar, or non-dict
|
||||||
mapping is rejected with a ValueError naming ``--stage-overrides`` and
|
mapping is rejected with a ValueError naming ``--stage-overrides`` and
|
||||||
@@ -706,6 +728,77 @@ def test_run_headless_diffusion_registers_and_spawns_proc(mocker: MockerFixture)
|
|||||||
assert manager_kwargs["omni_replica_id"] == 0
|
assert manager_kwargs["omni_replica_id"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_headless_generic_diffusion_launches_structured_stage(mocker: MockerFixture) -> None:
|
||||||
|
"""Headless resolution starts the typed stage through the real group launcher."""
|
||||||
|
from vllm_omni.engine import stage_engine_startup as startup_module
|
||||||
|
|
||||||
|
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||||
|
return_value=(True, "FakeDiffusionPipeline"),
|
||||||
|
)
|
||||||
|
mocker.patch("vllm_omni.engine.stage_init_utils.prepare_engine_environment")
|
||||||
|
mocker.patch.object(startup_module.stage_init_utils, "load_omni_transfer_config_for_model", return_value=None)
|
||||||
|
mocker.patch.object(
|
||||||
|
startup_module.initialization,
|
||||||
|
"resolve_omni_kv_config_for_stage",
|
||||||
|
return_value=(None, None, None),
|
||||||
|
)
|
||||||
|
captured: dict[str, Any] = {}
|
||||||
|
od_config = SimpleNamespace()
|
||||||
|
|
||||||
|
def _build_diffusion_config(model, stage_config, metadata):
|
||||||
|
captured.update(model=model, stage_config=stage_config, metadata=metadata)
|
||||||
|
return od_config
|
||||||
|
|
||||||
|
def _launch_replica_group(**kwargs):
|
||||||
|
captured.update(group_kwargs=kwargs)
|
||||||
|
captured["manager"] = kwargs["launch_one"](0)
|
||||||
|
|
||||||
|
def _launch_replica(**kwargs):
|
||||||
|
captured.update(replica_kwargs=kwargs)
|
||||||
|
return SimpleNamespace(exitcode=None)
|
||||||
|
|
||||||
|
mocker.patch.object(startup_module.stage_init_utils, "build_diffusion_config", side_effect=_build_diffusion_config)
|
||||||
|
mocker.patch.object(startup_module, "launch_headless_replica_group", side_effect=_launch_replica_group)
|
||||||
|
mocker.patch.object(startup_module, "launch_headless_diffusion_replica", side_effect=_launch_replica)
|
||||||
|
|
||||||
|
explicit_keys = frozenset(
|
||||||
|
{
|
||||||
|
"model",
|
||||||
|
"stage_id",
|
||||||
|
"omni_master_address",
|
||||||
|
"omni_master_port",
|
||||||
|
"worker_backend",
|
||||||
|
"model_class_name",
|
||||||
|
"num_gpus",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
args = _make_headless_args(
|
||||||
|
explicit_keys=explicit_keys,
|
||||||
|
model="generic-diffusion",
|
||||||
|
model_class_name="FakeDiffusionPipeline",
|
||||||
|
num_gpus=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
run_headless(args)
|
||||||
|
|
||||||
|
stage = captured["stage_config"]
|
||||||
|
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||||
|
assert captured["metadata"].stage_type == "diffusion"
|
||||||
|
assert captured["metadata"].model_stage == "diffusion"
|
||||||
|
assert captured["model"] == "generic-diffusion"
|
||||||
|
assert captured["group_kwargs"]["stage_id"] == 0
|
||||||
|
assert captured["group_kwargs"]["omni_dp_size_local"] == 1
|
||||||
|
assert captured["group_kwargs"]["per_replica_devices"] == ["0"]
|
||||||
|
assert captured["replica_kwargs"]["stage_config"] is stage
|
||||||
|
assert captured["replica_kwargs"]["stage_id"] == 0
|
||||||
|
assert captured["replica_kwargs"]["omni_master_address"] == "127.0.0.1"
|
||||||
|
assert captured["replica_kwargs"]["omni_master_port"] == 26000
|
||||||
|
assert captured["replica_kwargs"]["od_config"] is od_config
|
||||||
|
assert stage.runtime_config.devices == "0"
|
||||||
|
|
||||||
|
|
||||||
def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixture) -> None:
|
def test_run_headless_diffusion_raises_on_nonzero_proc_exit(mocker: MockerFixture) -> None:
|
||||||
"""A diffusion replica that exits with a non-zero code must surface as a
|
"""A diffusion replica that exits with a non-zero code must surface as a
|
||||||
RuntimeError from ``run_headless`` (the head needs the signal to roll
|
RuntimeError from ``run_headless`` (the head needs the signal to roll
|
||||||
|
|||||||
+124
-50
@@ -13,6 +13,14 @@ import torch
|
|||||||
from pytest_mock import MockerFixture
|
from pytest_mock import MockerFixture
|
||||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||||
|
|
||||||
|
from vllm_omni.config.composable_parallel import (
|
||||||
|
Broadcast,
|
||||||
|
FanInByStage,
|
||||||
|
MeshAxisSpec,
|
||||||
|
RouteByStage,
|
||||||
|
StrategySpec,
|
||||||
|
TakeRank,
|
||||||
|
)
|
||||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||||
from vllm_omni.config.resolver import (
|
from vllm_omni.config.resolver import (
|
||||||
OmniConfigResolution,
|
OmniConfigResolution,
|
||||||
@@ -21,7 +29,6 @@ from vllm_omni.config.resolver import (
|
|||||||
resolve_omni_config,
|
resolve_omni_config,
|
||||||
)
|
)
|
||||||
from vllm_omni.config.stage_config import PipelineConfig
|
from vllm_omni.config.stage_config import PipelineConfig
|
||||||
from vllm_omni.config.yaml_util import create_config
|
|
||||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||||
from vllm_omni.entrypoints.utils import (
|
from vllm_omni.entrypoints.utils import (
|
||||||
@@ -356,16 +363,12 @@ class TestResolveOmniConfig:
|
|||||||
)
|
)
|
||||||
assert resolved.config_path is None
|
assert resolved.config_path is None
|
||||||
assert len(resolved.stage_configs) == 1
|
assert len(resolved.stage_configs) == 1
|
||||||
engine_args = resolved.stage_configs[0]["engine_args"]
|
assert resolved.stage_configs[0].diffusion_config.dtype is torch.float32
|
||||||
assert "dtype" in engine_args
|
assert resolved.stage_configs[0].diffusion_config.engine_backend == engine_backend
|
||||||
assert engine_args["engine_backend"] == engine_backend
|
assert resolved.stage_configs[0].diffusion_config.revision == "pinned-revision"
|
||||||
assert engine_args["revision"] == "pinned-revision"
|
|
||||||
|
|
||||||
def test_generic_diffusion_uses_registered_model_metadata(self, mocker: MockerFixture):
|
def test_generic_diffusion_uses_registered_model_metadata(self, mocker: MockerFixture):
|
||||||
mocker.patch(
|
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
|
||||||
return_value=None,
|
|
||||||
)
|
|
||||||
resolve_model_class = mocker.patch(
|
resolve_model_class = mocker.patch(
|
||||||
"vllm_omni.config.resolver.resolve_model_class_name",
|
"vllm_omni.config.resolver.resolve_model_class_name",
|
||||||
return_value="WanImageToVideoPipeline",
|
return_value="WanImageToVideoPipeline",
|
||||||
@@ -374,59 +377,98 @@ class TestResolveOmniConfig:
|
|||||||
"vllm_omni.config.resolver.DiffusionModelRegistry.get_supported_archs",
|
"vllm_omni.config.resolver.DiffusionModelRegistry.get_supported_archs",
|
||||||
return_value={"WanImageToVideoPipeline"},
|
return_value={"WanImageToVideoPipeline"},
|
||||||
)
|
)
|
||||||
|
load_model_class = mocker.patch("vllm_omni.config.resolver.DiffusionModelRegistry._try_load_model_cls")
|
||||||
|
|
||||||
resolved = resolve_omni_config(
|
resolved = resolve_omni_config(
|
||||||
"/models/Wan2.2-I2V",
|
"/models/Wan2.2-I2V",
|
||||||
trust_remote_code=False,
|
trust_remote_code=False,
|
||||||
deploy_config_path=None,
|
deploy_config_path=None,
|
||||||
cli_overrides={
|
cli_overrides={"diffusion_load_format": "diffusers", "revision": "pinned-revision"},
|
||||||
"diffusion_load_format": "diffusers",
|
|
||||||
"revision": "pinned-revision",
|
|
||||||
},
|
|
||||||
stage_overrides=None,
|
stage_overrides=None,
|
||||||
strategy_config_path=None,
|
strategy_config_path=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
resolve_model_class.assert_called_once_with("/models/Wan2.2-I2V", "diffusers", "pinned-revision")
|
||||||
|
load_model_class.assert_not_called()
|
||||||
stage = resolved.stage_configs[0]
|
stage = resolved.stage_configs[0]
|
||||||
resolve_model_class.assert_called_once_with(
|
assert stage.diffusion_config.model_class_name == "WanImageToVideoPipeline"
|
||||||
"/models/Wan2.2-I2V",
|
assert stage.diffusion_config.revision == "pinned-revision"
|
||||||
"diffusers",
|
|
||||||
"pinned-revision",
|
|
||||||
)
|
|
||||||
assert stage.engine_args.model_class_name == "WanImageToVideoPipeline"
|
|
||||||
assert stage.final_output_type == "video"
|
assert stage.final_output_type == "video"
|
||||||
|
|
||||||
|
def test_generic_diffusion_stage_overrides_reach_typed_backend(self, mocker: MockerFixture):
|
||||||
|
from vllm_omni.engine.stage_init_utils import build_engine_args_dict_from_omni_stage_config
|
||||||
|
|
||||||
|
mocker.patch("vllm_omni.config.resolver.StageConfigFactory.create_from_model", return_value=None)
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.config.resolver._resolve_generic_diffusion_model_class",
|
||||||
|
return_value=(True, "LTX2Pipeline"),
|
||||||
|
)
|
||||||
|
resolved = resolve_omni_config(
|
||||||
|
"/models/LTX-2.5-Diffusers",
|
||||||
|
trust_remote_code=False,
|
||||||
|
deploy_config_path=None,
|
||||||
|
cli_overrides={
|
||||||
|
"extras": {"ltx2_use_conv_vae": False, "keep": "global"},
|
||||||
|
"tensor_parallel_size": 1,
|
||||||
|
"diffusion_streaming_output": True,
|
||||||
|
},
|
||||||
|
stage_overrides={
|
||||||
|
"0": {
|
||||||
|
"extras": {"ltx2_use_conv_vae": True},
|
||||||
|
"tensor_parallel_size": 2,
|
||||||
|
"devices": "2,3",
|
||||||
|
"engine_backend": "custom.backend",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
strategy_config_path=None,
|
||||||
|
)
|
||||||
|
stage = resolved.stage_by_id(0)
|
||||||
|
engine_args = build_engine_args_dict_from_omni_stage_config(stage, model="/models/LTX-2.5-Diffusers")
|
||||||
|
|
||||||
|
assert stage.runtime_config.devices == "2,3"
|
||||||
|
assert stage.parallel_config.world_size == 2
|
||||||
|
assert engine_args["engine_backend"] == "custom.backend"
|
||||||
|
assert engine_args["extras"]["ltx2_use_conv_vae"] is True
|
||||||
|
assert engine_args["extras"]["keep"] == "global"
|
||||||
|
assert engine_args["streaming_output"] is True
|
||||||
|
|
||||||
def test_registered_pipeline_uses_structured_metadata_and_preserves_override_trust(self, mocker: MockerFixture):
|
def test_registered_pipeline_uses_structured_metadata_and_preserves_override_trust(self, mocker: MockerFixture):
|
||||||
endpoint_restriction = SimpleNamespace(name="chat")
|
endpoint_restriction = SimpleNamespace(name="chat")
|
||||||
|
typed_stage = SimpleNamespace(stage_id=1)
|
||||||
structured_config = SimpleNamespace(
|
structured_config = SimpleNamespace(
|
||||||
orchestrator_config=SimpleNamespace(deploy_config_path="/resolved/deploy.yaml"),
|
orchestrator_config=SimpleNamespace(
|
||||||
|
deploy_config_path="/resolved/deploy.yaml",
|
||||||
|
omni_lb_policy="round_robin",
|
||||||
|
),
|
||||||
pipeline_config=SimpleNamespace(endpoint_restrictions=(endpoint_restriction,)),
|
pipeline_config=SimpleNamespace(endpoint_restrictions=(endpoint_restriction,)),
|
||||||
|
stage_configs=(typed_stage,),
|
||||||
|
strategy_omni_lb_policy="round-robin",
|
||||||
)
|
)
|
||||||
runtime_stage = create_config(
|
|
||||||
{
|
|
||||||
"stage_id": 1,
|
|
||||||
"runtime": {"devices": "1,2,3", "num_replicas": 3},
|
|
||||||
"engine_args": {"model": "dummy-model"},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
legacy_stage = SimpleNamespace(to_omegaconf=lambda: runtime_stage)
|
|
||||||
create_structured = mocker.patch(
|
create_structured = mocker.patch(
|
||||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
||||||
return_value=structured_config,
|
return_value=structured_config,
|
||||||
)
|
)
|
||||||
create_legacy = mocker.patch(
|
create_legacy = mocker.patch(
|
||||||
"vllm_omni.config.resolver.StageConfigFactory._resolve_legacy_from_registry",
|
"vllm_omni.config.resolver.StageConfigFactory.create_legacy_stage_configs_from_model",
|
||||||
return_value=SimpleNamespace(
|
|
||||||
stage_configs=[legacy_stage],
|
|
||||||
pipeline_config=structured_config.pipeline_config,
|
|
||||||
omni_lb_policy="round_robin",
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
strategy_specs = {"stage_1": {"dp": 3}}
|
strategy_specs = {
|
||||||
|
"stage_1": [
|
||||||
|
StrategySpec(
|
||||||
|
"stage_replica",
|
||||||
|
MeshAxisSpec("stage_replica", 2),
|
||||||
|
RouteByStage("round_robin"),
|
||||||
|
FanInByStage(),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
}
|
||||||
load_strategy = mocker.patch(
|
load_strategy = mocker.patch(
|
||||||
"vllm_omni.config.resolver._load_strategy_specs",
|
"vllm_omni.config.resolver._load_strategy_specs",
|
||||||
return_value=strategy_specs,
|
return_value=strategy_specs,
|
||||||
)
|
)
|
||||||
|
apply_strategy = mocker.patch(
|
||||||
|
"vllm_omni.config.composable_parallel.apply_strategy_specs",
|
||||||
|
return_value=SimpleNamespace(omni_lb_policy="round_robin"),
|
||||||
|
)
|
||||||
|
|
||||||
resolved = resolve_omni_config(
|
resolved = resolve_omni_config(
|
||||||
"dummy-model",
|
"dummy-model",
|
||||||
@@ -447,31 +489,63 @@ class TestResolveOmniConfig:
|
|||||||
trust_remote_code=None,
|
trust_remote_code=None,
|
||||||
cli_overrides=expected_overrides,
|
cli_overrides=expected_overrides,
|
||||||
deploy_config_path="deploy.yaml",
|
deploy_config_path="deploy.yaml",
|
||||||
)
|
|
||||||
create_legacy.assert_called_once_with(
|
|
||||||
structured_config.pipeline_config,
|
|
||||||
expected_overrides,
|
|
||||||
"/resolved/deploy.yaml",
|
|
||||||
strategy_specs=strategy_specs,
|
strategy_specs=strategy_specs,
|
||||||
)
|
)
|
||||||
|
create_legacy.assert_not_called()
|
||||||
load_strategy.assert_called_once_with("strategy.yaml")
|
load_strategy.assert_called_once_with("strategy.yaml")
|
||||||
|
apply_strategy.assert_not_called()
|
||||||
assert resolved.config_path == "/resolved/deploy.yaml"
|
assert resolved.config_path == "/resolved/deploy.yaml"
|
||||||
assert resolved.pipeline_config is structured_config.pipeline_config
|
assert resolved.pipeline_config is structured_config.pipeline_config
|
||||||
assert resolved.omni_lb_policy == "round_robin"
|
assert resolved.omni_lb_policy == "round-robin"
|
||||||
assert resolved.endpoint_restrictions == (endpoint_restriction,)
|
assert resolved.endpoint_restrictions == (endpoint_restriction,)
|
||||||
assert resolved.stage_configs == (runtime_stage,)
|
assert resolved.stage_configs == (typed_stage,)
|
||||||
|
|
||||||
def test_registered_resolution_exposes_forced_aligner_topology(self, mocker: MockerFixture):
|
def test_tp_only_strategy_does_not_report_default_lb_policy_as_derived(self, mocker: MockerFixture):
|
||||||
pipeline = OMNI_PIPELINES["qwen3_tts"]
|
|
||||||
assert isinstance(pipeline, PipelineConfig)
|
|
||||||
structured_config = SimpleNamespace(
|
structured_config = SimpleNamespace(
|
||||||
orchestrator_config=SimpleNamespace(deploy_config_path=None),
|
orchestrator_config=SimpleNamespace(
|
||||||
pipeline_config=pipeline,
|
deploy_config_path="/resolved/deploy.yaml",
|
||||||
|
omni_lb_policy="round-robin",
|
||||||
|
),
|
||||||
|
pipeline_config=SimpleNamespace(endpoint_restrictions=()),
|
||||||
|
stage_configs=(SimpleNamespace(stage_id=0),),
|
||||||
|
strategy_omni_lb_policy=None,
|
||||||
)
|
)
|
||||||
mocker.patch(
|
mocker.patch(
|
||||||
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
"vllm_omni.config.resolver.StageConfigFactory.create_from_model",
|
||||||
return_value=structured_config,
|
return_value=structured_config,
|
||||||
)
|
)
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.config.resolver._load_strategy_specs",
|
||||||
|
return_value={
|
||||||
|
"thinker": [
|
||||||
|
StrategySpec(
|
||||||
|
"tp",
|
||||||
|
MeshAxisSpec("tp", 2),
|
||||||
|
Broadcast(),
|
||||||
|
TakeRank(),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
resolved = resolve_omni_config(
|
||||||
|
"dummy-model",
|
||||||
|
trust_remote_code=True,
|
||||||
|
deploy_config_path="deploy.yaml",
|
||||||
|
cli_overrides={"omni_lb_policy": "round-robin"},
|
||||||
|
stage_overrides=None,
|
||||||
|
strategy_config_path="strategy.yaml",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resolved.omni_lb_policy is None
|
||||||
|
|
||||||
|
def test_registered_resolution_exposes_forced_aligner_topology(self, mocker: MockerFixture):
|
||||||
|
pipeline = OMNI_PIPELINES["qwen3_tts"]
|
||||||
|
assert isinstance(pipeline, PipelineConfig)
|
||||||
|
mocker.patch(
|
||||||
|
"vllm_omni.config.config_factory.StageConfigFactory.get_pipeline_config",
|
||||||
|
return_value=pipeline,
|
||||||
|
)
|
||||||
|
|
||||||
resolved = resolve_omni_config(
|
resolved = resolve_omni_config(
|
||||||
"Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
"Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||||
@@ -483,9 +557,9 @@ class TestResolveOmniConfig:
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert resolved.pipeline_config is not None
|
assert resolved.pipeline_config is not None
|
||||||
pipeline_stage_ids = [stage.stage_id for stage in resolved.pipeline_config.stages]
|
assert [stage.stage_id for stage in resolved.pipeline_config.stages] == [
|
||||||
runtime_stage_ids = [stage.stage_id for stage in resolved.stage_configs]
|
stage.stage_id for stage in resolved.stage_configs
|
||||||
assert pipeline_stage_ids == runtime_stage_ids
|
]
|
||||||
assert resolved.pipeline_config.stages[-1].model_stage == "forced_aligner"
|
assert resolved.pipeline_config.stages[-1].model_stage == "forced_aligner"
|
||||||
assert len(resolved.pipeline_config.stages) == len(pipeline.stages) + 1
|
assert len(resolved.pipeline_config.stages) == len(pipeline.stages) + 1
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
"""Apply declarative parallel strategies onto merged stage configs.
|
"""Apply declarative parallel strategies onto merged stage configs.
|
||||||
|
|
||||||
This is the *override-after-merge* seam. ``merge_pipeline_deploy`` first fuses
|
This is the *override-after-merge* seam. ``merge_pipeline_deploy`` first fuses
|
||||||
@@ -198,10 +198,41 @@ def _check_devices(runtime: dict[str, Any], cfg: OmniParallelConfig, *, role: Ro
|
|||||||
|
|
||||||
|
|
||||||
def _apply_to_stage(stage: Any, cfg: OmniParallelConfig, *, role: RoleKey) -> None:
|
def _apply_to_stage(stage: Any, cfg: OmniParallelConfig, *, role: RoleKey) -> None:
|
||||||
|
declared = set(cfg.l1_owners.keys())
|
||||||
|
if hasattr(stage, "parallel_config") and hasattr(stage, "runtime_config"):
|
||||||
|
parallel = stage.parallel_config
|
||||||
|
for kind, field_name in _ENGINE_FIELD_BY_KIND.items():
|
||||||
|
if kind in declared:
|
||||||
|
existing = getattr(parallel, field_name, None)
|
||||||
|
value = getattr(cfg, field_name)
|
||||||
|
if existing not in (None, 1, value):
|
||||||
|
_fail(
|
||||||
|
f"role {role!r}: strategy derives {field_name}={value} "
|
||||||
|
f"but structured config already set {existing}"
|
||||||
|
)
|
||||||
|
setattr(parallel, field_name, value)
|
||||||
|
if "ep" in declared and cfg.enable_expert_parallel:
|
||||||
|
setattr(parallel, "enable_expert_parallel", True)
|
||||||
|
if "stage_replica" in declared:
|
||||||
|
existing = stage.runtime_config.num_replicas
|
||||||
|
if existing not in (None, 1, cfg.stage_replica_size):
|
||||||
|
_fail(
|
||||||
|
f"role {role!r}: strategy derives "
|
||||||
|
f"num_replicas={cfg.stage_replica_size} but structured config already set {existing}"
|
||||||
|
)
|
||||||
|
stage.runtime_config.num_replicas = cfg.stage_replica_size
|
||||||
|
check_device_layout(
|
||||||
|
stage.runtime_config.devices,
|
||||||
|
tensor_parallel_size=cfg.tensor_parallel_size,
|
||||||
|
data_parallel_size=cfg.data_parallel_size,
|
||||||
|
pipeline_parallel_size=cfg.pipeline_parallel_size,
|
||||||
|
num_replicas=int(stage.runtime_config.num_replicas or 1),
|
||||||
|
role=role,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
engine_args = stage.yaml_engine_args
|
engine_args = stage.yaml_engine_args
|
||||||
runtime = stage.yaml_runtime
|
runtime = stage.yaml_runtime
|
||||||
|
|
||||||
declared = set(cfg.l1_owners.keys())
|
|
||||||
for kind, field_name in _ENGINE_FIELD_BY_KIND.items():
|
for kind, field_name in _ENGINE_FIELD_BY_KIND.items():
|
||||||
if kind in declared:
|
if kind in declared:
|
||||||
_set_explicit(engine_args, field_name, getattr(cfg, field_name), role=role)
|
_set_explicit(engine_args, field_name, getattr(cfg, field_name), role=role)
|
||||||
@@ -209,7 +240,6 @@ def _apply_to_stage(stage: Any, cfg: OmniParallelConfig, *, role: RoleKey) -> No
|
|||||||
_set_explicit(engine_args, "enable_expert_parallel", True, role=role)
|
_set_explicit(engine_args, "enable_expert_parallel", True, role=role)
|
||||||
if "stage_replica" in declared:
|
if "stage_replica" in declared:
|
||||||
_set_num_replicas(runtime, cfg.stage_replica_size, role=role)
|
_set_num_replicas(runtime, cfg.stage_replica_size, role=role)
|
||||||
|
|
||||||
_check_devices(runtime, cfg, role=role)
|
_check_devices(runtime, cfg, role=role)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,9 @@ from vllm_omni.config.stage_config import (
|
|||||||
DeployConfig,
|
DeployConfig,
|
||||||
PipelineConfig,
|
PipelineConfig,
|
||||||
StageConfig,
|
StageConfig,
|
||||||
|
StageDeployConfig,
|
||||||
|
StageExecutionType,
|
||||||
|
StagePipelineConfig,
|
||||||
StageType,
|
StageType,
|
||||||
build_stage_runtime_overrides,
|
build_stage_runtime_overrides,
|
||||||
load_deploy_config,
|
load_deploy_config,
|
||||||
@@ -373,6 +376,7 @@ class StageConfigFactory:
|
|||||||
trust_remote_code: bool | None,
|
trust_remote_code: bool | None,
|
||||||
cli_overrides: dict[str, Any],
|
cli_overrides: dict[str, Any],
|
||||||
deploy_config_path: str | None,
|
deploy_config_path: str | None,
|
||||||
|
strategy_specs: Mapping[Any, Any] | None = None,
|
||||||
) -> VllmOmniConfig | None:
|
) -> VllmOmniConfig | None:
|
||||||
"""Build the structured Omni config for a model/deploy pair."""
|
"""Build the structured Omni config for a model/deploy pair."""
|
||||||
user_deploy_config = cls._load_user_deploy_config(deploy_config_path)
|
user_deploy_config = cls._load_user_deploy_config(deploy_config_path)
|
||||||
@@ -387,6 +391,20 @@ class StageConfigFactory:
|
|||||||
if pipeline_cfg is None:
|
if pipeline_cfg is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# Keep the typed representation's topology in lockstep with the
|
||||||
|
# compatibility launch representation. The aligner is a real extra
|
||||||
|
# stage, not an engine-only setting, so it must be injected before
|
||||||
|
# either representation is built.
|
||||||
|
if user_deploy_config is None:
|
||||||
|
if pipeline_cfg.default_deploy_config_name is not None:
|
||||||
|
default_deploy_path = _DEPLOY_DIR / pipeline_cfg.default_deploy_config_name
|
||||||
|
user_deploy_config = load_deploy_config(default_deploy_path)
|
||||||
|
deploy_config_path = str(default_deploy_path)
|
||||||
|
else:
|
||||||
|
user_deploy_config = DeployConfig()
|
||||||
|
from vllm_omni.utils.forced_aligner import inject_forced_aligner_stage
|
||||||
|
|
||||||
|
pipeline_cfg, user_deploy_config = inject_forced_aligner_stage(pipeline_cfg, user_deploy_config, cli_overrides)
|
||||||
registry_cli_overrides = with_trust_remote_code_override(
|
registry_cli_overrides = with_trust_remote_code_override(
|
||||||
{**cli_overrides, "model": model},
|
{**cli_overrides, "model": model},
|
||||||
trust_remote_code,
|
trust_remote_code,
|
||||||
@@ -396,6 +414,7 @@ class StageConfigFactory:
|
|||||||
user_deploy_config=user_deploy_config,
|
user_deploy_config=user_deploy_config,
|
||||||
deploy_config_path=deploy_config_path,
|
deploy_config_path=deploy_config_path,
|
||||||
cli_overrides=registry_cli_overrides,
|
cli_overrides=registry_cli_overrides,
|
||||||
|
strategy_specs=strategy_specs,
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -621,13 +640,11 @@ class StageConfigFactory:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def create_default_diffusion(cls, kwargs: dict[str, Any]) -> list[dict[str, Any]]:
|
def _normalize_default_diffusion(
|
||||||
"""Build the temporary runtime ABI for a generic diffusion stage.
|
cls,
|
||||||
|
kwargs: dict[str, Any],
|
||||||
The terminal diffusion config owns engine defaults and normalization;
|
) -> tuple[dict[str, Any], DiffusionParallelConfig, dict[str, Any], str]:
|
||||||
this compatibility builder only adds Omni stage topology, request
|
"""Normalize inputs shared by typed and compatibility diffusion builders."""
|
||||||
defaults, and device placement.
|
|
||||||
"""
|
|
||||||
raw_sampling_params = kwargs.get("default_sampling_params")
|
raw_sampling_params = kwargs.get("default_sampling_params")
|
||||||
if isinstance(raw_sampling_params, str):
|
if isinstance(raw_sampling_params, str):
|
||||||
try:
|
try:
|
||||||
@@ -643,8 +660,6 @@ class StageConfigFactory:
|
|||||||
if kwargs.get("num_gpus") is not None:
|
if kwargs.get("num_gpus") is not None:
|
||||||
parallel_config.resolve_data_parallel_size(int(kwargs["num_gpus"]))
|
parallel_config.resolve_data_parallel_size(int(kwargs["num_gpus"]))
|
||||||
engine_args = OmniDiffusionConfig.normalize_init_kwargs(kwargs)
|
engine_args = OmniDiffusionConfig.normalize_init_kwargs(kwargs)
|
||||||
engine_args["parallel_config"] = asdict(parallel_config)
|
|
||||||
engine_args["model_stage"] = "diffusion"
|
|
||||||
|
|
||||||
extras = dict(engine_args.get("extras") or {})
|
extras = dict(engine_args.get("extras") or {})
|
||||||
for key, default in (
|
for key, default in (
|
||||||
@@ -665,6 +680,21 @@ class StageConfigFactory:
|
|||||||
final_output_type,
|
final_output_type,
|
||||||
model_class_name,
|
model_class_name,
|
||||||
)
|
)
|
||||||
|
return engine_args, parallel_config, default_sampling_params, final_output_type
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create_default_diffusion(cls, kwargs: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
"""Build the temporary runtime ABI for a generic diffusion stage.
|
||||||
|
|
||||||
|
The terminal diffusion config owns engine defaults and normalization;
|
||||||
|
this compatibility builder only adds Omni stage topology, request
|
||||||
|
defaults, and device placement.
|
||||||
|
"""
|
||||||
|
engine_args, parallel_config, default_sampling_params, final_output_type = cls._normalize_default_diffusion(
|
||||||
|
kwargs
|
||||||
|
)
|
||||||
|
engine_args["parallel_config"] = asdict(parallel_config)
|
||||||
|
engine_args["model_stage"] = "diffusion"
|
||||||
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
@@ -681,6 +711,50 @@ class StageConfigFactory:
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create_typed_default_diffusion(
|
||||||
|
cls,
|
||||||
|
model: str,
|
||||||
|
kwargs: dict[str, Any],
|
||||||
|
) -> VllmOmniConfig:
|
||||||
|
"""Build generic diffusion directly into the structured runtime config."""
|
||||||
|
engine_overrides, parallel_config, default_sampling_params, final_output_type = (
|
||||||
|
cls._normalize_default_diffusion(kwargs)
|
||||||
|
)
|
||||||
|
engine_overrides.update(asdict(parallel_config))
|
||||||
|
model_class_name = engine_overrides.get("model_class_name")
|
||||||
|
pipeline = PipelineConfig(
|
||||||
|
model_type="generic_diffusion",
|
||||||
|
model_arch=str(model_class_name or ""),
|
||||||
|
stages=(
|
||||||
|
StagePipelineConfig(
|
||||||
|
stage_id=0,
|
||||||
|
model_stage="diffusion",
|
||||||
|
execution_type=StageExecutionType.DIFFUSION,
|
||||||
|
final_output=True,
|
||||||
|
final_output_type=final_output_type,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# These values have already been normalized for this diffusion stage.
|
||||||
|
# Scope them explicitly so diffusion-only fields such as engine_backend
|
||||||
|
# and extras are not filtered by the global LLM CLI argument surface.
|
||||||
|
stage_overrides = {
|
||||||
|
f"stage_0_{key}": value for key, value in engine_overrides.items() if key not in {"model", "stage_id"}
|
||||||
|
}
|
||||||
|
stage_overrides["stage_0_devices"] = (
|
||||||
|
kwargs.get("stage_0_devices")
|
||||||
|
or kwargs.get("devices")
|
||||||
|
or ",".join(str(i) for i in range(parallel_config.world_size))
|
||||||
|
)
|
||||||
|
# Caller defaults belong to deployment, not immutable pipeline constraints:
|
||||||
|
# explicit request sampling parameters must still be able to override them.
|
||||||
|
deploy = DeployConfig(stages=[StageDeployConfig(stage_id=0, default_sampling_params=default_sampling_params)])
|
||||||
|
return VllmOmniConfig.from_pipeline_config(
|
||||||
|
pipeline, user_deploy_config=deploy, cli_overrides={"model": model, **stage_overrides}
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _merge_cli_overrides(
|
def _merge_cli_overrides(
|
||||||
cls,
|
cls,
|
||||||
|
|||||||
@@ -29,7 +29,9 @@ from vllm.config import ProfilerConfig as VllmProfilerConfig
|
|||||||
from vllm.config import SchedulerConfig as VllmSchedulerConfig
|
from vllm.config import SchedulerConfig as VllmSchedulerConfig
|
||||||
from vllm.config.utils import config
|
from vllm.config.utils import config
|
||||||
from vllm.engine.arg_utils import EngineArgs as VllmEngineArgs
|
from vllm.engine.arg_utils import EngineArgs as VllmEngineArgs
|
||||||
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||||
|
from vllm.pooling_params import PoolingParams
|
||||||
|
|
||||||
from vllm_omni.config.stage_config import (
|
from vllm_omni.config.stage_config import (
|
||||||
_DEPLOY_DIR,
|
_DEPLOY_DIR,
|
||||||
@@ -42,11 +44,13 @@ from vllm_omni.config.stage_config import (
|
|||||||
StagePipelineConfig,
|
StagePipelineConfig,
|
||||||
StageType,
|
StageType,
|
||||||
_apply_platform_overrides,
|
_apply_platform_overrides,
|
||||||
|
_get_recursively_merged_dict,
|
||||||
_resolve_scheduler,
|
_resolve_scheduler,
|
||||||
_scheduler_path,
|
_scheduler_path,
|
||||||
_select_processor_funcs,
|
_select_processor_funcs,
|
||||||
build_stage_runtime_overrides,
|
build_stage_runtime_overrides,
|
||||||
load_deploy_config,
|
load_deploy_config,
|
||||||
|
merge_pipeline_deploy,
|
||||||
merge_sampling_constraints,
|
merge_sampling_constraints,
|
||||||
normalize_pipeline_cli_overrides,
|
normalize_pipeline_cli_overrides,
|
||||||
reconcile_diffusion_attention_overrides,
|
reconcile_diffusion_attention_overrides,
|
||||||
@@ -55,6 +59,8 @@ from vllm_omni.config.stage_config import (
|
|||||||
)
|
)
|
||||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||||
|
|
||||||
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
_EXECUTION_TYPE_TO_STAGE_WORKER: dict[StageExecutionType, tuple[StageType, str | None]] = {
|
_EXECUTION_TYPE_TO_STAGE_WORKER: dict[StageExecutionType, tuple[StageType, str | None]] = {
|
||||||
StageExecutionType.LLM_AR: (StageType.LLM, "ar"),
|
StageExecutionType.LLM_AR: (StageType.LLM, "ar"),
|
||||||
StageExecutionType.LLM_GENERATION: (StageType.LLM, "generation"),
|
StageExecutionType.LLM_GENERATION: (StageType.LLM, "generation"),
|
||||||
@@ -186,6 +192,11 @@ class _ModelEngineOverrides(TypedDict, total=False):
|
|||||||
enable_prompt_embeds: bool
|
enable_prompt_embeds: bool
|
||||||
|
|
||||||
|
|
||||||
|
class _PoolingEngineOverrides(TypedDict, total=False):
|
||||||
|
runner: str
|
||||||
|
pooling_output_decoder: str
|
||||||
|
|
||||||
|
|
||||||
class _LoadEngineOverrides(TypedDict, total=False):
|
class _LoadEngineOverrides(TypedDict, total=False):
|
||||||
tokenizer: str
|
tokenizer: str
|
||||||
download_dir: str
|
download_dir: str
|
||||||
@@ -261,6 +272,7 @@ class _StageEngineValues:
|
|||||||
quantization: _QuantizationEngineOverrides
|
quantization: _QuantizationEngineOverrides
|
||||||
model: _ModelEngineOverrides
|
model: _ModelEngineOverrides
|
||||||
load: _LoadEngineOverrides
|
load: _LoadEngineOverrides
|
||||||
|
pooling: _PoolingEngineOverrides
|
||||||
cache: _CacheEngineOverrides
|
cache: _CacheEngineOverrides
|
||||||
scheduler: _SchedulerEngineOverrides
|
scheduler: _SchedulerEngineOverrides
|
||||||
connector: _ConnectorEngineOverrides
|
connector: _ConnectorEngineOverrides
|
||||||
@@ -476,6 +488,15 @@ class OmniStageModelConfig(_TrackExplicitConfigFields):
|
|||||||
enable_prompt_embeds: bool | None = None
|
enable_prompt_embeds: bool | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@config(config=ConfigDict(arbitrary_types_allowed=True))
|
||||||
|
class OmniStagePoolingConfig:
|
||||||
|
"""Typed inputs owned by vLLM pooling stages."""
|
||||||
|
|
||||||
|
runner: str | None = None
|
||||||
|
pooling_output_decoder: str | None = None
|
||||||
|
default_pooling_params: PoolingParams | None = None
|
||||||
|
|
||||||
|
|
||||||
@_enforce_keyword_only_init
|
@_enforce_keyword_only_init
|
||||||
@config(kw_only=True)
|
@config(kw_only=True)
|
||||||
class OmniStageLoadConfig(_TrackExplicitConfigFields, VllmLoadConfig):
|
class OmniStageLoadConfig(_TrackExplicitConfigFields, VllmLoadConfig):
|
||||||
@@ -833,12 +854,12 @@ class _DiffusionConfigProjection:
|
|||||||
from vllm_omni.diffusion.data import normalize_omni_diffusion_kwargs
|
from vllm_omni.diffusion.data import normalize_omni_diffusion_kwargs
|
||||||
from vllm_omni.diffusion.offloader.config import parse_diffusion_offload_config
|
from vllm_omni.diffusion.offloader.config import parse_diffusion_offload_config
|
||||||
|
|
||||||
normalized_kwargs = normalize_omni_diffusion_kwargs(kwargs)
|
config_kwargs = normalize_omni_diffusion_kwargs(kwargs)
|
||||||
# Validate before stage construction while retaining the raw mapping
|
# Validate before stage construction while retaining the raw mapping
|
||||||
# needed by dataclass/config serialization across process boundaries.
|
# needed by dataclass/config serialization across process boundaries.
|
||||||
parse_diffusion_offload_config(normalized_kwargs.get("diffusion_offload_config"))
|
parse_diffusion_offload_config(config_kwargs.get("diffusion_offload_config"))
|
||||||
valid_fields = {f.name for f in fields(cast(Any, cls))}
|
valid_fields = {f.name for f in fields(cast(Any, cls))}
|
||||||
return cls(**{k: v for k, v in normalized_kwargs.items() if k in valid_fields})
|
return cls(**{k: v for k, v in config_kwargs.items() if k in valid_fields})
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
def __post_init__(self) -> None:
|
||||||
# Keep diffusion imports lazy so importing vllm_omni.config does not
|
# Keep diffusion imports lazy so importing vllm_omni.config does not
|
||||||
@@ -1145,6 +1166,7 @@ _MODEL_ENGINE_FIELDS = frozenset(_ModelEngineOverrides.__annotations__)
|
|||||||
_LOAD_ENGINE_FIELDS = frozenset(_LoadEngineOverrides.__annotations__)
|
_LOAD_ENGINE_FIELDS = frozenset(_LoadEngineOverrides.__annotations__)
|
||||||
_CACHE_ENGINE_FIELDS = frozenset(_CacheEngineOverrides.__annotations__)
|
_CACHE_ENGINE_FIELDS = frozenset(_CacheEngineOverrides.__annotations__)
|
||||||
_SCHEDULER_ENGINE_FIELDS = frozenset(_SchedulerEngineOverrides.__annotations__)
|
_SCHEDULER_ENGINE_FIELDS = frozenset(_SchedulerEngineOverrides.__annotations__)
|
||||||
|
_POOLING_ENGINE_FIELDS = frozenset(_PoolingEngineOverrides.__annotations__)
|
||||||
_CONNECTOR_ENGINE_FIELDS = frozenset(_ConnectorEngineOverrides.__annotations__)
|
_CONNECTOR_ENGINE_FIELDS = frozenset(_ConnectorEngineOverrides.__annotations__)
|
||||||
_RUNTIME_ENGINE_FIELDS = frozenset(_RuntimeEngineOverrides.__annotations__)
|
_RUNTIME_ENGINE_FIELDS = frozenset(_RuntimeEngineOverrides.__annotations__)
|
||||||
_DIRECT_VLLM_CONFIG_ENGINE_FIELDS = frozenset({"compilation_config", "profiler_config"})
|
_DIRECT_VLLM_CONFIG_ENGINE_FIELDS = frozenset({"compilation_config", "profiler_config"})
|
||||||
@@ -1175,6 +1197,7 @@ _LLM_STAGE_ENGINE_FIELDS = (
|
|||||||
| _LLM_CACHE_ENGINE_FIELDS
|
| _LLM_CACHE_ENGINE_FIELDS
|
||||||
| _LLM_SCHEDULER_ENGINE_FIELDS
|
| _LLM_SCHEDULER_ENGINE_FIELDS
|
||||||
| _LLM_PARALLEL_CONFIG_ENGINE_FIELDS
|
| _LLM_PARALLEL_CONFIG_ENGINE_FIELDS
|
||||||
|
| _POOLING_ENGINE_FIELDS
|
||||||
| {"parallel_config"}
|
| {"parallel_config"}
|
||||||
)
|
)
|
||||||
_DIFFUSION_OWNED_STAGE_ENGINE_FIELDS = (
|
_DIFFUSION_OWNED_STAGE_ENGINE_FIELDS = (
|
||||||
@@ -1308,7 +1331,12 @@ def _stage_engine_values(
|
|||||||
if topology.execution_type == StageExecutionType.DIFFUSION:
|
if topology.execution_type == StageExecutionType.DIFFUSION:
|
||||||
# Mirror StageConfig.to_omegaconf so both projections resolve alike.
|
# Mirror StageConfig.to_omegaconf so both projections resolve alike.
|
||||||
reconcile_diffusion_attention_overrides(engine, stage_cli_overrides)
|
reconcile_diffusion_attention_overrides(engine, stage_cli_overrides)
|
||||||
engine.update(_copy_value(stage_cli_overrides))
|
for key, value in stage_cli_overrides.items():
|
||||||
|
existing = engine.get(key)
|
||||||
|
if key != "omni_kv_config" and isinstance(existing, dict) and isinstance(value, Mapping):
|
||||||
|
engine[key] = _get_recursively_merged_dict(existing, dict(value))
|
||||||
|
else:
|
||||||
|
engine[key] = _copy_value(value)
|
||||||
_validate_stage_engine_override_ownership(
|
_validate_stage_engine_override_ownership(
|
||||||
topology.stage_id,
|
topology.stage_id,
|
||||||
topology.execution_type,
|
topology.execution_type,
|
||||||
@@ -1334,6 +1362,10 @@ def _stage_engine_values(
|
|||||||
),
|
),
|
||||||
model=cast(_ModelEngineOverrides, _select_engine_overrides(engine, _MODEL_ENGINE_FIELDS)),
|
model=cast(_ModelEngineOverrides, _select_engine_overrides(engine, _MODEL_ENGINE_FIELDS)),
|
||||||
load=cast(_LoadEngineOverrides, _select_engine_overrides(engine, load_engine_fields)),
|
load=cast(_LoadEngineOverrides, _select_engine_overrides(engine, load_engine_fields)),
|
||||||
|
pooling=cast(
|
||||||
|
_PoolingEngineOverrides,
|
||||||
|
_select_engine_overrides(engine, _POOLING_ENGINE_FIELDS),
|
||||||
|
),
|
||||||
cache=cast(_CacheEngineOverrides, _select_engine_overrides(engine, cache_engine_fields)),
|
cache=cast(_CacheEngineOverrides, _select_engine_overrides(engine, cache_engine_fields)),
|
||||||
scheduler=cast(
|
scheduler=cast(
|
||||||
_SchedulerEngineOverrides,
|
_SchedulerEngineOverrides,
|
||||||
@@ -1406,6 +1438,7 @@ class BaseVllmOmniStageConfig:
|
|||||||
cache_config: OmniStageCacheConfig = field(default_factory=OmniStageCacheConfig)
|
cache_config: OmniStageCacheConfig = field(default_factory=OmniStageCacheConfig)
|
||||||
scheduler_config: OmniStageSchedulerConfig = field(default_factory=OmniStageSchedulerConfig)
|
scheduler_config: OmniStageSchedulerConfig = field(default_factory=OmniStageSchedulerConfig)
|
||||||
connector_config: OmniStageConnectorConfig = field(default_factory=OmniStageConnectorConfig)
|
connector_config: OmniStageConnectorConfig = field(default_factory=OmniStageConnectorConfig)
|
||||||
|
pooling_config: OmniStagePoolingConfig = field(default_factory=OmniStagePoolingConfig)
|
||||||
runtime_config: OmniStageRuntimeConfig = field(default_factory=OmniStageRuntimeConfig)
|
runtime_config: OmniStageRuntimeConfig = field(default_factory=OmniStageRuntimeConfig)
|
||||||
parallel_config: OmniStageParallelConfig = field(default_factory=OmniStageParallelConfig)
|
parallel_config: OmniStageParallelConfig = field(default_factory=OmniStageParallelConfig)
|
||||||
compilation_config: VllmCompilationConfig | None = None
|
compilation_config: VllmCompilationConfig | None = None
|
||||||
@@ -1490,6 +1523,10 @@ class BaseVllmOmniStageConfig:
|
|||||||
def prompt_transform_func(self) -> str | None:
|
def prompt_transform_func(self) -> str | None:
|
||||||
return self.stage_pipeline_config.prompt_transform_func
|
return self.stage_pipeline_config.prompt_transform_func
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sampling_constraints(self) -> dict[str, Any]:
|
||||||
|
return dict(self.stage_pipeline_config.sampling_constraints)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def cfg_kv_collect_func(self) -> str | None:
|
def cfg_kv_collect_func(self) -> str | None:
|
||||||
return self.stage_pipeline_config.cfg_kv_collect_func
|
return self.stage_pipeline_config.cfg_kv_collect_func
|
||||||
@@ -1543,6 +1580,7 @@ def _build_common_stage_config_kwargs(
|
|||||||
model=model,
|
model=model,
|
||||||
),
|
),
|
||||||
"load_config": _build_load_config(topology, engine.load),
|
"load_config": _build_load_config(topology, engine.load),
|
||||||
|
"pooling_config": _build_pooling_config(stage_deploy, engine.pooling),
|
||||||
"cache_config": _build_cache_config(
|
"cache_config": _build_cache_config(
|
||||||
deploy,
|
deploy,
|
||||||
engine.cache,
|
engine.cache,
|
||||||
@@ -1756,6 +1794,20 @@ def _build_model_config(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_pooling_config(
|
||||||
|
stage_deploy: StageDeployConfig | None,
|
||||||
|
engine: _PoolingEngineOverrides,
|
||||||
|
) -> OmniStagePoolingConfig:
|
||||||
|
default_pooling_params = None
|
||||||
|
if stage_deploy is not None and stage_deploy.default_pooling_params:
|
||||||
|
default_pooling_params = PoolingParams(**dict(stage_deploy.default_pooling_params))
|
||||||
|
return cast(Any, OmniStagePoolingConfig)(
|
||||||
|
runner=_copy_value(engine.get("runner")),
|
||||||
|
pooling_output_decoder=_copy_value(engine.get("pooling_output_decoder")),
|
||||||
|
default_pooling_params=default_pooling_params,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _build_load_config(
|
def _build_load_config(
|
||||||
topology: StagePipelineConfig,
|
topology: StagePipelineConfig,
|
||||||
engine: _LoadEngineOverrides,
|
engine: _LoadEngineOverrides,
|
||||||
@@ -1934,6 +1986,10 @@ class VllmOmniConfig:
|
|||||||
pipeline_config: PipelineConfig
|
pipeline_config: PipelineConfig
|
||||||
stage_configs: tuple[StageConfigType, ...]
|
stage_configs: tuple[StageConfigType, ...]
|
||||||
orchestrator_config: VllmOmniOrchestratorConfig = field(default_factory=VllmOmniOrchestratorConfig)
|
orchestrator_config: VllmOmniOrchestratorConfig = field(default_factory=VllmOmniOrchestratorConfig)
|
||||||
|
# Keep strategy provenance separate from the effective orchestrator value:
|
||||||
|
# an explicit CLI policy may be present even when the strategy declares no
|
||||||
|
# stage-replica axis.
|
||||||
|
strategy_omni_lb_policy: str | None = None
|
||||||
|
|
||||||
def stage_by_id(self, stage_id: int) -> StageConfigType:
|
def stage_by_id(self, stage_id: int) -> StageConfigType:
|
||||||
for stage in self.stage_configs:
|
for stage in self.stage_configs:
|
||||||
@@ -1949,6 +2005,7 @@ class VllmOmniConfig:
|
|||||||
user_deploy_config: DeployConfig | None = None,
|
user_deploy_config: DeployConfig | None = None,
|
||||||
deploy_config_path: str | None = None,
|
deploy_config_path: str | None = None,
|
||||||
cli_overrides: dict[str, Any] | None = None,
|
cli_overrides: dict[str, Any] | None = None,
|
||||||
|
strategy_specs: Mapping[Any, Any] | None = None,
|
||||||
) -> VllmOmniConfig:
|
) -> VllmOmniConfig:
|
||||||
"""Create a structured config from a resolved pipeline and deploy YAML."""
|
"""Create a structured config from a resolved pipeline and deploy YAML."""
|
||||||
if cli_overrides is None:
|
if cli_overrides is None:
|
||||||
@@ -1973,6 +2030,62 @@ class VllmOmniConfig:
|
|||||||
deploy.async_chunk = False
|
deploy.async_chunk = False
|
||||||
_validate_async_chunk_support(pipeline_cfg, deploy)
|
_validate_async_chunk_support(pipeline_cfg, deploy)
|
||||||
validate_stage_async_chunk_edges(pipeline_cfg, deploy)
|
validate_stage_async_chunk_edges(pipeline_cfg, deploy)
|
||||||
|
|
||||||
|
strategy_result = None
|
||||||
|
if strategy_specs:
|
||||||
|
from vllm_omni.config.composable_parallel import apply_strategy_specs
|
||||||
|
|
||||||
|
strategy_stages = merge_pipeline_deploy(pipeline_cfg, copy.deepcopy(deploy), {})
|
||||||
|
strategy_result = apply_strategy_specs(strategy_stages, strategy_specs)
|
||||||
|
strategy_overrides: dict[str, Any] = {}
|
||||||
|
axis_fields = {
|
||||||
|
"tp": "tensor_parallel_size",
|
||||||
|
"dp": "data_parallel_size",
|
||||||
|
"pp": "pipeline_parallel_size",
|
||||||
|
}
|
||||||
|
for stage_id, parallel in strategy_result.per_stage_config.items():
|
||||||
|
declared = set(parallel.l1_owners)
|
||||||
|
explicit = build_stage_runtime_overrides(stage_id, cli_overrides)
|
||||||
|
for axis, field_name in axis_fields.items():
|
||||||
|
if axis not in declared:
|
||||||
|
continue
|
||||||
|
derived = getattr(parallel, field_name)
|
||||||
|
value = explicit.get(field_name)
|
||||||
|
if value is None:
|
||||||
|
value = derived
|
||||||
|
elif value != derived:
|
||||||
|
logger.warning(
|
||||||
|
"[composable_parallel] stage %s: CLI %s=%s overrides the "
|
||||||
|
"strategy-derived %s=%s. The CLI value wins; remove one to avoid ambiguity.",
|
||||||
|
stage_id,
|
||||||
|
field_name,
|
||||||
|
value,
|
||||||
|
field_name,
|
||||||
|
derived,
|
||||||
|
)
|
||||||
|
strategy_overrides[f"stage_{stage_id}_{field_name}"] = value
|
||||||
|
if "ep" in declared and parallel.enable_expert_parallel:
|
||||||
|
strategy_overrides[f"stage_{stage_id}_enable_expert_parallel"] = bool(
|
||||||
|
explicit.get("enable_expert_parallel", True)
|
||||||
|
)
|
||||||
|
if "stage_replica" in declared:
|
||||||
|
derived = parallel.stage_replica_size
|
||||||
|
value = explicit.get("num_replicas")
|
||||||
|
if value is None:
|
||||||
|
value = derived
|
||||||
|
elif value != derived:
|
||||||
|
logger.warning(
|
||||||
|
"[composable_parallel] stage %s: CLI num_replicas=%s overrides the "
|
||||||
|
"strategy-derived num_replicas=%s. The CLI value wins; remove one to avoid ambiguity.",
|
||||||
|
stage_id,
|
||||||
|
value,
|
||||||
|
derived,
|
||||||
|
)
|
||||||
|
strategy_overrides[f"stage_{stage_id}_num_replicas"] = value
|
||||||
|
cli_overrides = {**strategy_overrides, **cli_overrides}
|
||||||
|
if strategy_result.omni_lb_policy is not None and cli_overrides.get("omni_lb_policy") is None:
|
||||||
|
cli_overrides["omni_lb_policy"] = strategy_result.omni_lb_policy
|
||||||
|
|
||||||
deploy_by_id = {stage.stage_id: stage for stage in deploy.stages}
|
deploy_by_id = {stage.stage_id: stage for stage in deploy.stages}
|
||||||
model = cli_overrides.get("model")
|
model = cli_overrides.get("model")
|
||||||
|
|
||||||
@@ -1995,6 +2108,20 @@ class VllmOmniConfig:
|
|||||||
)
|
)
|
||||||
for topology in pipeline_cfg.stages
|
for topology in pipeline_cfg.stages
|
||||||
)
|
)
|
||||||
|
if strategy_result is not None:
|
||||||
|
from vllm_omni.config.composable_parallel import check_device_layout
|
||||||
|
|
||||||
|
by_id = {stage.stage_id: stage for stage in stage_configs}
|
||||||
|
for stage_id in strategy_result.per_stage_config:
|
||||||
|
stage = by_id[stage_id]
|
||||||
|
check_device_layout(
|
||||||
|
stage.runtime_config.devices,
|
||||||
|
tensor_parallel_size=stage.parallel_config.tensor_parallel_size,
|
||||||
|
data_parallel_size=stage.parallel_config.data_parallel_size,
|
||||||
|
pipeline_parallel_size=stage.parallel_config.pipeline_parallel_size,
|
||||||
|
num_replicas=stage.runtime_config.num_replicas,
|
||||||
|
role=stage.model_stage,
|
||||||
|
)
|
||||||
|
|
||||||
orchestrator_config = cast(Any, VllmOmniOrchestratorConfig)(
|
orchestrator_config = cast(Any, VllmOmniOrchestratorConfig)(
|
||||||
deploy_config_path=loaded_deploy_config_path,
|
deploy_config_path=loaded_deploy_config_path,
|
||||||
@@ -2004,6 +2131,7 @@ class VllmOmniConfig:
|
|||||||
pipeline_config=pipeline_cfg,
|
pipeline_config=pipeline_cfg,
|
||||||
stage_configs=stage_configs,
|
stage_configs=stage_configs,
|
||||||
orchestrator_config=orchestrator_config,
|
orchestrator_config=orchestrator_config,
|
||||||
|
strategy_omni_lb_policy=(strategy_result.omni_lb_policy if strategy_result is not None else None),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ from vllm_omni.config.config_factory import StageConfigFactory, with_trust_remot
|
|||||||
from vllm_omni.config.endpoint_policy import EndpointRestriction
|
from vllm_omni.config.endpoint_policy import EndpointRestriction
|
||||||
from vllm_omni.config.omni_config import VllmOmniConfig
|
from vllm_omni.config.omni_config import VllmOmniConfig
|
||||||
from vllm_omni.config.stage_config import PipelineConfig
|
from vllm_omni.config.stage_config import PipelineConfig
|
||||||
from vllm_omni.config.yaml_util import create_config
|
|
||||||
from vllm_omni.diffusion.data import resolve_model_class_name
|
from vllm_omni.diffusion.data import resolve_model_class_name
|
||||||
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
from vllm_omni.diffusion.registry import DiffusionModelRegistry
|
||||||
from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
|
from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model
|
||||||
@@ -24,19 +23,10 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class OmniConfigResolution:
|
class OmniConfigResolution:
|
||||||
"""Migration envelope returned by the production config resolver.
|
"""Canonical typed result returned by the production config resolver."""
|
||||||
|
|
||||||
``stage_configs`` intentionally carries the current OmegaConf-compatible
|
|
||||||
runtime ABI only until stage startup consumes ``VllmOmniConfig`` directly.
|
|
||||||
``pipeline_config`` is the effective topology after runtime stage injection,
|
|
||||||
so both fields always describe the same set of stages.
|
|
||||||
It is not a stable authoring or extension API; new production callers
|
|
||||||
should resolve through :func:`resolve_omni_config` and must not construct or
|
|
||||||
merge this compatibility shape themselves.
|
|
||||||
"""
|
|
||||||
|
|
||||||
config_path: str | None
|
config_path: str | None
|
||||||
stage_configs: tuple[Any, ...] # Temporary StageConfig/OmegaConf bridge.
|
stage_configs: tuple[Any, ...]
|
||||||
pipeline_config: PipelineConfig | None = None
|
pipeline_config: PipelineConfig | None = None
|
||||||
omni_lb_policy: str | None = None
|
omni_lb_policy: str | None = None
|
||||||
|
|
||||||
@@ -176,27 +166,13 @@ def _load_strategy_specs(strategy_config_path: str | None) -> Mapping[Any, Any]
|
|||||||
|
|
||||||
def _build_registered_resolution(
|
def _build_registered_resolution(
|
||||||
structured_config: VllmOmniConfig,
|
structured_config: VllmOmniConfig,
|
||||||
*,
|
|
||||||
cli_overrides: dict[str, Any],
|
|
||||||
strategy_config_path: str | None,
|
|
||||||
) -> OmniConfigResolution:
|
) -> OmniConfigResolution:
|
||||||
"""Build the temporary runtime view for an already-resolved pipeline."""
|
"""Return the canonical structured runtime view for a resolved pipeline."""
|
||||||
# Runtime consumers have not yet moved to typed per-stage configs. Use the
|
|
||||||
# factory-owned compatibility bridge instead of reimplementing legacy YAML
|
|
||||||
# discovery and merging in this resolver.
|
|
||||||
effective_deploy_path = structured_config.orchestrator_config.deploy_config_path
|
|
||||||
legacy_resolution = StageConfigFactory._resolve_legacy_from_registry(
|
|
||||||
structured_config.pipeline_config,
|
|
||||||
cli_overrides,
|
|
||||||
effective_deploy_path,
|
|
||||||
strategy_specs=_load_strategy_specs(strategy_config_path),
|
|
||||||
)
|
|
||||||
|
|
||||||
return OmniConfigResolution(
|
return OmniConfigResolution(
|
||||||
config_path=effective_deploy_path,
|
config_path=structured_config.orchestrator_config.deploy_config_path,
|
||||||
stage_configs=tuple(stage.to_omegaconf() for stage in legacy_resolution.stage_configs),
|
stage_configs=tuple(structured_config.stage_configs),
|
||||||
pipeline_config=legacy_resolution.pipeline_config,
|
pipeline_config=structured_config.pipeline_config,
|
||||||
omni_lb_policy=legacy_resolution.omni_lb_policy,
|
omni_lb_policy=getattr(structured_config, "strategy_omni_lb_policy", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -231,18 +207,16 @@ def resolve_omni_config(
|
|||||||
registry_overrides = dict(normalized_overrides)
|
registry_overrides = dict(normalized_overrides)
|
||||||
_flatten_stage_overrides(registry_overrides, stage_overrides)
|
_flatten_stage_overrides(registry_overrides, stage_overrides)
|
||||||
|
|
||||||
|
strategy_specs = _load_strategy_specs(strategy_config_path)
|
||||||
structured_config = StageConfigFactory.create_from_model(
|
structured_config = StageConfigFactory.create_from_model(
|
||||||
model,
|
model,
|
||||||
trust_remote_code=trust_remote_code,
|
trust_remote_code=trust_remote_code,
|
||||||
cli_overrides=registry_overrides,
|
cli_overrides=registry_overrides,
|
||||||
deploy_config_path=deploy_config_path,
|
deploy_config_path=deploy_config_path,
|
||||||
|
strategy_specs=strategy_specs,
|
||||||
)
|
)
|
||||||
if structured_config is not None:
|
if structured_config is not None:
|
||||||
return _build_registered_resolution(
|
return _build_registered_resolution(structured_config)
|
||||||
structured_config,
|
|
||||||
cli_overrides=registry_overrides,
|
|
||||||
strategy_config_path=strategy_config_path,
|
|
||||||
)
|
|
||||||
|
|
||||||
_apply_generic_stage_overrides(normalized_overrides, stage_overrides)
|
_apply_generic_stage_overrides(normalized_overrides, stage_overrides)
|
||||||
supported, model_class_name = _resolve_generic_diffusion_model_class(model, normalized_overrides)
|
supported, model_class_name = _resolve_generic_diffusion_model_class(model, normalized_overrides)
|
||||||
@@ -252,11 +226,12 @@ def resolve_omni_config(
|
|||||||
)
|
)
|
||||||
if model_class_name is not None:
|
if model_class_name is not None:
|
||||||
normalized_overrides.setdefault("model_class_name", model_class_name)
|
normalized_overrides.setdefault("model_class_name", model_class_name)
|
||||||
default_stages = StageConfigFactory.create_default_diffusion(normalized_overrides)
|
default_config = StageConfigFactory.create_typed_default_diffusion(model, normalized_overrides)
|
||||||
|
|
||||||
return OmniConfigResolution(
|
return OmniConfigResolution(
|
||||||
config_path=deploy_config_path,
|
config_path=deploy_config_path,
|
||||||
stage_configs=tuple(create_config(_convert_dataclasses_to_dict(default_stages))),
|
stage_configs=tuple(default_config.stage_configs),
|
||||||
|
pipeline_config=default_config.pipeline_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -272,7 +272,14 @@ class OmniEngineBase:
|
|||||||
|
|
||||||
self.num_stages = len(self.stage_configs)
|
self.num_stages = len(self.stage_configs)
|
||||||
self.async_chunk = any(
|
self.async_chunk = any(
|
||||||
bool(getattr(getattr(stage, "engine_args", None), "async_chunk", False)) for stage in self.stage_configs
|
bool(
|
||||||
|
getattr(
|
||||||
|
getattr(stage, "connector_config", None),
|
||||||
|
"async_chunk",
|
||||||
|
getattr(getattr(stage, "engine_args", None), "async_chunk", False),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for stage in self.stage_configs
|
||||||
)
|
)
|
||||||
self.stage_pools: list[StagePool] = []
|
self.stage_pools: list[StagePool] = []
|
||||||
self.stage_clients: list[StageClient] = [] # logical-stage view for external readers
|
self.stage_clients: list[StageClient] = [] # logical-stage view for external readers
|
||||||
|
|||||||
@@ -287,8 +287,13 @@ def check_admission(
|
|||||||
|
|
||||||
|
|
||||||
def _diffusion_utilization(replica: Any) -> float | None:
|
def _diffusion_utilization(replica: Any) -> float | None:
|
||||||
"""Best-effort gpu_memory_utilization for a diffusion replica from raw args."""
|
"""Resolve a diffusion replica's memory budget from typed or legacy config."""
|
||||||
stage_cfg = getattr(replica, "stage_cfg", None)
|
stage_cfg = getattr(replica, "stage_cfg", None)
|
||||||
|
cache_config = getattr(stage_cfg, "cache_config", None)
|
||||||
|
util = getattr(cache_config, "gpu_memory_utilization", None)
|
||||||
|
if util is not None:
|
||||||
|
return float(util)
|
||||||
|
|
||||||
engine_args = getattr(stage_cfg, "engine_args", None)
|
engine_args = getattr(stage_cfg, "engine_args", None)
|
||||||
if isinstance(engine_args, dict):
|
if isinstance(engine_args, dict):
|
||||||
util = engine_args.get("gpu_memory_utilization")
|
util = engine_args.get("gpu_memory_utilization")
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from types import SimpleNamespace
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
|
import torch
|
||||||
import zmq
|
import zmq
|
||||||
from omegaconf import OmegaConf
|
from omegaconf import OmegaConf
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
@@ -33,6 +34,7 @@ from vllm.v1.engine.utils import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.executor import Executor
|
from vllm.v1.executor import Executor
|
||||||
|
|
||||||
|
from vllm_omni.config.omni_config import BaseVllmOmniStageConfig
|
||||||
from vllm_omni.distributed.omni_connectors.utils import initialization
|
from vllm_omni.distributed.omni_connectors.utils import initialization
|
||||||
from vllm_omni.engine import stage_init_utils
|
from vllm_omni.engine import stage_init_utils
|
||||||
from vllm_omni.engine.stage_init_utils import (
|
from vllm_omni.engine.stage_init_utils import (
|
||||||
@@ -67,11 +69,13 @@ def _serialize_stage_config(stage_config: Any) -> Any:
|
|||||||
"""Convert a stage config to msgpack-friendly builtins."""
|
"""Convert a stage config to msgpack-friendly builtins."""
|
||||||
if stage_config is None or isinstance(stage_config, (str, bytes, int, float, bool)):
|
if stage_config is None or isinstance(stage_config, (str, bytes, int, float, bool)):
|
||||||
return stage_config
|
return stage_config
|
||||||
|
if isinstance(stage_config, torch.dtype):
|
||||||
|
return str(stage_config).removeprefix("torch.")
|
||||||
|
|
||||||
if OmegaConf.is_config(stage_config):
|
if OmegaConf.is_config(stage_config):
|
||||||
return _serialize_stage_config(OmegaConf.to_container(stage_config, resolve=True))
|
return _serialize_stage_config(OmegaConf.to_container(stage_config, resolve=True))
|
||||||
|
|
||||||
if dataclasses.is_dataclass(stage_config):
|
if dataclasses.is_dataclass(stage_config) and not isinstance(stage_config, type):
|
||||||
return _serialize_stage_config(dataclasses.asdict(stage_config))
|
return _serialize_stage_config(dataclasses.asdict(stage_config))
|
||||||
|
|
||||||
if isinstance(stage_config, dict):
|
if isinstance(stage_config, dict):
|
||||||
@@ -1403,7 +1407,7 @@ def get_headless_replica_devices(
|
|||||||
omni_dp_size_local: int,
|
omni_dp_size_local: int,
|
||||||
) -> list[str | None]:
|
) -> list[str | None]:
|
||||||
"""Return per-replica device slices for a headless stage."""
|
"""Return per-replica device slices for a headless stage."""
|
||||||
runtime_cfg = getattr(stage_cfg, "runtime", None)
|
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", None))
|
||||||
devices_str: str | None = None
|
devices_str: str | None = None
|
||||||
if runtime_cfg is not None:
|
if runtime_cfg is not None:
|
||||||
devices_str = (
|
devices_str = (
|
||||||
@@ -1509,10 +1513,11 @@ def launch_headless_diffusion_replicas(
|
|||||||
stage_id,
|
stage_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Headless diffusion startup and its downstream helpers still consume the
|
metadata = (
|
||||||
# legacy StageConfig shape; switch this with the coordinated RFC #4021
|
stage_init_utils.extract_stage_metadata_from_omni_stage_config(stage_cfg)
|
||||||
# stage-init cutover.
|
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
|
||||||
metadata = stage_init_utils.extract_legacy_stage_metadata(stage_cfg)
|
else stage_init_utils.extract_legacy_stage_metadata(stage_cfg)
|
||||||
|
)
|
||||||
if omni_conn_cfg:
|
if omni_conn_cfg:
|
||||||
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
|
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
|
||||||
# Headless single-stage launch must still infer cross-stage TP topology
|
# Headless single-stage launch must still infer cross-stage TP topology
|
||||||
|
|||||||
@@ -402,6 +402,11 @@ def _tp_size_for_stage(stage_configs: Sequence[Any], stage_id: Any) -> int | Non
|
|||||||
for stage_cfg in stage_configs:
|
for stage_cfg in stage_configs:
|
||||||
if str(getattr(stage_cfg, "stage_id", None)) not in id_strs:
|
if str(getattr(stage_cfg, "stage_id", None)) not in id_strs:
|
||||||
continue
|
continue
|
||||||
|
if isinstance(stage_cfg, BaseVllmOmniStageConfig):
|
||||||
|
try:
|
||||||
|
return max(1, int(stage_cfg.parallel_config.tensor_parallel_size))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 1
|
||||||
engine_args = getattr(stage_cfg, "engine_args", None)
|
engine_args = getattr(stage_cfg, "engine_args", None)
|
||||||
if engine_args is None:
|
if engine_args is None:
|
||||||
return 1
|
return 1
|
||||||
@@ -489,11 +494,15 @@ def inject_kv_stage_info(stage_cfg: Any, stage_id: int, stage_configs: Sequence[
|
|||||||
rank mappings automatically.
|
rank mappings automatically.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
engine_args = stage_cfg.engine_args
|
connector_config = getattr(stage_cfg, "connector_config", None)
|
||||||
if hasattr(engine_args, "get"):
|
if connector_config is not None:
|
||||||
omni_kv = engine_args.get("omni_kv_config", None)
|
omni_kv = connector_config.omni_kv_config
|
||||||
else:
|
else:
|
||||||
omni_kv = getattr(engine_args, "omni_kv_config", None)
|
engine_args = stage_cfg.engine_args
|
||||||
|
if hasattr(engine_args, "get"):
|
||||||
|
omni_kv = engine_args.get("omni_kv_config", None)
|
||||||
|
else:
|
||||||
|
omni_kv = getattr(engine_args, "omni_kv_config", None)
|
||||||
|
|
||||||
if omni_kv is None:
|
if omni_kv is None:
|
||||||
return
|
return
|
||||||
@@ -504,7 +513,11 @@ def inject_kv_stage_info(stage_cfg: Any, stage_id: int, stage_configs: Sequence[
|
|||||||
if "stage_id" not in omni_kv:
|
if "stage_id" not in omni_kv:
|
||||||
omni_kv["stage_id"] = stage_id
|
omni_kv["stage_id"] = stage_id
|
||||||
|
|
||||||
engine_input_source = getattr(stage_cfg, "engine_input_source", None)
|
engine_input_source = (
|
||||||
|
getattr(stage_cfg, "input_sources", None)
|
||||||
|
if connector_config is not None
|
||||||
|
else getattr(stage_cfg, "engine_input_source", None)
|
||||||
|
)
|
||||||
if engine_input_source is not None:
|
if engine_input_source is not None:
|
||||||
if hasattr(omni_kv, "setdefault"):
|
if hasattr(omni_kv, "setdefault"):
|
||||||
omni_kv.setdefault("engine_input_source", list(engine_input_source))
|
omni_kv.setdefault("engine_input_source", list(engine_input_source))
|
||||||
@@ -703,17 +716,16 @@ def _resolve_omni_metadata_hook(path: str | None) -> Callable | None:
|
|||||||
def extract_stage_metadata_from_omni_stage_config(
|
def extract_stage_metadata_from_omni_stage_config(
|
||||||
stage_config: BaseVllmOmniStageConfig,
|
stage_config: BaseVllmOmniStageConfig,
|
||||||
) -> StageMetadata:
|
) -> StageMetadata:
|
||||||
"""Project one typed stage config into metadata for a future cutover.
|
"""Project one typed stage config into production runtime metadata."""
|
||||||
|
|
||||||
This projection is not used by production startup yet. Current replica
|
|
||||||
layout, engine-argument, remote-diffusion, and platform setup paths still
|
|
||||||
require the legacy StageConfig/OmegaConf shape.
|
|
||||||
"""
|
|
||||||
stage_type: Literal["llm", "diffusion"] = "diffusion" if stage_config.stage_type == StageType.DIFFUSION else "llm"
|
stage_type: Literal["llm", "diffusion"] = "diffusion" if stage_config.stage_type == StageType.DIFFUSION else "llm"
|
||||||
sampling_params_cls = SamplingParams if stage_type == "llm" else OmniDiffusionSamplingParams
|
pooling_config = stage_config.pooling_config
|
||||||
sampling_params: OmniSamplingParams = sampling_params_cls(
|
if stage_type == "llm" and (pooling_config.runner or "").lower() == "pooling":
|
||||||
**(stage_config.model_config.default_sampling_params or {})
|
sampling_params: OmniSamplingParams | PoolingParams = (
|
||||||
)
|
copy.deepcopy(pooling_config.default_pooling_params) or PoolingParams()
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sampling_params_cls = SamplingParams if stage_type == "llm" else OmniDiffusionSamplingParams
|
||||||
|
sampling_params = sampling_params_cls(**(stage_config.model_config.default_sampling_params or {}))
|
||||||
custom_process_input_func = _resolve_omni_metadata_hook(stage_config.custom_process_input_func)
|
custom_process_input_func = _resolve_omni_metadata_hook(stage_config.custom_process_input_func)
|
||||||
|
|
||||||
if stage_type == "diffusion":
|
if stage_type == "diffusion":
|
||||||
@@ -878,7 +890,9 @@ def _get_local_llm_parallel_sizes(
|
|||||||
unset.
|
unset.
|
||||||
"""
|
"""
|
||||||
if engine_args is None:
|
if engine_args is None:
|
||||||
engine_args = getattr(stage_cfg, "engine_args", {})
|
engine_args = getattr(stage_cfg, "parallel_config", None)
|
||||||
|
if engine_args is None:
|
||||||
|
engine_args = getattr(stage_cfg, "engine_args", {})
|
||||||
tp_size = int(_get_attr_or_item(engine_args, "tensor_parallel_size", 1) or 1)
|
tp_size = int(_get_attr_or_item(engine_args, "tensor_parallel_size", 1) or 1)
|
||||||
pp_size = int(_get_attr_or_item(engine_args, "pipeline_parallel_size", 1) or 1)
|
pp_size = int(_get_attr_or_item(engine_args, "pipeline_parallel_size", 1) or 1)
|
||||||
local_dp_size = _get_attr_or_item(engine_args, "data_parallel_size_local", None)
|
local_dp_size = _get_attr_or_item(engine_args, "data_parallel_size_local", None)
|
||||||
@@ -889,9 +903,12 @@ def _get_local_llm_parallel_sizes(
|
|||||||
|
|
||||||
def get_stage_devices_per_replica(stage_cfg: Any, engine_args: Any | None = None) -> int:
|
def get_stage_devices_per_replica(stage_cfg: Any, engine_args: Any | None = None) -> int:
|
||||||
"""Return the number of devices consumed by one replica of *stage_cfg*."""
|
"""Return the number of devices consumed by one replica of *stage_cfg*."""
|
||||||
if engine_args is None:
|
|
||||||
engine_args = getattr(stage_cfg, "engine_args", {})
|
|
||||||
if getattr(stage_cfg, "stage_type", "llm") == "diffusion":
|
if getattr(stage_cfg, "stage_type", "llm") == "diffusion":
|
||||||
|
typed_parallel_config = getattr(stage_cfg, "parallel_config", None)
|
||||||
|
if typed_parallel_config is not None:
|
||||||
|
return max(1, int(typed_parallel_config.world_size))
|
||||||
|
if engine_args is None:
|
||||||
|
engine_args = getattr(stage_cfg, "engine_args", {})
|
||||||
parallel_config = _get_attr_or_item(engine_args, "parallel_config")
|
parallel_config = _get_attr_or_item(engine_args, "parallel_config")
|
||||||
if parallel_config is None:
|
if parallel_config is None:
|
||||||
return 1
|
return 1
|
||||||
@@ -933,7 +950,7 @@ def compute_replica_layout(
|
|||||||
"""
|
"""
|
||||||
replicas_per_stage: list[int] = []
|
replicas_per_stage: list[int] = []
|
||||||
for stage_cfg in stage_configs:
|
for stage_cfg in stage_configs:
|
||||||
runtime_cfg = getattr(stage_cfg, "runtime", {})
|
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", {}))
|
||||||
num_replicas = int(
|
num_replicas = int(
|
||||||
runtime_cfg.get("num_replicas", 1)
|
runtime_cfg.get("num_replicas", 1)
|
||||||
if hasattr(runtime_cfg, "get")
|
if hasattr(runtime_cfg, "get")
|
||||||
@@ -948,7 +965,7 @@ def compute_replica_layout(
|
|||||||
num_replicas = replicas_per_stage[stage_id]
|
num_replicas = replicas_per_stage[stage_id]
|
||||||
if num_replicas <= 1:
|
if num_replicas <= 1:
|
||||||
continue
|
continue
|
||||||
runtime_cfg = getattr(stage_cfg, "runtime", {})
|
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", {}))
|
||||||
devices_str = (
|
devices_str = (
|
||||||
runtime_cfg.get("devices") if hasattr(runtime_cfg, "get") else getattr(runtime_cfg, "devices", None)
|
runtime_cfg.get("devices") if hasattr(runtime_cfg, "get") else getattr(runtime_cfg, "devices", None)
|
||||||
)
|
)
|
||||||
@@ -1086,15 +1103,31 @@ def _project_omni_stage_engine_args(
|
|||||||
diffusion_stage = cast(VllmOmniDiffusionStageConfig, stage_config)
|
diffusion_stage = cast(VllmOmniDiffusionStageConfig, stage_config)
|
||||||
engine_args.update(_project_omni_config_fields(diffusion_stage.diffusion_config))
|
engine_args.update(_project_omni_config_fields(diffusion_stage.diffusion_config))
|
||||||
|
|
||||||
|
model_excluded_fields = {
|
||||||
|
"default_sampling_params",
|
||||||
|
"has_sampling_extra_args",
|
||||||
|
}
|
||||||
|
runtime_excluded_fields = {"devices", "num_replicas", "env", "num_gpus"}
|
||||||
|
if not is_diffusion:
|
||||||
|
# These values configure OmniDiffusionConfig or its worker process;
|
||||||
|
# OmniEngineArgs has no matching fields for LLM stages.
|
||||||
|
model_excluded_fields.update(
|
||||||
|
{
|
||||||
|
"disable_autocast",
|
||||||
|
"enable_multithread_weight_load",
|
||||||
|
"num_weight_load_threads",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
runtime_excluded_fields.add("log_level")
|
||||||
|
|
||||||
for config, excluded_fields in (
|
for config, excluded_fields in (
|
||||||
(
|
(
|
||||||
stage_config.model_config,
|
stage_config.model_config,
|
||||||
frozenset({"default_sampling_params", "has_sampling_extra_args"}),
|
frozenset(model_excluded_fields),
|
||||||
),
|
),
|
||||||
(
|
(
|
||||||
stage_config.runtime_config,
|
stage_config.runtime_config,
|
||||||
frozenset({"devices", "num_replicas", "env", "num_gpus"})
|
frozenset(runtime_excluded_fields) | (frozenset({"additional_config"}) if is_diffusion else frozenset()),
|
||||||
| (frozenset({"additional_config"}) if is_diffusion else frozenset()),
|
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
engine_args.update(
|
engine_args.update(
|
||||||
@@ -1103,6 +1136,11 @@ def _project_omni_stage_engine_args(
|
|||||||
exclude=excluded_fields,
|
exclude=excluded_fields,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
pooling_config = stage_config.pooling_config
|
||||||
|
if pooling_config.runner is not None:
|
||||||
|
engine_args["runner"] = pooling_config.runner
|
||||||
|
if pooling_config.pooling_output_decoder is not None:
|
||||||
|
engine_args["pooling_output_decoder"] = pooling_config.pooling_output_decoder
|
||||||
|
|
||||||
if is_diffusion:
|
if is_diffusion:
|
||||||
for config, field_map in (
|
for config, field_map in (
|
||||||
@@ -1330,11 +1368,9 @@ def build_engine_args_dict(
|
|||||||
stage_connector_spec: dict[str, Any] | None = None,
|
stage_connector_spec: dict[str, Any] | None = None,
|
||||||
cli_tokenizer: str | None = None,
|
cli_tokenizer: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Build engine arguments through the stable production entry point.
|
"""Preserve the legacy engine-argument API for compatibility callers.
|
||||||
|
|
||||||
Production inputs still use the legacy stage representation. Keep that
|
Typed runtime stages use ``build_engine_args_dict_from_omni_stage_config``.
|
||||||
compatibility choice behind this function so callers do not bind directly
|
|
||||||
to a representation-specific implementation.
|
|
||||||
"""
|
"""
|
||||||
return build_legacy_engine_args_dict(
|
return build_legacy_engine_args_dict(
|
||||||
stage_config,
|
stage_config,
|
||||||
@@ -1350,13 +1386,7 @@ def build_engine_args_dict_from_omni_stage_config(
|
|||||||
stage_connector_spec: dict[str, Any] | None = None,
|
stage_connector_spec: dict[str, Any] | None = None,
|
||||||
cli_tokenizer: str | None = None,
|
cli_tokenizer: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Project one typed stage config into backend engine arguments.
|
"""Project one typed production stage config into backend engine arguments."""
|
||||||
|
|
||||||
This projection is prepared for the RFC #4021 stage-init cutover. Current
|
|
||||||
production startup reaches the legacy implementation through
|
|
||||||
``build_engine_args_dict`` while strategy and startup-plan inputs still
|
|
||||||
use the legacy representation.
|
|
||||||
"""
|
|
||||||
engine_args_dict = _project_omni_stage_engine_args(stage_config)
|
engine_args_dict = _project_omni_stage_engine_args(stage_config)
|
||||||
_apply_rocm_attention_backend(engine_args_dict, stage_config.stage_type)
|
_apply_rocm_attention_backend(engine_args_dict, stage_config.stage_type)
|
||||||
return _finalize_engine_args_dict(
|
return _finalize_engine_args_dict(
|
||||||
@@ -1391,7 +1421,7 @@ def _check_stage_device_layout(stage_config: Any, engine_args_dict: dict[str, An
|
|||||||
"""
|
"""
|
||||||
from vllm_omni.config.composable_parallel import StrategyApplyError, check_device_layout
|
from vllm_omni.config.composable_parallel import StrategyApplyError, check_device_layout
|
||||||
|
|
||||||
runtime = getattr(stage_config, "runtime", None)
|
runtime = getattr(stage_config, "runtime_config", getattr(stage_config, "runtime", None))
|
||||||
devices = _get_attr_or_item(runtime, "devices", None) if runtime is not None else None
|
devices = _get_attr_or_item(runtime, "devices", None) if runtime is not None else None
|
||||||
if devices is None:
|
if devices is None:
|
||||||
# No explicit placement -> vLLM assigns devices itself; nothing to check.
|
# No explicit placement -> vLLM assigns devices itself; nothing to check.
|
||||||
@@ -1452,10 +1482,12 @@ def build_vllm_config(
|
|||||||
(vllm_config, executor_class)
|
(vllm_config, executor_class)
|
||||||
"""
|
"""
|
||||||
if engine_args_dict is None:
|
if engine_args_dict is None:
|
||||||
engine_args_dict = build_engine_args_dict(
|
engine_args_dict = (
|
||||||
stage_config,
|
build_engine_args_dict_from_omni_stage_config(
|
||||||
model,
|
stage_config, model, stage_connector_spec=stage_connector_spec
|
||||||
stage_connector_spec=stage_connector_spec,
|
)
|
||||||
|
if isinstance(stage_config, BaseVllmOmniStageConfig)
|
||||||
|
else build_engine_args_dict(stage_config, model, stage_connector_spec=stage_connector_spec)
|
||||||
)
|
)
|
||||||
|
|
||||||
filtered_engine_args_dict = filter_dataclass_kwargs(OmniEngineArgs, engine_args_dict)
|
filtered_engine_args_dict = filter_dataclass_kwargs(OmniEngineArgs, engine_args_dict)
|
||||||
@@ -1927,7 +1959,11 @@ def build_diffusion_config(
|
|||||||
) -> Any:
|
) -> Any:
|
||||||
"""Build diffusion config for a stage."""
|
"""Build diffusion config for a stage."""
|
||||||
|
|
||||||
engine_args_dict = build_engine_args_dict(stage_cfg, model)
|
engine_args_dict = (
|
||||||
|
build_engine_args_dict_from_omni_stage_config(stage_cfg, model)
|
||||||
|
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
|
||||||
|
else build_engine_args_dict(stage_cfg, model)
|
||||||
|
)
|
||||||
od_config = OmniDiffusionConfig.from_kwargs(**engine_args_dict)
|
od_config = OmniDiffusionConfig.from_kwargs(**engine_args_dict)
|
||||||
|
|
||||||
num_devices_per_stage = od_config.parallel_config.world_size
|
num_devices_per_stage = od_config.parallel_config.world_size
|
||||||
|
|||||||
@@ -16,9 +16,9 @@ from dataclasses import dataclass
|
|||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
import janus
|
import janus
|
||||||
from omegaconf import OmegaConf
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
|
from vllm_omni.config.omni_config import BaseVllmOmniStageConfig
|
||||||
from vllm_omni.distributed.omni_connectors.utils.initialization import (
|
from vllm_omni.distributed.omni_connectors.utils.initialization import (
|
||||||
resolve_omni_kv_config_for_stage,
|
resolve_omni_kv_config_for_stage,
|
||||||
)
|
)
|
||||||
@@ -49,11 +49,13 @@ from vllm_omni.engine.stage_init_utils import (
|
|||||||
_inject_inferred_kv_tp_topology,
|
_inject_inferred_kv_tp_topology,
|
||||||
acquire_device_locks,
|
acquire_device_locks,
|
||||||
build_engine_args_dict,
|
build_engine_args_dict,
|
||||||
|
build_engine_args_dict_from_omni_stage_config,
|
||||||
build_llm_stage_output_processor,
|
build_llm_stage_output_processor,
|
||||||
build_vllm_config,
|
build_vllm_config,
|
||||||
compute_replica_layout,
|
compute_replica_layout,
|
||||||
device_overlap_group_keys,
|
device_overlap_group_keys,
|
||||||
extract_legacy_stage_metadata,
|
extract_legacy_stage_metadata,
|
||||||
|
extract_stage_metadata_from_omni_stage_config,
|
||||||
get_stage_connector_spec,
|
get_stage_connector_spec,
|
||||||
inject_kv_stage_info,
|
inject_kv_stage_info,
|
||||||
inject_omni_kv_connector_config,
|
inject_omni_kv_connector_config,
|
||||||
@@ -345,12 +347,12 @@ class StageRuntime:
|
|||||||
"""Build startup plans for every logical stage and replica."""
|
"""Build startup plans for every logical stage and replica."""
|
||||||
stage_plans: list[LogicalStageInitPlan] = []
|
stage_plans: list[LogicalStageInitPlan] = []
|
||||||
|
|
||||||
# RFC #4021 transition boundary: stage planning still relies on legacy
|
|
||||||
# StageConfig.runtime and StageConfig.engine_args for replica and engine
|
|
||||||
# setup. Keep metadata extraction on the legacy path until the
|
|
||||||
# coordinated stage-init cutover.
|
|
||||||
for stage_idx, stage_cfg in enumerate(self._stage_configs):
|
for stage_idx, stage_cfg in enumerate(self._stage_configs):
|
||||||
base_metadata = extract_legacy_stage_metadata(stage_cfg)
|
base_metadata = (
|
||||||
|
extract_stage_metadata_from_omni_stage_config(stage_cfg)
|
||||||
|
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
|
||||||
|
else extract_legacy_stage_metadata(stage_cfg)
|
||||||
|
)
|
||||||
stage_id = int(base_metadata.stage_id)
|
stage_id = int(base_metadata.stage_id)
|
||||||
if stage_id != stage_idx:
|
if stage_id != stage_idx:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -374,14 +376,20 @@ class StageRuntime:
|
|||||||
executor_class = None
|
executor_class = None
|
||||||
engine_args_dict = None
|
engine_args_dict = None
|
||||||
if base_metadata.stage_type != "diffusion":
|
if base_metadata.stage_type != "diffusion":
|
||||||
# The stable adapter entry point still receives the same
|
engine_args_dict = (
|
||||||
# legacy stage object as replica planning. Its implementation
|
build_engine_args_dict_from_omni_stage_config(
|
||||||
# switches only at the coordinated RFC #4021 cutover.
|
stage_cfg,
|
||||||
engine_args_dict = build_engine_args_dict(
|
self._model,
|
||||||
stage_cfg,
|
stage_connector_spec=stage_connector_spec,
|
||||||
self._model,
|
cli_tokenizer=self._tokenizer,
|
||||||
stage_connector_spec=stage_connector_spec,
|
)
|
||||||
cli_tokenizer=self._tokenizer,
|
if isinstance(stage_cfg, BaseVllmOmniStageConfig)
|
||||||
|
else build_engine_args_dict(
|
||||||
|
stage_cfg,
|
||||||
|
self._model,
|
||||||
|
stage_connector_spec=stage_connector_spec,
|
||||||
|
cli_tokenizer=self._tokenizer,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
inject_omni_kv_connector_config(
|
inject_omni_kv_connector_config(
|
||||||
engine_args_dict,
|
engine_args_dict,
|
||||||
@@ -401,11 +409,20 @@ class StageRuntime:
|
|||||||
)
|
)
|
||||||
|
|
||||||
for replica_id in range(num_replicas):
|
for replica_id in range(num_replicas):
|
||||||
replica_cfg = copy.deepcopy(stage_cfg) if replica_id > 0 else stage_cfg
|
# Keep the logical stage's device pool intact; each replica owns
|
||||||
|
# the same config throughout planning, metadata and launch.
|
||||||
|
replica_cfg = copy.deepcopy(stage_cfg) if num_replicas > 1 else stage_cfg
|
||||||
if stage_idx in replica_devices_map:
|
if stage_idx in replica_devices_map:
|
||||||
replica_cfg.runtime.devices = replica_devices_map[stage_idx][replica_id]
|
devices = replica_devices_map[stage_idx][replica_id]
|
||||||
|
runtime_cfg = getattr(replica_cfg, "runtime_config", getattr(replica_cfg, "runtime", None))
|
||||||
|
if runtime_cfg is not None:
|
||||||
|
runtime_cfg.devices = devices
|
||||||
|
|
||||||
replica_metadata = extract_legacy_stage_metadata(replica_cfg)
|
replica_metadata = (
|
||||||
|
extract_stage_metadata_from_omni_stage_config(replica_cfg)
|
||||||
|
if isinstance(replica_cfg, BaseVllmOmniStageConfig)
|
||||||
|
else extract_legacy_stage_metadata(replica_cfg)
|
||||||
|
)
|
||||||
replica_metadata.replica_id = replica_id
|
replica_metadata.replica_id = replica_id
|
||||||
if launch_mode == "remote" and replica_metadata.stage_type != "diffusion":
|
if launch_mode == "remote" and replica_metadata.stage_type != "diffusion":
|
||||||
replica_metadata.runtime_cfg = None
|
replica_metadata.runtime_cfg = None
|
||||||
@@ -859,17 +876,25 @@ class StageRuntime:
|
|||||||
raise RuntimeError("Omni KV connector requires source and destination stages")
|
raise RuntimeError("Omni KV connector requires source and destination stages")
|
||||||
inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to)
|
inject_omni_kv_config(plan.stage_cfg, omni_conn_cfg, omni_from, omni_to)
|
||||||
inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self._stage_configs)
|
inject_kv_stage_info(plan.stage_cfg, plan.metadata.stage_id, self._stage_configs)
|
||||||
engine_args = getattr(plan.stage_cfg, "engine_args", {})
|
if isinstance(plan.stage_cfg, BaseVllmOmniStageConfig):
|
||||||
inline_diffusion = (
|
inline_diffusion = plan.stage_cfg.stage_pipeline_config.inline_diffusion
|
||||||
engine_args.get("inline_diffusion", False)
|
custom_pipeline_args = getattr(
|
||||||
if hasattr(engine_args, "get")
|
getattr(plan.stage_cfg, "diffusion_config", None),
|
||||||
else getattr(engine_args, "inline_diffusion", False)
|
"custom_pipeline_args",
|
||||||
)
|
None,
|
||||||
custom_pipeline_args = (
|
)
|
||||||
engine_args.get("custom_pipeline_args")
|
else:
|
||||||
if hasattr(engine_args, "get")
|
engine_args = getattr(plan.stage_cfg, "engine_args", {})
|
||||||
else getattr(engine_args, "custom_pipeline_args", None)
|
inline_diffusion = (
|
||||||
)
|
engine_args.get("inline_diffusion", False)
|
||||||
|
if hasattr(engine_args, "get")
|
||||||
|
else getattr(engine_args, "inline_diffusion", False)
|
||||||
|
)
|
||||||
|
custom_pipeline_args = (
|
||||||
|
engine_args.get("custom_pipeline_args")
|
||||||
|
if hasattr(engine_args, "get")
|
||||||
|
else getattr(engine_args, "custom_pipeline_args", None)
|
||||||
|
)
|
||||||
client, resources = launch_diffusion_stage_replica(
|
client, resources = launch_diffusion_stage_replica(
|
||||||
model=self._model,
|
model=self._model,
|
||||||
stage_config=plan.stage_cfg,
|
stage_config=plan.stage_cfg,
|
||||||
@@ -1028,7 +1053,7 @@ class DistStageRuntime(StageRuntime):
|
|||||||
|
|
||||||
for idx, stage_cfg in enumerate(self._stage_configs):
|
for idx, stage_cfg in enumerate(self._stage_configs):
|
||||||
stage_id = int(getattr(stage_cfg, "stage_id", idx))
|
stage_id = int(getattr(stage_cfg, "stage_id", idx))
|
||||||
runtime_cfg = getattr(stage_cfg, "runtime", None)
|
runtime_cfg = getattr(stage_cfg, "runtime_config", getattr(stage_cfg, "runtime", None))
|
||||||
if runtime_cfg is None:
|
if runtime_cfg is None:
|
||||||
continue
|
continue
|
||||||
if stage_id == target_stage_id:
|
if stage_id == target_stage_id:
|
||||||
@@ -1100,13 +1125,9 @@ class DistStageRuntime(StageRuntime):
|
|||||||
if registered_stage_cfg is None:
|
if registered_stage_cfg is None:
|
||||||
raise ValueError(f"Remote stage {plan.metadata.stage_id} registered without stage config")
|
raise ValueError(f"Remote stage {plan.metadata.stage_id} registered without stage config")
|
||||||
|
|
||||||
# Remote diffusion registration still transports the legacy mapping
|
# Registration is transport-only: the head-side typed plan remains
|
||||||
# shape. Reconstruct and project that shape until its RFC #4021 cutover.
|
# authoritative for metadata and topology.
|
||||||
metadata = (
|
metadata = extract_stage_metadata_from_omni_stage_config(plan.stage_cfg)
|
||||||
extract_legacy_stage_metadata(OmegaConf.create(registered_stage_cfg))
|
|
||||||
if plan.metadata.stage_type == "diffusion"
|
|
||||||
else copy.deepcopy(plan.metadata)
|
|
||||||
)
|
|
||||||
metadata.replica_id = plan.replica_id
|
metadata.replica_id = plan.replica_id
|
||||||
ctx = StageRemoteFactoryContext(
|
ctx = StageRemoteFactoryContext(
|
||||||
stage_id=plan.metadata.stage_id,
|
stage_id=plan.metadata.stage_id,
|
||||||
|
|||||||
@@ -1048,6 +1048,7 @@ def run_headless(args: TrackingNamespace) -> None:
|
|||||||
)
|
)
|
||||||
from vllm_omni.engine.stage_init_utils import (
|
from vllm_omni.engine.stage_init_utils import (
|
||||||
build_engine_args_dict,
|
build_engine_args_dict,
|
||||||
|
build_engine_args_dict_from_omni_stage_config,
|
||||||
build_vllm_config,
|
build_vllm_config,
|
||||||
get_stage_connector_spec,
|
get_stage_connector_spec,
|
||||||
inject_omni_kv_connector_config,
|
inject_omni_kv_connector_config,
|
||||||
@@ -1141,21 +1142,21 @@ def run_headless(args: TrackingNamespace) -> None:
|
|||||||
stage_connector_spec = get_stage_connector_spec(
|
stage_connector_spec = get_stage_connector_spec(
|
||||||
omni_transfer_config=omni_transfer_config,
|
omni_transfer_config=omni_transfer_config,
|
||||||
stage_id=stage_id,
|
stage_id=stage_id,
|
||||||
async_chunk=bool(stage_cfg.engine_args.get("async_chunk", False)),
|
async_chunk=bool(
|
||||||
|
getattr(getattr(stage_cfg, "connector_config", None), "async_chunk", None)
|
||||||
|
if hasattr(stage_cfg, "connector_config")
|
||||||
|
else stage_cfg.engine_args.get("async_chunk", False)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# ``runtime_cfg`` is mostly inherited from the parent's
|
engine_args_dict = (
|
||||||
# CUDA_VISIBLE_DEVICES; when ``--omni-dp-size-local > 1`` we additionally
|
build_engine_args_dict_from_omni_stage_config(
|
||||||
# bracket each replica's spawn below with setup_stage_devices so they
|
stage_cfg, model, stage_connector_spec=stage_connector_spec, cli_tokenizer=getattr(args, "tokenizer", None)
|
||||||
# don't all stack on cuda:0 (see ``per_replica_devices`` above).
|
)
|
||||||
# Headless startup still supplies the legacy OmegaConf stage shape through
|
if hasattr(stage_cfg, "connector_config")
|
||||||
# the stable adapter entry point. The implementation switches only when
|
else build_engine_args_dict(
|
||||||
# RFC #4021 threads structured stage configs through the launch plan.
|
stage_cfg, model, stage_connector_spec=stage_connector_spec, cli_tokenizer=getattr(args, "tokenizer", None)
|
||||||
engine_args_dict = build_engine_args_dict(
|
)
|
||||||
stage_cfg,
|
|
||||||
model,
|
|
||||||
stage_connector_spec=stage_connector_spec,
|
|
||||||
cli_tokenizer=getattr(args, "tokenizer", None),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
inject_omni_kv_connector_config(engine_args_dict, omni_kv_connector, stage_id)
|
inject_omni_kv_connector_config(engine_args_dict, omni_kv_connector, stage_id)
|
||||||
|
|||||||
@@ -248,10 +248,27 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
|||||||
return obj.get(key, default)
|
return obj.get(key, default)
|
||||||
return getattr(obj, key, default)
|
return getattr(obj, key, default)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _stage_model_metadata(cls, stage: Any) -> tuple[str | None, str | None]:
|
||||||
|
"""Return ``(model_arch, model_stage)`` for typed or legacy stages."""
|
||||||
|
model_config = cls._stage_get(stage, "model_config")
|
||||||
|
model_arch = cls._stage_get(model_config, "model_arch")
|
||||||
|
model_stage = cls._stage_get(stage, "model_stage")
|
||||||
|
|
||||||
|
topology = cls._stage_get(stage, "stage_pipeline_config")
|
||||||
|
if model_stage is None:
|
||||||
|
model_stage = cls._stage_get(topology, "model_stage")
|
||||||
|
|
||||||
|
engine_args = cls._stage_get(stage, "engine_args")
|
||||||
|
if model_arch is None:
|
||||||
|
model_arch = cls._stage_get(engine_args, "model_arch")
|
||||||
|
if model_stage is None:
|
||||||
|
model_stage = cls._stage_get(engine_args, "model_stage")
|
||||||
|
return model_arch, model_stage
|
||||||
|
|
||||||
def _has_minicpmo45_stage(self) -> bool:
|
def _has_minicpmo45_stage(self) -> bool:
|
||||||
for stage in getattr(self.engine_client, "stage_configs", []) or []:
|
for stage in getattr(self.engine_client, "stage_configs", []) or []:
|
||||||
engine_args = self._stage_get(stage, "engine_args")
|
model_arch, _ = self._stage_model_metadata(stage)
|
||||||
model_arch = self._stage_get(engine_args, "model_arch")
|
|
||||||
if model_arch == "MiniCPMO45OmniForConditionalGeneration":
|
if model_arch == "MiniCPMO45OmniForConditionalGeneration":
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
@@ -324,11 +341,10 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
|||||||
if not hasattr(sp, "output_kind"):
|
if not hasattr(sp, "output_kind"):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
engine_args = self._stage_get(stage, "engine_args")
|
model_arch, model_stage = self._stage_model_metadata(stage)
|
||||||
if self._stage_get(engine_args, "model_arch") != "MiniCPMO45OmniForConditionalGeneration":
|
if model_arch != "MiniCPMO45OmniForConditionalGeneration":
|
||||||
continue
|
continue
|
||||||
|
|
||||||
model_stage = self._stage_get(engine_args, "model_stage")
|
|
||||||
if model_stage == "llm":
|
if model_stage == "llm":
|
||||||
sp.output_kind = RequestOutputKind.FINAL_ONLY
|
sp.output_kind = RequestOutputKind.FINAL_ONLY
|
||||||
elif model_stage == "tts":
|
elif model_stage == "tts":
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
|
|||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||||
|
|
||||||
|
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||||
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler
|
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler
|
||||||
from vllm_omni.entrypoints.openai.protocol.audio import (
|
from vllm_omni.entrypoints.openai.protocol.audio import (
|
||||||
AudioResponse,
|
AudioResponse,
|
||||||
@@ -76,6 +77,24 @@ _SPEECH_USAGE_TOTAL_TOKENS_HEADER = "X-VLLM-OMNI-TOTAL-TOKENS"
|
|||||||
_SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TEXT-TOKENS"
|
_SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TEXT-TOKENS"
|
||||||
_SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS"
|
_SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS"
|
||||||
|
|
||||||
|
|
||||||
|
def _stage_speech_metadata(stage: Any) -> tuple[str | None, str | None, str | None]:
|
||||||
|
"""Return speech routing metadata from typed or legacy stage configs."""
|
||||||
|
topology = getattr(stage, "stage_pipeline_config", None)
|
||||||
|
if isinstance(topology, StagePipelineConfig):
|
||||||
|
return (
|
||||||
|
getattr(topology, "model_stage", None),
|
||||||
|
getattr(getattr(stage, "model_config", None), "model_arch", None),
|
||||||
|
getattr(stage, "worker_type", None),
|
||||||
|
)
|
||||||
|
engine_args = getattr(stage, "engine_args", None)
|
||||||
|
return (
|
||||||
|
getattr(engine_args, "model_stage", None),
|
||||||
|
getattr(engine_args, "model_arch", None),
|
||||||
|
getattr(engine_args, "worker_type", None),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# TTS Configuration
|
# TTS Configuration
|
||||||
#
|
#
|
||||||
# The stage-key -> model-type mapping is NOT declared here: it is derived from
|
# The stage-key -> model-type mapping is NOT declared here: it is derived from
|
||||||
@@ -472,12 +491,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
"""Find and return the TTS stage config, or None if not found."""
|
"""Find and return the TTS stage config, or None if not found."""
|
||||||
tts_stage_keys = all_tts_stage_keys()
|
tts_stage_keys = all_tts_stage_keys()
|
||||||
entry_stage_archs = tts_entry_stage_archs()
|
entry_stage_archs = tts_entry_stage_archs()
|
||||||
all_stages = frozenset(stage.engine_args.model_stage for stage in self.engine_client.stage_configs)
|
all_stages = frozenset(
|
||||||
|
model_stage
|
||||||
|
for stage in self.engine_client.stage_configs
|
||||||
|
if (model_stage := _stage_speech_metadata(stage)[0]) is not None
|
||||||
|
)
|
||||||
for stage in self.engine_client.stage_configs:
|
for stage in self.engine_client.stage_configs:
|
||||||
engine_args = stage.engine_args
|
model_stage, model_arch, worker_type = _stage_speech_metadata(stage)
|
||||||
model_stage = engine_args.model_stage
|
|
||||||
model_arch = getattr(engine_args, "model_arch", None)
|
|
||||||
worker_type = getattr(engine_args, "worker_type", None)
|
|
||||||
if model_stage in tts_stage_keys:
|
if model_stage in tts_stage_keys:
|
||||||
# Owning the stage key is not always enough: a model may be
|
# Owning the stage key is not always enough: a model may be
|
||||||
# speech-capable only in some deployment topologies. Ask the
|
# speech-capable only in some deployment topologies. Ask the
|
||||||
@@ -501,10 +521,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
"""
|
"""
|
||||||
if self._tts_stage is None:
|
if self._tts_stage is None:
|
||||||
return None
|
return None
|
||||||
return detect_tts_model_type(
|
return detect_tts_model_type(*_stage_speech_metadata(self._tts_stage)[:2])
|
||||||
getattr(self._tts_stage.engine_args, "model_stage", None),
|
|
||||||
getattr(self._tts_stage.engine_args, "model_arch", None),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _compute_max_instructions_length(self) -> int:
|
def _compute_max_instructions_length(self) -> int:
|
||||||
"""Compute max instructions length with precedence: CLI > stage config > default.
|
"""Compute max instructions length with precedence: CLI > stage config > default.
|
||||||
@@ -518,7 +535,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
|
|
||||||
# 2. Try to get from TTS stage config
|
# 2. Try to get from TTS stage config
|
||||||
if self._tts_stage is not None:
|
if self._tts_stage is not None:
|
||||||
tts_args = getattr(self._tts_stage, "tts_args", {})
|
topology = getattr(self._tts_stage, "stage_pipeline_config", None)
|
||||||
|
tts_args = (
|
||||||
|
getattr(topology, "extras", {}).get("tts_args", {})
|
||||||
|
if isinstance(topology, StagePipelineConfig)
|
||||||
|
else getattr(self._tts_stage, "tts_args", {})
|
||||||
|
)
|
||||||
if "max_instructions_length" in tts_args:
|
if "max_instructions_length" in tts_args:
|
||||||
return tts_args["max_instructions_length"]
|
return tts_args["max_instructions_length"]
|
||||||
|
|
||||||
@@ -1896,9 +1918,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
# chat-template markers (im_start_token_id 151644). A raw-text
|
# chat-template markers (im_start_token_id 151644). A raw-text
|
||||||
# prompt produces a 1-token placeholder that crashes the talker's
|
# prompt produces a 1-token placeholder that crashes the talker's
|
||||||
# prefill/decode handoff. Reject early with an actionable message.
|
# prefill/decode handoff. Reject early with an actionable message.
|
||||||
stage_names = {
|
stage_names = {_stage_speech_metadata(s)[0] for s in self.engine_client.stage_configs}
|
||||||
getattr(getattr(s, "engine_args", None), "model_stage", None) for s in self.engine_client.stage_configs
|
|
||||||
}
|
|
||||||
if "talker" in stage_names:
|
if "talker" in stage_names:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"The /v1/audio/speech endpoint is only supported for "
|
"The /v1/audio/speech endpoint is only supported for "
|
||||||
|
|||||||
@@ -53,6 +53,30 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
_VIDEO_RESPONSE_FRAME_CONVERSION_WORKERS = 8
|
_VIDEO_RESPONSE_FRAME_CONVERSION_WORKERS = 8
|
||||||
|
|
||||||
|
|
||||||
|
def _config_value(config: Any, key: str, default: Any = None) -> Any:
|
||||||
|
if isinstance(config, Mapping):
|
||||||
|
return config.get(key, default)
|
||||||
|
return getattr(config, key, default)
|
||||||
|
|
||||||
|
|
||||||
|
def _stage_diffusion_model_class_name(stage_config: Any) -> str | None:
|
||||||
|
"""Resolve a stage's diffusion class across typed and legacy shapes."""
|
||||||
|
diffusion_config = _config_value(stage_config, "diffusion_config")
|
||||||
|
model_class_name = _config_value(diffusion_config, "model_class_name")
|
||||||
|
if model_class_name:
|
||||||
|
return str(model_class_name)
|
||||||
|
|
||||||
|
model_config = _config_value(stage_config, "model_config")
|
||||||
|
model_arch = _config_value(model_config, "model_arch") or _config_value(stage_config, "model_arch")
|
||||||
|
if model_arch:
|
||||||
|
return str(model_arch)
|
||||||
|
|
||||||
|
engine_args = _config_value(stage_config, "engine_args", {})
|
||||||
|
model_class_name = _config_value(engine_args, "model_class_name")
|
||||||
|
return str(model_class_name) if model_class_name else None
|
||||||
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||||
|
|
||||||
@@ -195,18 +219,7 @@ class OmniOpenAIServingVideo:
|
|||||||
model_class_name = getattr(od_config, "model_class_name", None)
|
model_class_name = getattr(od_config, "model_class_name", None)
|
||||||
model_archs = [model_class_name]
|
model_archs = [model_class_name]
|
||||||
for stage_config in self.stage_configs or ():
|
for stage_config in self.stage_configs or ():
|
||||||
stage_get = (
|
model_archs.append(_stage_diffusion_model_class_name(stage_config))
|
||||||
stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None)
|
|
||||||
)
|
|
||||||
engine_args = stage_get("engine_args") or {}
|
|
||||||
model_archs.extend(
|
|
||||||
(
|
|
||||||
stage_get("model_arch"),
|
|
||||||
engine_args.get("model_class_name")
|
|
||||||
if isinstance(engine_args, Mapping)
|
|
||||||
else getattr(engine_args, "model_class_name", None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
metadata_capability = any(
|
metadata_capability = any(
|
||||||
get_diffusion_model_metadata(model_arch).supports_mixed_reference_inputs for model_arch in model_archs
|
get_diffusion_model_metadata(model_arch).supports_mixed_reference_inputs for model_arch in model_archs
|
||||||
)
|
)
|
||||||
@@ -223,18 +236,7 @@ class OmniOpenAIServingVideo:
|
|||||||
od_config = self._resolve_diffusion_od_config()
|
od_config = self._resolve_diffusion_od_config()
|
||||||
model_archs = [None if od_config is None else getattr(od_config, "model_class_name", None)]
|
model_archs = [None if od_config is None else getattr(od_config, "model_class_name", None)]
|
||||||
for stage_config in self.stage_configs or ():
|
for stage_config in self.stage_configs or ():
|
||||||
stage_get = (
|
model_archs.append(_stage_diffusion_model_class_name(stage_config))
|
||||||
stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None)
|
|
||||||
)
|
|
||||||
engine_args = stage_get("engine_args") or {}
|
|
||||||
model_archs.extend(
|
|
||||||
(
|
|
||||||
stage_get("model_arch"),
|
|
||||||
engine_args.get("model_class_name")
|
|
||||||
if isinstance(engine_args, Mapping)
|
|
||||||
else getattr(engine_args, "model_class_name", None),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
supported: set[str] = set()
|
supported: set[str] = set()
|
||||||
for model_arch in model_archs:
|
for model_arch in model_archs:
|
||||||
|
|||||||
@@ -42,7 +42,11 @@ class AudexAdapter(ARTTSAdapter):
|
|||||||
|
|
||||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||||
|
|
||||||
stage = getattr(getattr(self.ctx.server._tts_stage, "engine_args", None), "model_stage", None)
|
stage_config = self.ctx.server._tts_stage
|
||||||
|
topology = getattr(stage_config, "stage_pipeline_config", None)
|
||||||
|
stage = getattr(topology, "model_stage", None)
|
||||||
|
if stage is None:
|
||||||
|
stage = getattr(getattr(stage_config, "engine_args", None), "model_stage", None)
|
||||||
profile, folder = (
|
profile, folder = (
|
||||||
("full", "checkpoint_folder_full") if stage == "audex_omni" else ("tts", "checkpoint_folder_audiogen")
|
("full", "checkpoint_folder_full") if stage == "audex_omni" else ("tts", "checkpoint_folder_audiogen")
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -39,6 +39,21 @@ DEFAULT_TTS_LANGUAGES = frozenset(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_stage_model_path(engine_client: Any) -> str | None:
|
||||||
|
"""Prefer typed stage model overrides, then legacy and served-model values."""
|
||||||
|
stages = getattr(engine_client, "stage_configs", ()) or ()
|
||||||
|
for stage in stages:
|
||||||
|
model_path = getattr(getattr(stage, "model_config", None), "model", None)
|
||||||
|
if model_path:
|
||||||
|
return str(model_path)
|
||||||
|
for stage in stages:
|
||||||
|
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
|
||||||
|
if model_path:
|
||||||
|
return str(model_path)
|
||||||
|
model_path = getattr(engine_client, "model", None)
|
||||||
|
return str(model_path) if model_path else None
|
||||||
|
|
||||||
|
|
||||||
def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: dict | None = None) -> str:
|
def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: dict | None = None) -> str:
|
||||||
"""Stable hash of the real Stage 0 conditioning for the prefix cache.
|
"""Stable hash of the real Stage 0 conditioning for the prefix cache.
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,12 @@ import numpy as np
|
|||||||
from vllm.inputs import tokens_input
|
from vllm.inputs import tokens_input
|
||||||
|
|
||||||
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
|
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
|
||||||
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
|
from vllm_omni.entrypoints.openai.tts_adapters.base import (
|
||||||
|
ARTTSAdapter,
|
||||||
|
PreparedRequest,
|
||||||
|
apply_max_new_tokens,
|
||||||
|
resolve_stage_model_path,
|
||||||
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||||
@@ -81,17 +86,7 @@ class HiggsAudioV2Adapter(ARTTSAdapter):
|
|||||||
|
|
||||||
from transformers import AutoProcessor
|
from transformers import AutoProcessor
|
||||||
|
|
||||||
model_path = None
|
model_path = resolve_stage_model_path(self.engine_client)
|
||||||
for stage in self.engine_client.stage_configs:
|
|
||||||
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
|
|
||||||
if model_path:
|
|
||||||
break
|
|
||||||
if model_path is None:
|
|
||||||
# Fallback: the orchestrator stores the served model id on the engine
|
|
||||||
# itself (set by AsyncOmniEngine.__init__). Stage-level engine_args
|
|
||||||
# may not surface ``model`` when the deploy yaml doesn't set it per
|
|
||||||
# stage (the CLI-passed model id is the single source of truth).
|
|
||||||
model_path = getattr(self.engine_client, "model", None)
|
|
||||||
if model_path is None:
|
if model_path is None:
|
||||||
raise RuntimeError("higgs_audio_v2 serving could not resolve the model path from the engine stage configs")
|
raise RuntimeError("higgs_audio_v2 serving could not resolve the model path from the engine stage configs")
|
||||||
processor = AutoProcessor.from_pretrained(model_path)
|
processor = AutoProcessor.from_pretrained(model_path)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import (
|
|||||||
PreparedRequest,
|
PreparedRequest,
|
||||||
apply_max_new_tokens,
|
apply_max_new_tokens,
|
||||||
conditioning_cache_salt,
|
conditioning_cache_salt,
|
||||||
|
resolve_stage_model_path,
|
||||||
)
|
)
|
||||||
|
|
||||||
_REF_CODE_CACHE_MAX_ENTRIES = 256
|
_REF_CODE_CACHE_MAX_ENTRIES = 256
|
||||||
@@ -170,13 +171,7 @@ class HiggsAudioV3Adapter(ARTTSAdapter):
|
|||||||
HiggsAudioV3TokenizerAdapter,
|
HiggsAudioV3TokenizerAdapter,
|
||||||
)
|
)
|
||||||
|
|
||||||
model_path = None
|
model_path = resolve_stage_model_path(self.engine_client)
|
||||||
for stage in self.engine_client.stage_configs:
|
|
||||||
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
|
|
||||||
if model_path:
|
|
||||||
break
|
|
||||||
if model_path is None:
|
|
||||||
model_path = getattr(self.engine_client, "model", None)
|
|
||||||
if model_path is None:
|
if model_path is None:
|
||||||
raise RuntimeError("higgs_audio_v3 serving could not resolve model path")
|
raise RuntimeError("higgs_audio_v3 serving could not resolve model path")
|
||||||
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
||||||
|
|||||||
@@ -58,6 +58,7 @@ from vllm_omni.entrypoints.openai.serving_video import (
|
|||||||
ReferenceAudio,
|
ReferenceAudio,
|
||||||
ReferenceImage,
|
ReferenceImage,
|
||||||
ReferenceVideo,
|
ReferenceVideo,
|
||||||
|
_stage_diffusion_model_class_name,
|
||||||
)
|
)
|
||||||
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
|
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
|
||||||
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE
|
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE
|
||||||
@@ -136,10 +137,6 @@ def _config_get(config: Any, key: str, default: Any = None) -> Any:
|
|||||||
return getattr(config, key, default)
|
return getattr(config, key, default)
|
||||||
|
|
||||||
|
|
||||||
def _stage_engine_args(stage_cfg: Any) -> Any:
|
|
||||||
return _config_get(stage_cfg, "engine_args", {}) or {}
|
|
||||||
|
|
||||||
|
|
||||||
def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
|
def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
|
||||||
if not stage_configs:
|
if not stage_configs:
|
||||||
return []
|
return []
|
||||||
@@ -150,7 +147,7 @@ def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
|
|||||||
for stage_cfg in stage_configs:
|
for stage_cfg in stage_configs:
|
||||||
if get_stage_type(stage_cfg) != "diffusion":
|
if get_stage_type(stage_cfg) != "diffusion":
|
||||||
continue
|
continue
|
||||||
model_class_name = _config_get(_stage_engine_args(stage_cfg), "model_class_name")
|
model_class_name = _stage_diffusion_model_class_name(stage_cfg)
|
||||||
if not model_class_name:
|
if not model_class_name:
|
||||||
continue
|
continue
|
||||||
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
||||||
|
|||||||
@@ -103,6 +103,14 @@ class ServingRealtimeRobotOpenPI:
|
|||||||
for stage_config in getattr(engine_client, "stage_configs", []) or []:
|
for stage_config in getattr(engine_client, "stage_configs", []) or []:
|
||||||
if getattr(stage_config, "stage_type", None) != "diffusion":
|
if getattr(stage_config, "stage_type", None) != "diffusion":
|
||||||
continue
|
continue
|
||||||
|
# Typed diffusion stages keep model-owned OpenPI handshake
|
||||||
|
# metadata in diffusion_config.model_config. The out-of-process
|
||||||
|
# head only has this stage view because full od_config lives in
|
||||||
|
# the worker.
|
||||||
|
diffusion_config = getattr(stage_config, "diffusion_config", None)
|
||||||
|
model_config = getattr(diffusion_config, "model_config", None)
|
||||||
|
if model_config is not None:
|
||||||
|
break
|
||||||
engine_args = getattr(stage_config, "engine_args", None)
|
engine_args = getattr(stage_config, "engine_args", None)
|
||||||
model_config = getattr(engine_args, "model_config", None)
|
model_config = getattr(engine_args, "model_config", None)
|
||||||
if model_config is not None:
|
if model_config is not None:
|
||||||
|
|||||||
@@ -4,18 +4,20 @@
|
|||||||
|
|
||||||
|
|
||||||
def _should_enable_profiler_endpoints(stage_configs: list | None) -> bool:
|
def _should_enable_profiler_endpoints(stage_configs: list | None) -> bool:
|
||||||
"""Check if any stage has profiler_config set in its engine_args."""
|
"""Check whether any typed or legacy stage enables a profiler."""
|
||||||
if not stage_configs:
|
if not stage_configs:
|
||||||
return False
|
return False
|
||||||
for stage in stage_configs:
|
for stage in stage_configs:
|
||||||
engine_args = stage.get("engine_args") if isinstance(stage, dict) else getattr(stage, "engine_args", None)
|
|
||||||
if engine_args is None:
|
|
||||||
continue
|
|
||||||
profiler_config = (
|
profiler_config = (
|
||||||
engine_args.get("profiler_config")
|
stage.get("profiler_config") if isinstance(stage, dict) else getattr(stage, "profiler_config", None)
|
||||||
if isinstance(engine_args, dict)
|
|
||||||
else getattr(engine_args, "profiler_config", None)
|
|
||||||
)
|
)
|
||||||
|
if profiler_config is None:
|
||||||
|
engine_args = stage.get("engine_args") if isinstance(stage, dict) else getattr(stage, "engine_args", None)
|
||||||
|
profiler_config = (
|
||||||
|
engine_args.get("profiler_config")
|
||||||
|
if isinstance(engine_args, dict)
|
||||||
|
else getattr(engine_args, "profiler_config", None)
|
||||||
|
)
|
||||||
if profiler_config is not None:
|
if profiler_config is not None:
|
||||||
profiler = (
|
profiler = (
|
||||||
profiler_config.get("profiler")
|
profiler_config.get("profiler")
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
from dataclasses import asdict, is_dataclass
|
||||||
from multiprocessing import shared_memory as _shm
|
from multiprocessing import shared_memory as _shm
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -274,6 +278,8 @@ def _to_dict(x: Any) -> dict[str, Any]:
|
|||||||
try:
|
try:
|
||||||
if isinstance(x, dict):
|
if isinstance(x, dict):
|
||||||
return dict(x)
|
return dict(x)
|
||||||
|
if is_dataclass(x) and not isinstance(x, type):
|
||||||
|
return asdict(x)
|
||||||
return _omega_to_dict(x)
|
return _omega_to_dict(x)
|
||||||
except Exception:
|
except Exception:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -18,6 +18,13 @@ logger = init_logger(__name__)
|
|||||||
|
|
||||||
def inject_omni_kv_config(stage: Any, omni_conn_cfg: dict[str, Any], omni_from: str, omni_to: str) -> None:
|
def inject_omni_kv_config(stage: Any, omni_conn_cfg: dict[str, Any], omni_from: str, omni_to: str) -> None:
|
||||||
"""Inject connector configuration into stage engine arguments."""
|
"""Inject connector configuration into stage engine arguments."""
|
||||||
|
typed_connector_config = getattr(stage, "connector_config", None)
|
||||||
|
if typed_connector_config is not None:
|
||||||
|
omni_conf_dict = dict(typed_connector_config.omni_kv_config or {})
|
||||||
|
omni_conf_dict.update(connector_config=omni_conn_cfg, omni_from_stage=omni_from, omni_to_stage=omni_to)
|
||||||
|
typed_connector_config.omni_kv_config = omni_conf_dict
|
||||||
|
return
|
||||||
|
|
||||||
# Prepare omni_kv_config dict
|
# Prepare omni_kv_config dict
|
||||||
omni_conf_dict = {}
|
omni_conf_dict = {}
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in new issue
Block a user