[Config Refactor] sentinel default precedence (#3078)

Signed-off-by: lishunyang <lishunyang12@163.com>
Signed-off-by: xiaohajiayou <923390377@qq.com>
Co-authored-by: lishunyang <lishunyang12@163.com>
This commit is contained in:
Hacoandlishunyang authored and GitHub committed 2026-04-24 20:07:24 +08:00
1 parent b33d536f63
commit 22ffb85063
29 files changed
+458 -188

No files matched your search

+1
View File
@@ -121,6 +121,7 @@ def extract_omni_serve_subparser_init():
"FlexibleArgumentParser": _FlexibleArgumentParser,
"make_arg_parser": lambda parser: parser, # no-op for doc
"_ensure_vllm_platform": lambda: None, # no-op for doc
"nullify_stage_engine_defaults": lambda parser: None, # no-op for doc
"VLLM_SUBCMD_PARSER_EPILOG": "",
"logger": logger,
"DummySubparsers": DummySubparsers,
@@ -116,6 +116,9 @@ def parse_args():
help="Temperature for text generation sampling (default: 0.3).",
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
args = parser.parse_args()
return args
@@ -57,7 +57,6 @@ def run_e2e():
omni = Omni(
model=args.model,
deploy_config=args.deploy_config,
trust_remote_code=True,
tokenizer=args.tokenizer,
log_stats=True,
)
@@ -118,6 +118,9 @@ def parse_args() -> argparse.Namespace:
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -970,6 +970,9 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
parser.add_argument("--vq-model-audio-local-files-only", action=argparse.BooleanOptionalAction, default=None)
parser.add_argument("--disable-hf-xet", action=argparse.BooleanOptionalAction, default=True)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -196,6 +196,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -135,6 +135,9 @@ def parse_args():
parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -378,6 +378,9 @@ def parse_args() -> argparse.Namespace:
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -211,6 +211,9 @@ def parse_args() -> argparse.Namespace:
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -24,6 +24,9 @@ def parse_args():
parser.add_argument("--width", type=int, default=448, help="Video width.")
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -19,6 +19,7 @@ from vllm import SamplingParams
from vllm.multimodal.image import convert_image_mode
from vllm_omni import Omni
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
DEFAULT_SYSTEM = "You are a helpful assistant."
DEFAULT_QUESTION = "Please summarize the content of this image."
@@ -48,6 +49,7 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Enable diffusion pipeline profiler to display stage durations.",
)
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -29,6 +29,7 @@ from PIL import Image
from vllm.sampling_params import SamplingParams
from vllm_omni import Omni
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__)
@@ -117,6 +118,7 @@ def parse_args() -> argparse.Namespace:
)
p.add_argument("--out", type=str, default="output.png", help="Path to save the generated image.")
p.add_argument("--trust-remote-code", action="store_true", help="Trust remote code when loading the model.")
nullify_stage_engine_defaults(p)
args = p.parse_args()
if not args.prompt:
args.prompt = ["A stylish woman with sunglasses riding a motorcycle in NYC."]
@@ -97,7 +97,6 @@ def main():
omni = Omni(
model=args.model,
stage_configs_path=args.stage_configs_path,
trust_remote_code=True,
log_stats=args.log_stats,
init_timeout=args.init_timeout,
stage_init_timeout=args.stage_init_timeout,
@@ -89,7 +89,6 @@ def run_e2e():
omni = Omni(
model=args.model,
stage_configs_path=args.stage_config,
trust_remote_code=True,
log_stats=True,
)
@@ -21,6 +21,7 @@ from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.media.audio import load_audio
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
from vllm_omni.entrypoints.omni import Omni
SEED = 42
@@ -550,6 +551,7 @@ def parse_args():
help="Model dtype (auto, half, float16, bfloat16, float, float32).",
)
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -300,6 +300,9 @@ def parse_args() -> argparse.Namespace:
default=None,
help=("Custom system prompt. Used when --use-system-prompt is custom. "),
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -226,6 +226,9 @@ def parse_args() -> argparse.Namespace:
default=1,
help="Number of HSDP replica groups.",
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -71,6 +71,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
+33
View File
@@ -422,6 +422,39 @@ class TestSingleStageModeDetection:
)
assert engine._single_stage_id_filter is None
def test_engine_args_create_only_forwards_explicit_fields(self, mocker: MockerFixture):
from vllm_omni.engine.arg_utils import OmniEngineArgs
captured: dict[str, Any] = {}
def fake_resolve(self, model: str, kwargs: dict[str, Any]):
captured.update(kwargs)
return "/fake/path", [_make_stage_cfg(0)]
mocker.patch.object(AsyncOmniEngine, "_resolve_stage_configs", fake_resolve)
mocker.patch.object(AsyncOmniEngine, "_bootstrap_orchestrator")
mock_thread_cls = mocker.patch("threading.Thread")
mock_future_cls = mocker.patch("concurrent.futures.Future")
mock_future = mocker.Mock()
mock_future.result.return_value = mocker.Mock()
mock_future_cls.return_value = mock_future
mock_thread = mocker.Mock()
mock_thread.is_alive.return_value = False
mock_thread_cls.return_value = mock_thread
ea = OmniEngineArgs.create(model="ignored", gpu_memory_utilization=0.5)
AsyncOmniEngine(model="fake-model", engine_args=ea)
assert captured["gpu_memory_utilization"] == 0.5
assert "model" not in captured
assert "max_num_seqs" not in captured
def test_bare_engine_args_rejected(self, mocker: MockerFixture):
from vllm_omni.engine.arg_utils import OmniEngineArgs
with pytest.raises(TypeError, match="OmniEngineArgs.create"):
self._make_engine_no_thread(mocker, engine_args=OmniEngineArgs(model="fake-model"))
def test_master_address_and_port_stored(self, mocker: MockerFixture):
engine = self._make_engine_no_thread(
mocker,
@@ -1,5 +1,6 @@
from __future__ import annotations
import argparse
import queue
from collections.abc import Callable
from types import SimpleNamespace
@@ -166,6 +167,31 @@ def _patch_engine(monkeypatch: pytest.MonkeyPatch, engine: FakeAsyncOmniEngine)
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
def test_from_cli_args_only_nulls_untyped_override_fields(monkeypatch: pytest.MonkeyPatch):
from vllm_omni.entrypoints.omni import Omni
captured: dict[str, Any] = {}
def fake_engine(*args: Any, **kwargs: Any) -> FakeAsyncOmniEngine:
captured.update(kwargs)
return FakeAsyncOmniEngine()
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", fake_engine)
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
monkeypatch.setattr("sys.argv", ["prog"])
parser = argparse.ArgumentParser()
parser.add_argument("--gpu-memory-utilization", type=float, default=0.9)
parser.add_argument("--hsdp-shard-size", type=int, default=-1)
args = parser.parse_args([])
args.model = "fake-model"
Omni.from_cli_args(args, parser=parser)
assert captured["gpu_memory_utilization"] is None
assert captured["hsdp_shard_size"] == -1
def _make_base():
from vllm_omni.entrypoints.omni_base import OmniBase
+135
View File
@@ -351,3 +351,138 @@ def test_ambiguous_field_non_strict_routes_to_orchestrator(caplog):
assert orch.deploy_config == "x"
assert "deploy_config" not in engine
assert any("both OrchestratorArgs" in r.message for r in caplog.records)
# Sentinel-default precedence invariants (#3035)
def _build_full_serve_parser():
from vllm.utils.argparse_utils import FlexibleArgumentParser
try:
from vllm.entrypoints.openai.cli_args import make_arg_parser
except ImportError:
pytest.skip("vllm parser not importable")
return make_arg_parser(FlexibleArgumentParser())
def test_nullify_stage_engine_defaults_resets_inherited_defaults():
import argparse
from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
nullify_stage_engine_defaults,
)
parser = _build_full_serve_parser()
nullify_stage_engine_defaults(parser)
override_dests = deploy_override_field_names()
offenders = [
(a.dest, a.default)
for a in parser._actions
if a.dest not in ("help", "version")
and a.option_strings
and a.dest in override_dests
and a.default is not None
and a.default is not argparse.SUPPRESS
]
assert not offenders, f"Stage flags with non-None defaults after nullify: {offenders}"
def test_non_override_flags_keep_real_defaults_after_nullify():
import argparse
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
parser = argparse.ArgumentParser()
parser.add_argument("--hsdp-shard-size", type=int, default=-1, help="HSDP shard size.")
parser.add_argument("--max-num-seqs", type=int, default=64, help="Max num seqs.")
nullify_stage_engine_defaults(parser)
hsdp = next(a for a in parser._actions if a.dest == "hsdp_shard_size")
max_num_seqs = next(a for a in parser._actions if a.dest == "max_num_seqs")
assert hsdp.default == -1
assert max_num_seqs.default is None
def test_help_text_preserves_default_after_nullify():
# Real defaults must stay visible in --help even though parser stores None.
import argparse
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
parser = argparse.ArgumentParser()
parser.add_argument("--max-num-seqs", type=int, default=42, help="Example knob.")
nullify_stage_engine_defaults(parser)
action = next(a for a in parser._actions if a.dest == "max_num_seqs")
assert action.default is None
assert "(default: 42)" in action.help
_OMNIENGINEARGS_USER_INPUT_FIELDS = frozenset(
{
"model_stage",
"model_arch",
"engine_output_type",
"hf_config_name",
"custom_process_next_stage_input_func",
"subtalker_sampling_params",
"async_chunk",
"omni_kv_config",
"quantization_config",
"worker_type",
"task_type",
"worker_cls",
"enable_sleep_mode",
"omni_master_address",
"omni_master_port",
"stage_configs_path",
"output_modalities",
"log_stats",
"custom_pipeline_args",
}
)
def test_omniengineargs_user_input_fields_default_to_none():
try:
from vllm_omni.engine.arg_utils import OmniEngineArgs
except Exception as exc:
pytest.skip(f"OmniEngineArgs not importable: {exc}")
offenders = [
(f.name, f.default)
for f in fields(OmniEngineArgs)
if f.name in _OMNIENGINEARGS_USER_INPUT_FIELDS
and f.default is not dataclasses.MISSING
and f.default is not None
]
assert not offenders, f"User-input fields with non-None defaults: {offenders}"
def test_omniengineargs_create_tracks_explicit_fields():
try:
from vllm_omni.engine.arg_utils import OmniEngineArgs
except Exception as exc:
pytest.skip(f"OmniEngineArgs not importable: {exc}")
ea = OmniEngineArgs.create(model="x", gpu_memory_utilization=0.5)
assert ea._explicit_fields == frozenset({"model", "gpu_memory_utilization"})
assert ea.explicit_kwargs() == {"model": "x", "gpu_memory_utilization": 0.5}
def test_omniengineargs_bare_constructor_has_no_explicit_tracking():
try:
from vllm_omni.engine.arg_utils import OmniEngineArgs
except Exception as exc:
pytest.skip(f"OmniEngineArgs not importable: {exc}")
ea = OmniEngineArgs(model="x")
assert not hasattr(ea, "_explicit_fields")
assert "model" in ea.explicit_kwargs()
# dataclasses already imported via ``from dataclasses import dataclass, fields``
import dataclasses # noqa: E402 -- needed for MISSING sentinel above
+61 -70
View File
@@ -4,6 +4,7 @@
Unit tests for StageConfigFactory and related classes.
"""
import warnings
from dataclasses import dataclass
from pathlib import Path
@@ -1196,99 +1197,89 @@ class TestCLIOverrideFlow:
assert s.runtime_overrides["enforce_eager"] is True
class TestCLIExplicitPrecedence:
"""Verify YAML > argparse defaults; explicit CLI args > YAML."""
class TestSentinelDefaultPrecedence:
"""Caller-typed (non-None) values win over YAML; None values fall through
to YAML / dataclass defaults (#3035)."""
def _stages(self, cli_overrides, cli_explicit_keys):
def _stages(self, cli_overrides):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
return StageConfigFactory._create_from_registry(
"qwen3_omni_moe",
cli_overrides=cli_overrides,
cli_explicit_keys=cli_explicit_keys,
)
def test_explicit_cli_overrides_yaml(self):
"""User-typed --max-num-seqs wins over the deploy YAML value."""
stages = self._stages(
cli_overrides={"max_num_seqs": 999},
cli_explicit_keys={"max_num_seqs"},
)
# Stage 2 yaml has max_num_seqs=1; explicit CLI must beat it.
def test_typed_kwarg_overrides_yaml(self):
stages = self._stages({"max_num_seqs": 999})
assert stages[2].runtime_overrides.get("max_num_seqs") == 999
def test_default_cli_does_not_override_yaml(self):
"""Argparse defaults must NOT clobber values that are present in YAML."""
stages = self._stages(
cli_overrides={"max_num_seqs": 256},
cli_explicit_keys=set(), # user typed nothing
)
# Stage 2's YAML value (1) should win because the user didn't type --max-num-seqs.
assert stages[2].runtime_overrides.get("max_num_seqs") != 256
def test_none_value_skipped_yaml_wins(self):
stages = self._stages({"max_num_seqs": None})
assert stages[2].runtime_overrides.get("max_num_seqs") is None
assert stages[2].yaml_engine_args.get("max_num_seqs") == 1
def test_default_cli_fills_missing_yaml_field(self):
"""Argparse defaults still fill fields the YAML doesn't set."""
stages = self._stages(
cli_overrides={"some_unrelated_knob": "fallback"},
cli_explicit_keys=set(),
)
# Field absent from YAML → CLI default flows through as a fallback.
assert stages[0].runtime_overrides.get("some_unrelated_knob") == "fallback"
def test_empty_kwargs_yaml_only(self):
stages = self._stages({})
for stage in stages:
assert stage.runtime_overrides == {}
def test_per_stage_overrides_always_explicit(self):
"""``stage_<id>_*`` keys are always treated as explicit."""
stages = self._stages(
cli_overrides={"stage_0_gpu_memory_utilization": 0.42},
cli_explicit_keys=set(), # not in the explicit set, but per-stage
)
def test_typed_kwarg_equal_to_dataclass_default_still_overrides(self):
# Caller intent honored regardless of value coincidence (no heuristic).
stages = self._stages({"gpu_memory_utilization": 0.9})
assert stages[2].runtime_overrides.get("gpu_memory_utilization") == 0.9
def test_per_stage_kwarg_routed_to_correct_stage(self):
stages = self._stages({"stage_0_gpu_memory_utilization": 0.42})
assert stages[0].runtime_overrides.get("gpu_memory_utilization") == 0.42
assert stages[2].runtime_overrides.get("gpu_memory_utilization") is None
def test_none_explicit_set_treats_all_as_explicit(self):
"""Programmatic Omni() callers (cli_explicit_keys=None) keep current behavior."""
stages = self._stages(
cli_overrides={"max_num_seqs": 999},
cli_explicit_keys=None,
)
assert stages[2].runtime_overrides.get("max_num_seqs") == 999
def test_explicit_async_chunk_false_overrides_yaml(self):
"""``--no-async-chunk`` flips the deploy-level async_chunk to False even
when the YAML sets it to True. Verifies that the per-stage
``async_chunk: True`` injection in ``merge_pipeline_deploy`` is skipped
and that ``async_chunk`` does not leak through ``_merge_cli_overrides``.
"""
stages = self._stages(
cli_overrides={"async_chunk": False},
cli_explicit_keys={"async_chunk"},
)
# qwen3_omni_moe.yaml has `async_chunk: true`, so by default every
# stage's engine_args would carry it. With the explicit override, it
# must NOT show up.
def test_async_chunk_false_overrides_yaml_true(self):
stages = self._stages({"async_chunk": False})
for stage in stages:
assert stage.yaml_engine_args.get("async_chunk") is not True
assert stage.runtime_overrides.get("async_chunk") is None
def test_default_async_chunk_leaves_yaml_alone(self):
"""An unset ``--async-chunk`` (default None) must leave the YAML's True
in force on every stage."""
stages = self._stages(
cli_overrides={"async_chunk": None},
cli_explicit_keys=set(),
)
# qwen3_omni_moe.yaml: `async_chunk: true` → injected on every stage.
def test_async_chunk_none_keeps_yaml_true(self):
stages = self._stages({"async_chunk": None})
for stage in stages:
assert stage.yaml_engine_args.get("async_chunk") is True
def test_explicit_enable_prefix_caching_overrides_yaml(self):
"""``--enable-prefix-caching`` (global) flips every stage's
``enable_prefix_caching`` to True regardless of the YAML default."""
stages = self._stages(
cli_overrides={"enable_prefix_caching": True},
cli_explicit_keys={"enable_prefix_caching"},
)
def test_enable_prefix_caching_typed_overrides_yaml(self):
stages = self._stages({"enable_prefix_caching": True})
for stage in stages:
assert stage.runtime_overrides.get("enable_prefix_caching") is True
def test_omni_with_vars_args_anti_pattern_is_safe(self):
# Omni(**vars(args)) with mostly-None namespace must not clobber YAML.
simulated_vars_args = {
"gpu_memory_utilization": None,
"max_num_seqs": None,
"async_chunk": None,
"enable_prefix_caching": None,
"dtype": None,
}
stages = self._stages(simulated_vars_args)
for stage in stages:
assert stage.runtime_overrides == {}
def test_create_from_registry_no_cli_explicit_keys_param(self):
import inspect
sig = inspect.signature(StageConfigFactory._create_from_registry)
named = [p for p in sig.parameters.values() if p.kind != p.VAR_KEYWORD]
assert "cli_explicit_keys" not in {p.name for p in named}
def test_cli_explicit_keys_kwarg_emits_deprecation(self):
import vllm_omni.model_executor.models.qwen3_omni.pipeline # noqa: F401
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
StageConfigFactory._create_from_registry(
"qwen3_omni_moe",
cli_overrides={},
cli_explicit_keys={"max_num_seqs"},
)
assert any(issubclass(x.category, DeprecationWarning) for x in w)
def test_async_chunk_dispatches_processors(self):
"""A single ``qwen3_tts`` pipeline picks per-chunk vs end-to-end
processors based on ``deploy.async_chunk``, without needing a
+33 -48
View File
@@ -28,6 +28,15 @@ def get_pipeline_path(model_dir: str, filename: str) -> Path:
logger = init_logger(__name__)
def _warn_deprecated_kwargs(kwargs: dict[str, Any]) -> None:
if "cli_explicit_keys" in kwargs:
warnings.warn(
"cli_explicit_keys= is deprecated and ignored. Remove the kwarg.",
DeprecationWarning,
stacklevel=3,
)
_STAGE_OVERRIDE_PATTERN = re.compile(r"^stage_(\d+)_(.+)$")
@@ -734,8 +743,9 @@ def _build_engine_args(
continue
engine_args[k] = v
engine_args.update(ds.engine_extras)
if deploy.async_chunk:
engine_args["async_chunk"] = True
# Materialize the resolved pipeline-wide async_chunk value into every
# stage so explicit False overrides do not get lost downstream.
engine_args["async_chunk"] = bool(deploy.async_chunk)
return engine_args
@@ -996,17 +1006,14 @@ class StageConfigFactory:
model: str,
cli_overrides: dict[str, Any] | None = None,
deploy_config_path: str | None = None,
cli_explicit_keys: set[str] | None = None,
**deprecated_kwargs: Any,
) -> list[StageConfig] | None:
"""Load pipeline + deploy config, merge with CLI overrides.
Checks _PIPELINE_REGISTRY first (new path), falls back to legacy YAML.
``cli_explicit_keys`` is the set of CLI keys the user actually typed
(captured at the parser layer in ``vllm serve``). When ``None`` —
which is the case for programmatic ``Omni()`` callers — every kwarg
in ``cli_overrides`` is treated as explicit.
"""
_warn_deprecated_kwargs(deprecated_kwargs)
if cli_overrides is None:
cli_overrides = {}
@@ -1015,7 +1022,7 @@ class StageConfigFactory:
# --- New path: check pipeline registry by model_type first ---
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
if model_type and model_type in _PIPELINE_REGISTRY:
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path, cli_explicit_keys)
return cls._create_from_registry(model_type, cli_overrides, deploy_config_path)
# --- HF architecture fallback: some models report a generic
# model_type that collides with another model. Match by the
@@ -1025,9 +1032,7 @@ class StageConfigFactory:
if hf_archs:
for registered in _PIPELINE_REGISTRY.values():
if hf_archs.intersection(registered.hf_architectures):
return cls._create_from_registry(
registered.model_type, cli_overrides, deploy_config_path, cli_explicit_keys
)
return cls._create_from_registry(registered.model_type, cli_overrides, deploy_config_path)
# --- Legacy path: load from pipeline YAML ---
pipeline = cls._load_pipeline(model, trust_remote_code=trust_remote_code)
@@ -1039,14 +1044,14 @@ class StageConfigFactory:
if errors:
logger.warning(f"Pipeline validation warnings for {model}: {errors}")
# Inject pipeline-wide async_chunk into ALL stages' engine_args.
# The legacy loader (load_stage_configs_from_yaml) sets async_chunk
# on every stage so that build_engine_args_dict() can inject the
# stage_connector_spec. AsyncOmniEngine.__init__ also reads it
# from stage_configs[0].engine_args.async_chunk.
if pipeline.async_chunk:
for stage in pipeline.stages:
stage.yaml_engine_args.setdefault("async_chunk", True)
# Materialize the resolved pipeline-wide async_chunk value into every
# stage so build_engine_args_dict() can inject the stage connector
# spec and explicit False overrides are preserved.
resolved_async_chunk = cli_overrides.get("async_chunk")
if resolved_async_chunk is None:
resolved_async_chunk = bool(pipeline.async_chunk)
for stage in pipeline.stages:
stage.yaml_engine_args["async_chunk"] = bool(resolved_async_chunk)
# Apply CLI overrides
result: list[StageConfig] = []
@@ -1063,20 +1068,15 @@ class StageConfigFactory:
model_type: str,
cli_overrides: dict[str, Any],
deploy_config_path: str | None = None,
cli_explicit_keys: set[str] | None = None,
**deprecated_kwargs: Any,
) -> list[StageConfig]:
"""Create StageConfigs from pipeline registry + deploy YAML.
Precedence (high → low):
explicit CLI args > deploy YAML > parser default CLI values
``cli_explicit_keys`` carries the set of long-option attribute names
the user actually typed (captured in ``OmniServeCommand.cmd``). Any
kwarg whose key is not in that set is treated as a parser default
and is only used to fill fields YAML doesn't already cover. When the
set is ``None`` (programmatic ``Omni()`` callers, which have no
argparse layer), every kwarg is treated as explicit.
Precedence: caller-typed (non-None) value > deploy YAML >
StageDeployConfig dataclass default.
"""
_warn_deprecated_kwargs(deprecated_kwargs)
# Resolve deploy config path
if deploy_config_path is None:
deploy_path = _DEPLOY_DIR / f"{model_type}.yaml"
@@ -1093,7 +1093,7 @@ class StageConfigFactory:
deploy_cfg = load_deploy_config(deploy_path)
cli_async_chunk = cli_overrides.get("async_chunk")
if cli_async_chunk is not None and (cli_explicit_keys is None or "async_chunk" in cli_explicit_keys):
if cli_async_chunk is not None:
deploy_cfg.async_chunk = bool(cli_async_chunk)
pipeline_key = deploy_cfg.pipeline or model_type
@@ -1107,25 +1107,10 @@ class StageConfigFactory:
stages = merge_pipeline_deploy(pipeline_cfg, deploy_cfg, cli_overrides)
# Precedence: explicit CLI > yaml > parser-default CLI.
# Per-stage (``stage_N_*``) keys are always treated as explicit.
explicit_overrides: dict[str, Any] = {}
default_overrides: dict[str, Any] = {}
for key, value in cli_overrides.items():
if value is None:
continue
is_per_stage = bool(re.match(r"stage_\d+_", key))
is_explicit = cli_explicit_keys is None or key in cli_explicit_keys or is_per_stage
if is_explicit:
explicit_overrides[key] = value
else:
default_overrides[key] = value
explicit_overrides = {k: v for k, v in cli_overrides.items() if v is not None}
for stage in stages:
yaml_keys = set(stage.yaml_engine_args)
fallback = {k: v for k, v in default_overrides.items() if k not in yaml_keys}
merged = {**fallback, **explicit_overrides}
stage.runtime_overrides = cls._merge_cli_overrides(stage, merged)
stage.runtime_overrides = cls._merge_cli_overrides(stage, explicit_overrides)
return stages
+76
View File
@@ -185,8 +185,26 @@ class OmniEngineArgs(EngineArgs):
def from_cli_args(cls, args: argparse.Namespace) -> "OmniEngineArgs":
attrs = [attr.name for attr in dataclasses.fields(cls)]
engine_args = cls(**{attr: getattr(args, attr) for attr in attrs if hasattr(args, attr)})
engine_args._explicit_fields = frozenset(
attr for attr in attrs if hasattr(args, attr) and getattr(args, attr) is not None
)
return engine_args
@classmethod
def create(cls, **explicit: Any) -> "OmniEngineArgs":
"""Tracks caller-set fields for ``Omni(..., engine_args=ea)``."""
ea = cls(**explicit)
ea._explicit_fields = frozenset(explicit.keys())
return ea
def explicit_kwargs(self) -> dict[str, Any]:
explicit = getattr(self, "_explicit_fields", None)
if explicit is None:
return {
f.name: getattr(self, f.name) for f in dataclasses.fields(self) if getattr(self, f.name) is not None
}
return {k: getattr(self, k) for k in explicit}
def _ensure_omni_models_registered(self):
if hasattr(self, "_omni_models_registered"):
return True
@@ -442,12 +460,50 @@ SHARED_FIELDS: frozenset[str] = frozenset(
}
)
_DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
# Capacity / scheduling.
"async_scheduling",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
# Memory / parallelism.
"data_parallel_size",
"gpu_memory_utilization",
"pipeline_parallel_size",
"tensor_parallel_size",
# Execution / loading.
"enforce_eager",
"distributed_executor_backend",
"dtype",
"quantization",
"trust_remote_code",
# Caching / chunking.
"async_chunk",
"enable_prefix_caching",
"enable_chunked_prefill",
# Model-specific engine extras.
"subtalker_sampling_params",
}
)
_DEPLOY_RUNTIME_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
"devices",
}
)
def orchestrator_field_names() -> frozenset[str]:
"""Return the names of every field on OrchestratorArgs."""
return frozenset(f.name for f in fields(OrchestratorArgs))
def deploy_override_field_names() -> frozenset[str]:
"""Return kwargs whose parser defaults must not override deploy YAML."""
return _DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS | _DEPLOY_RUNTIME_OVERRIDE_FIELDS
def internal_blacklist_keys() -> frozenset[str]:
"""Return the set of CLI keys that must never be forwarded as per-stage
engine overrides.
@@ -595,3 +651,23 @@ def orchestrator_args_from_argparse(args: Any) -> OrchestratorArgs:
if value is not None or f.default is None:
kwargs[f.name] = value
return OrchestratorArgs(**kwargs)
def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
"""Reset stage-level engine flag defaults to ``None``; preserve real
default in help text. Only deploy-YAML override fields are touched.
Idempotent."""
override_dests = deploy_override_field_names()
for action in parser._actions:
if action.dest in ("help", "version") or not action.option_strings:
continue
if action.dest not in override_dests:
continue
if action.default is None or action.default is argparse.SUPPRESS:
continue
if action.help and "(default:" not in action.help and "%(default)" not in action.help:
action.help = f"{action.help} (default: {action.default})"
action.default = None
parser._omni_nullified = True # type: ignore[attr-defined]
+12 -11
View File
@@ -279,13 +279,16 @@ class AsyncOmniEngine:
logger.info(f"[AsyncOmniEngine] Initializing with model {model}")
# Merge typed engine_args fields into kwargs; explicit kwargs take priority.
# Merge tracked engine_args fields into kwargs; explicit kwargs take priority.
if engine_args is not None:
ea_dict = {
f.name: getattr(engine_args, f.name)
for f in dataclasses.fields(engine_args)
if not f.name.startswith("_")
}
if not hasattr(engine_args, "_explicit_fields"):
raise TypeError(
"engine_args=OmniEngineArgs(...) is ambiguous under "
"sentinel-default precedence. Use "
"OmniEngineArgs.create(**explicit) or pass explicit kwargs "
"directly."
)
ea_dict = engine_args.explicit_kwargs()
# Remove model since it is passed as a positional arg already.
ea_dict.pop("model", None)
kwargs = {**ea_dict, **kwargs}
@@ -1238,6 +1241,8 @@ class AsyncOmniEngine:
if not isinstance(default_sampling_params, dict):
default_sampling_params = None
stage_default_sampling_params = default_sampling_params.get("0", {}) if default_sampling_params else {}
if normalized_kwargs.get("dtype") is None:
normalized_kwargs["dtype"] = "auto"
# TODO: hack, convert dtype to string to avoid non-premitive omegaconf create error.
if "dtype" in normalized_kwargs and not isinstance(normalized_kwargs["dtype"], str):
@@ -1388,10 +1393,7 @@ class AsyncOmniEngine:
stage_configs_path = kwargs.get("stage_configs_path", None)
deploy_config_path = kwargs.pop("deploy_config", None)
stage_overrides_json = kwargs.pop("stage_overrides", None)
# Set of CLI keys the user actually typed; ``None`` means we have no
# parser-level info (e.g. programmatic Omni() call) and the lower
# layers should treat all kwargs as explicit.
cli_explicit_keys = kwargs.pop("_cli_explicit_keys", None)
kwargs.pop("_cli_explicit_keys", None)
explicit_stage_configs = kwargs.pop("stage_configs", None)
if explicit_stage_configs is not None:
logger.warning(
@@ -1424,7 +1426,6 @@ class AsyncOmniEngine:
default_stage_cfg_factory=lambda: self._create_default_diffusion_stage_cfg(kwargs),
deploy_config_path=deploy_config_path,
stage_overrides=stage_overrides,
cli_explicit_keys=cli_explicit_keys,
)
# Inject diffusion LoRA-related knobs from kwargs if not present in the stage config.
+4 -11
View File
@@ -9,7 +9,6 @@ import argparse
import json
import os
import signal
import sys
from types import FrameType
from typing import Any
@@ -20,9 +19,9 @@ from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
from vllm_omni.entrypoints.cli.logo import log_logo
from vllm_omni.entrypoints.openai.api_server import omni_run_server
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
logger = init_logger(__name__)
@@ -95,10 +94,6 @@ class OmniServeCommand(CLISubcommand):
if hasattr(args, "model_tag") and args.model_tag is not None:
args.model = args.model_tag
# Stash the set of long-option keys the user actually typed so the
# stage-config factory can give YAML precedence over argparse defaults.
args._cli_explicit_keys = detect_explicit_cli_keys(sys.argv[1:], OmniServeCommand._parser)
if args.headless:
run_headless(args)
else:
@@ -485,6 +480,8 @@ class OmniServeCommand(CLISubcommand):
# Stash via type(self) so the docs hook (which execs this function in a
# sandboxed globals dict via ``DummySelf``) doesn't fail on a NameError.
type(self)._parser = serve_parser
nullify_stage_engine_defaults(serve_parser)
return serve_parser
@@ -540,15 +537,11 @@ def run_headless(args: argparse.Namespace) -> None:
raise ValueError("headless mode requires worker_backend=multi_process")
args_dict = vars(args).copy()
# Preserve the explicit-keys set captured at parse time so per-stage yaml
# values (e.g. stage 1's ``gpu_memory_utilization: 0.5``) are not
# overwritten by argparse defaults for flags the user didn't type.
cli_explicit_keys = args_dict.pop("_cli_explicit_keys", None)
args_dict.pop("_cli_explicit_keys", None)
config_path, stage_configs = load_and_resolve_stage_configs(
model,
args_dict.get("stage_configs_path"),
args_dict,
cli_explicit_keys=cli_explicit_keys,
)
# Locate the stage config that matches stage_id.
+16 -30
View File
@@ -17,7 +17,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.entrypoints.client_request_state import ClientRequestState
from vllm_omni.entrypoints.pd_utils import PDDisaggregationMixin
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys, get_final_stage_id_for_e2e
from vllm_omni.entrypoints.utils import get_final_stage_id_for_e2e
from vllm_omni.metrics.stats import OrchestratorAggregator as OrchestratorMetrics
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from vllm_omni.outputs import OmniRequestOutput
@@ -96,38 +96,23 @@ class OmniBase(PDDisaggregationMixin):
parser: argparse.ArgumentParser | None = None,
**overrides: Any,
) -> OmniBase:
"""Construct an ``Omni`` / ``AsyncOmni`` from an ``argparse.Namespace``.
"""Build from argparse. If ``parser`` is passed and not yet nullified,
un-typed engine fields are reset to ``None``."""
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
Mirrors the ``EngineArgs.from_cli_args`` pattern used upstream and in
``OmniEngineArgs.from_cli_args``. This is the recommended entry point
for any argparse-based caller (offline scripts, tests, CI): it
expands ``vars(args)`` into kwargs and automatically captures which
flags the user typed on the command line so that argparse defaults
do not silently override deploy YAML values.
if parser is not None and not getattr(parser, "_omni_nullified", False):
from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
)
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
Passing ``parser`` is strongly recommended: without it, flag-to-dest
resolution falls back to a name-based heuristic that misidentifies
flags with ``dest=`` overrides, alias flags, and ``--disable-X`` /
``store_false`` pairs. See :func:`detect_explicit_cli_keys`.
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
override_dests = deploy_override_field_names()
for key in list(kwargs):
if key in override_dests and key not in explicit:
kwargs[key] = None
Args:
args: Parsed argparse namespace from ``parser.parse_args()``.
parser: The argparse parser used to produce ``args``. When
provided, each user-typed flag is resolved to its real
``dest`` via the parser's action table.
**overrides: Extra keyword arguments that take precedence over
attributes on ``args``.
Example::
parser = FlexibleArgumentParser()
parser.add_argument("--model", required=True)
args = parser.parse_args()
omni = Omni.from_cli_args(args, parser=parser) # preferred
omni = Omni.from_cli_args(args, parser=parser, model="other")
"""
kwargs: dict[str, Any] = {**vars(args), **overrides}
kwargs["_cli_explicit_keys"] = detect_explicit_cli_keys(sys.argv[1:], parser)
kwargs.update(overrides)
return cls(**kwargs)
def __init__(
@@ -136,6 +121,7 @@ class OmniBase(PDDisaggregationMixin):
**kwargs: Any,
) -> None:
engine_args: OmniEngineArgs | None = kwargs.pop("engine_args", None)
stage_init_timeout = kwargs.pop("stage_init_timeout", 300)
init_timeout = kwargs.pop("init_timeout", 600)
log_stats = kwargs.pop("log_stats", False)
@@ -2779,6 +2779,8 @@ if __name__ == "__main__":
from vllm.entrypoints.openai.cli_args import make_arg_parser
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
parser = argparse.ArgumentParser(description="vLLM-Omni OpenAI-Compatible REST API server")
parser = make_arg_parser(parser)
registered_flags = set()
@@ -2790,6 +2792,7 @@ if __name__ == "__main__":
parser.add_argument(
"--enable-sleep-mode", action="store_true", default=False, help="Enable GPU memory pool for sleep mode."
)
nullify_stage_engine_defaults(parser)
args = parser.parse_args()
if not hasattr(args, "model_tag"):
setattr(args, "model_tag", args.model)
+19 -15
View File
@@ -21,6 +21,18 @@ PROJECT_ROOT = Path(__file__).parent.parent.parent
logger = init_logger(__name__)
def _warn_deprecated_explicit_keys(kwargs: dict[str, Any]) -> None:
if "cli_explicit_keys" in kwargs:
import warnings
warnings.warn(
"cli_explicit_keys= is deprecated and ignored. Remove the kwarg.",
DeprecationWarning,
stacklevel=3,
)
_DIFFUSERS_CLASS_TO_CONFIG: dict[str, str] = {
"GlmImagePipeline": "glm_image",
}
@@ -343,7 +355,7 @@ def load_stage_configs_from_model(
base_engine_args: dict | None = None,
deploy_config_path: str | None = None,
stage_overrides: dict[str, dict[str, Any]] | None = None,
cli_explicit_keys: set[str] | None = None,
**deprecated_kwargs: Any,
) -> list:
"""Load stage configurations from model's default config file.
@@ -358,33 +370,25 @@ def load_stage_configs_from_model(
base_engine_args: Base engine args to merge as CLI overrides.
deploy_config_path: Optional explicit deploy config path.
stage_overrides: Per-stage overrides from --stage-overrides.
cli_explicit_keys: Set of CLI keys the user actually typed. When
provided, only these keys override deploy YAML; argparse defaults
stay subordinate to YAML. ``None`` means treat every kwarg as
explicit (programmatic ``Omni()`` calls).
Returns:
List of stage configuration dictionaries
"""
_warn_deprecated_explicit_keys(deprecated_kwargs)
if base_engine_args is None:
base_engine_args = {}
cli_overrides = _convert_dataclasses_to_dict(dict(base_engine_args))
# Per-stage JSON overrides are always explicit (the user typed --stage-overrides).
explicit = set(cli_explicit_keys) if cli_explicit_keys is not None else None
if stage_overrides:
for stage_id_str, overrides in stage_overrides.items():
for key, val in overrides.items():
stage_key = f"stage_{stage_id_str}_{key}"
cli_overrides[stage_key] = val
if explicit is not None:
explicit.add(stage_key)
cli_overrides[f"stage_{stage_id_str}_{key}"] = val
stages = StageConfigFactory.create_from_model(
model,
cli_overrides=cli_overrides,
deploy_config_path=deploy_config_path,
cli_explicit_keys=explicit,
)
if stages is not None:
# Convert StageConfig objects to OmegaConf for backward compat
@@ -545,7 +549,7 @@ def load_and_resolve_stage_configs(
default_stage_cfg_factory: Any = None,
deploy_config_path: str | None = None,
stage_overrides: dict[str, dict[str, Any]] | None = None,
cli_explicit_keys: set[str] | None = None,
**deprecated_kwargs: Any,
) -> tuple[str, list]:
"""Load stage configurations from model or YAML file with fallback to defaults.
@@ -589,6 +593,8 @@ def load_and_resolve_stage_configs(
stage_configs_path,
)
_warn_deprecated_explicit_keys(deprecated_kwargs)
if deploy_config_path is not None:
config_path = deploy_config_path
stage_configs = load_stage_configs_from_model(
@@ -596,7 +602,6 @@ def load_and_resolve_stage_configs(
base_engine_args=kwargs,
deploy_config_path=deploy_config_path,
stage_overrides=stage_overrides,
cli_explicit_keys=cli_explicit_keys,
)
if not stage_configs:
if default_stage_cfg_factory is not None:
@@ -610,7 +615,6 @@ def load_and_resolve_stage_configs(
model,
base_engine_args=kwargs,
stage_overrides=stage_overrides,
cli_explicit_keys=cli_explicit_keys,
)
if not stage_configs:
if default_stage_cfg_factory is not None: