489 lines
18 KiB
Python
489 lines
18 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Tests for vllm_omni.engine.arg_utils — invariants that must
|
|
hold for the orchestrator/engine/server CLI flag partition."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from dataclasses import dataclass, fields
|
|
|
|
import pytest
|
|
|
|
from vllm_omni.engine.arg_utils import (
|
|
SHARED_FIELDS,
|
|
derive_server_dests_from_vllm_parser,
|
|
internal_blacklist_keys,
|
|
orchestrator_args_from_argparse,
|
|
orchestrator_field_names,
|
|
split_kwargs,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fake engine class for unit testing — avoids pulling in the full vllm
|
|
# EngineArgs and its heavy __post_init__ at test time.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class _FakeEngineArgs:
|
|
"""Stand-in for OmniEngineArgs with a representative subset of fields."""
|
|
|
|
model: str = ""
|
|
stage_id: int = 0
|
|
max_num_seqs: int = 64
|
|
gpu_memory_utilization: float = 0.9
|
|
async_chunk: bool = False # also in OrchestratorArgs → shared
|
|
log_stats: bool = False # also in OrchestratorArgs → shared
|
|
stage_configs_path: str | None = None
|
|
|
|
|
|
# ============================================================================
|
|
# Invariant 1 — OrchestratorArgs and engine must not ambiguously overlap.
|
|
# ============================================================================
|
|
|
|
|
|
def test_no_ambiguous_overlap_with_fake_engine():
|
|
"""OrchestratorArgs ∩ engine fields must be ⊆ SHARED_FIELDS."""
|
|
orch = orchestrator_field_names()
|
|
engine = {f.name for f in fields(_FakeEngineArgs)}
|
|
overlap = orch & engine
|
|
unexpected = overlap - SHARED_FIELDS
|
|
assert not unexpected, (
|
|
f"Fields declared in both OrchestratorArgs and the engine class "
|
|
f"but not in SHARED_FIELDS: {sorted(unexpected)}. These cause "
|
|
f"double-routing — either remove the duplicate declaration or add "
|
|
f"to SHARED_FIELDS if sharing is intentional."
|
|
)
|
|
|
|
|
|
def test_no_ambiguous_overlap_with_real_engine():
|
|
"""Same check, but against the real OmniEngineArgs."""
|
|
try:
|
|
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
|
except Exception as exc:
|
|
pytest.skip(f"OmniEngineArgs not importable: {exc}")
|
|
|
|
orch = orchestrator_field_names()
|
|
engine = {f.name for f in fields(OmniEngineArgs)}
|
|
overlap = orch & engine
|
|
unexpected = overlap - SHARED_FIELDS
|
|
assert not unexpected, (
|
|
f"Real OmniEngineArgs has ambiguous overlap with OrchestratorArgs: "
|
|
f"{sorted(unexpected)}. Update SHARED_FIELDS or remove duplication."
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# Invariant 2 — split_kwargs partitions correctly.
|
|
# ============================================================================
|
|
|
|
|
|
def test_split_orchestrator_only():
|
|
"""Pure orchestrator fields go to OrchestratorArgs, not engine_kwargs."""
|
|
raw = {"stage_init_timeout": 500, "worker_backend": "ray"}
|
|
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
|
|
assert orch.stage_init_timeout == 500
|
|
assert orch.worker_backend == "ray"
|
|
assert "stage_init_timeout" not in engine
|
|
assert "worker_backend" not in engine
|
|
|
|
|
|
def test_split_engine_only():
|
|
"""Pure engine fields go to engine_kwargs, not OrchestratorArgs."""
|
|
raw = {"max_num_seqs": 128, "gpu_memory_utilization": 0.85}
|
|
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
|
|
assert engine["max_num_seqs"] == 128
|
|
assert engine["gpu_memory_utilization"] == 0.85
|
|
# These fields don't exist on OrchestratorArgs at all.
|
|
|
|
|
|
def test_split_shared_fields_go_to_both():
|
|
"""Fields in SHARED_FIELDS are copied to both buckets."""
|
|
raw = {"model": "Qwen/Qwen2.5-Omni-7B", "log_stats": True}
|
|
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
|
|
assert orch.log_stats is True
|
|
assert engine["model"] == "Qwen/Qwen2.5-Omni-7B"
|
|
assert engine["log_stats"] is True
|
|
|
|
|
|
def test_split_drops_unclassified():
|
|
"""Unclassified fields (uvicorn/server) are dropped silently."""
|
|
raw = {
|
|
"max_num_seqs": 64, # engine
|
|
"host": "0.0.0.0", # unclassified (server)
|
|
"port": 8091, # unclassified (server)
|
|
"ssl_keyfile": "key.pem", # unclassified (server)
|
|
}
|
|
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
|
|
assert engine == {"max_num_seqs": 64}
|
|
assert "host" not in engine
|
|
assert "port" not in engine
|
|
assert "ssl_keyfile" not in engine
|
|
|
|
|
|
def test_split_mixed_real_world():
|
|
"""End-to-end: raw CLI kwargs with all three classes present."""
|
|
raw = {
|
|
# orchestrator
|
|
"stage_init_timeout": 400,
|
|
"deploy_config": "/tmp/deploy.yaml",
|
|
"worker_backend": "multi_process",
|
|
"async_chunk": True,
|
|
# engine
|
|
"max_num_seqs": 32,
|
|
"gpu_memory_utilization": 0.8,
|
|
# shared
|
|
"model": "Qwen/Qwen3-Omni",
|
|
"log_stats": False,
|
|
# server / unclassified
|
|
"host": "0.0.0.0",
|
|
"port": 8091,
|
|
"api_key": "secret",
|
|
# None values
|
|
"ray_address": None,
|
|
}
|
|
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
|
|
|
|
# Orchestrator side
|
|
assert orch.stage_init_timeout == 400
|
|
assert orch.deploy_config == "/tmp/deploy.yaml"
|
|
assert orch.worker_backend == "multi_process"
|
|
assert orch.async_chunk is True
|
|
assert orch.log_stats is False # shared, read from raw
|
|
assert orch.ray_address is None # default preserved
|
|
|
|
# Engine side
|
|
assert engine["max_num_seqs"] == 32
|
|
assert engine["gpu_memory_utilization"] == 0.8
|
|
assert engine["model"] == "Qwen/Qwen3-Omni"
|
|
assert engine["log_stats"] is False
|
|
assert "host" not in engine
|
|
assert "port" not in engine
|
|
assert "api_key" not in engine
|
|
# orchestrator-only keys never reach engine
|
|
assert "stage_init_timeout" not in engine
|
|
assert "deploy_config" not in engine
|
|
assert "async_chunk" not in engine
|
|
|
|
|
|
# ============================================================================
|
|
# Invariant 3 — user-typed unclassifiable flags warn (don't fail silently).
|
|
# ============================================================================
|
|
|
|
|
|
def test_user_typed_unclassified_warns(caplog):
|
|
"""If the user types a flag we can't route, warn — don't silently drop."""
|
|
raw = {"bogus_flag": "value", "max_num_seqs": 64}
|
|
with caplog.at_level(logging.WARNING, logger="vllm_omni.engine.arg_utils"):
|
|
split_kwargs(raw, engine_cls=_FakeEngineArgs, user_typed={"bogus_flag"})
|
|
assert any("bogus_flag" in rec.message for rec in caplog.records), (
|
|
f"Expected warning mentioning 'bogus_flag', got: {[rec.message for rec in caplog.records]}"
|
|
)
|
|
|
|
|
|
def test_unclassified_without_user_typed_silent(caplog):
|
|
"""Without user_typed, unclassified keys drop silently (argparse defaults
|
|
for server flags shouldn't spam logs on every launch)."""
|
|
raw = {"host": "0.0.0.0", "port": 8091, "max_num_seqs": 64}
|
|
with caplog.at_level(logging.WARNING, logger="vllm_omni.engine.arg_utils"):
|
|
split_kwargs(raw, engine_cls=_FakeEngineArgs, user_typed=None)
|
|
# No warnings because we don't know these were user-typed.
|
|
assert not any("host" in rec.message or "port" in rec.message for rec in caplog.records)
|
|
|
|
|
|
# ============================================================================
|
|
# Invariant 4 — CLI flag classification completeness.
|
|
# Catches new flags added without updating OrchestratorArgs or OmniEngineArgs.
|
|
# ============================================================================
|
|
|
|
|
|
def test_all_omni_cli_flags_classified():
|
|
"""Every vllm-omni-added CLI flag must be classifiable.
|
|
|
|
Runs ``OmniServeCommand.subparser_init`` and checks that every new
|
|
argument (compared to vllm's base parser) is either:
|
|
- a field on OrchestratorArgs, OR
|
|
- a field on OmniEngineArgs, OR
|
|
- in SHARED_FIELDS
|
|
"""
|
|
try:
|
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
|
|
|
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
|
from vllm_omni.entrypoints.cli.serve import OmniServeCommand
|
|
except Exception as exc:
|
|
pytest.skip(f"Cannot build parser in this environment: {exc}")
|
|
|
|
# Build the serve parser
|
|
root = FlexibleArgumentParser()
|
|
subparsers = root.add_subparsers()
|
|
cmd = OmniServeCommand()
|
|
try:
|
|
parser = cmd.subparser_init(subparsers)
|
|
except Exception as exc:
|
|
pytest.skip(f"subparser_init failed (dev env issue): {exc}")
|
|
|
|
all_dests = {a.dest for a in parser._actions if a.dest and a.dest not in {"help", "model_tag"}}
|
|
|
|
orch = orchestrator_field_names()
|
|
engine = {f.name for f in fields(OmniEngineArgs)}
|
|
server_derived = derive_server_dests_from_vllm_parser()
|
|
|
|
unclassified = all_dests - orch - engine - SHARED_FIELDS - server_derived
|
|
# Some argparse-internal dests (suppressed, private) may not match —
|
|
# filter those out.
|
|
unclassified = {d for d in unclassified if not d.startswith("_")}
|
|
|
|
assert not unclassified, (
|
|
f"These CLI flags are not classified as "
|
|
f"orchestrator/engine/shared/server: {sorted(unclassified)}. "
|
|
f"Add them to OrchestratorArgs (if consumed by orchestrator), "
|
|
f"OmniEngineArgs (if consumed by per-stage engine), or the known-server "
|
|
f"allowlist (if they're vllm frontend flags). "
|
|
f"If intentional (e.g. a new CLI-only flag that doesn't map to either "
|
|
f"dataclass), add it to a KNOWN_UNROUTED allowlist."
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# argparse interop (Phase 3).
|
|
# ============================================================================
|
|
|
|
|
|
def test_orchestrator_args_from_argparse():
|
|
"""Can build OrchestratorArgs from an argparse.Namespace."""
|
|
import argparse
|
|
|
|
ns = argparse.Namespace(
|
|
stage_init_timeout=500,
|
|
deploy_config="/tmp/x.yaml",
|
|
max_num_seqs=64, # engine field — ignored
|
|
host="0.0.0.0", # server field — ignored
|
|
)
|
|
orch = orchestrator_args_from_argparse(ns)
|
|
assert orch.stage_init_timeout == 500
|
|
assert orch.deploy_config == "/tmp/x.yaml"
|
|
assert orch.worker_backend == "multi_process" # default
|
|
|
|
|
|
def test_derive_server_dests_returns_frozenset():
|
|
"""Server-dest derivation returns a frozenset (possibly empty)."""
|
|
result = derive_server_dests_from_vllm_parser()
|
|
assert isinstance(result, frozenset)
|
|
|
|
|
|
# ============================================================================
|
|
# internal_blacklist_keys — single source of truth for per-stage forwarding.
|
|
# ============================================================================
|
|
|
|
|
|
def test_internal_blacklist_keys_derived_from_orchestrator():
|
|
"""Blacklist is exactly OrchestratorArgs fields minus SHARED_FIELDS.
|
|
|
|
This function replaces the old hardcoded INTERNAL_STAGE_OVERRIDE_KEYS
|
|
frozenset. Asserts the contract so future changes to OrchestratorArgs
|
|
automatically propagate to the blacklist.
|
|
"""
|
|
blacklist = internal_blacklist_keys()
|
|
assert blacklist == orchestrator_field_names() - SHARED_FIELDS
|
|
# Spot-check expected entries
|
|
assert "stage_init_timeout" in blacklist
|
|
assert "deploy_config" in blacklist
|
|
assert "async_chunk" in blacklist
|
|
# Shared fields must NOT appear — they flow to both orchestrator and engine
|
|
assert "model" not in blacklist
|
|
assert "log_stats" not in blacklist
|
|
|
|
|
|
# ============================================================================
|
|
# Boundary value analysis — edge cases around split_kwargs.
|
|
# ============================================================================
|
|
|
|
|
|
def test_split_empty_kwargs():
|
|
"""Empty kwargs yields default OrchestratorArgs and empty engine dict."""
|
|
orch, engine = split_kwargs({}, engine_cls=_FakeEngineArgs)
|
|
assert orch.stage_init_timeout == 300 # dataclass default
|
|
assert orch.worker_backend == "multi_process" # dataclass default
|
|
assert engine == {}
|
|
|
|
|
|
def test_split_all_none_values_preserved_on_orchestrator():
|
|
"""None values for orchestrator fields are kept (represents 'not set')."""
|
|
raw = {"ray_address": None, "deploy_config": None, "max_num_seqs": None}
|
|
orch, engine = split_kwargs(raw, engine_cls=_FakeEngineArgs)
|
|
assert orch.ray_address is None
|
|
assert orch.deploy_config is None
|
|
# Engine-side None still passes through; caller decides semantics downstream.
|
|
assert engine.get("max_num_seqs") is None
|
|
|
|
|
|
def test_split_user_typed_with_empty_kwargs_no_warn(caplog):
|
|
"""user_typed non-empty but kwargs empty — no warnings emitted."""
|
|
with caplog.at_level(logging.WARNING, logger="vllm_omni.engine.arg_utils"):
|
|
split_kwargs({}, engine_cls=_FakeEngineArgs, user_typed={"nothing"})
|
|
assert not caplog.records
|
|
|
|
|
|
def test_ambiguous_field_strict_raises():
|
|
"""strict=True raises ValueError on overlap outside SHARED_FIELDS."""
|
|
|
|
# deploy_config is on OrchestratorArgs; declaring it on the engine class
|
|
# too (without adding to SHARED_FIELDS) creates an ambiguous route.
|
|
@dataclass
|
|
class _AmbiguousEngine:
|
|
deploy_config: str | None = None
|
|
|
|
with pytest.raises(ValueError, match="both OrchestratorArgs and"):
|
|
split_kwargs({"deploy_config": "x"}, engine_cls=_AmbiguousEngine, strict=True)
|
|
|
|
|
|
def test_ambiguous_field_non_strict_routes_to_orchestrator(caplog):
|
|
"""strict=False logs ERROR but routes the ambiguous field to orchestrator."""
|
|
|
|
@dataclass
|
|
class _AmbiguousEngine:
|
|
deploy_config: str | None = None
|
|
|
|
with caplog.at_level(logging.ERROR, logger="vllm_omni.engine.arg_utils"):
|
|
orch, engine = split_kwargs({"deploy_config": "x"}, engine_cls=_AmbiguousEngine, strict=False)
|
|
assert orch.deploy_config == "x"
|
|
assert "deploy_config" not in engine
|
|
assert any("both OrchestratorArgs" in r.message for r in caplog.records)
|
|
|
|
|
|
# Sentinel-default precedence invariants (#3035)
|
|
|
|
|
|
def _build_full_serve_parser():
|
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
|
|
|
try:
|
|
from vllm.entrypoints.openai.cli_args import make_arg_parser
|
|
except ImportError:
|
|
pytest.skip("vllm parser not importable")
|
|
return make_arg_parser(FlexibleArgumentParser())
|
|
|
|
|
|
def test_nullify_stage_engine_defaults_resets_inherited_defaults():
|
|
import argparse
|
|
|
|
from vllm_omni.config.stage_config import deploy_override_field_names
|
|
from vllm_omni.engine.arg_utils import (
|
|
nullify_stage_engine_defaults,
|
|
)
|
|
|
|
parser = _build_full_serve_parser()
|
|
nullify_stage_engine_defaults(parser)
|
|
|
|
override_dests = deploy_override_field_names()
|
|
offenders = [
|
|
(a.dest, a.default)
|
|
for a in parser._actions
|
|
if a.dest not in ("help", "version")
|
|
and a.option_strings
|
|
and a.dest in override_dests
|
|
and a.default is not None
|
|
and a.default is not argparse.SUPPRESS
|
|
]
|
|
assert not offenders, f"Stage flags with non-None defaults after nullify: {offenders}"
|
|
|
|
|
|
def test_non_override_flags_keep_real_defaults_after_nullify():
|
|
import argparse
|
|
|
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--hsdp-shard-size", type=int, default=-1, help="HSDP shard size.")
|
|
parser.add_argument("--max-num-seqs", type=int, default=64, help="Max num seqs.")
|
|
nullify_stage_engine_defaults(parser)
|
|
|
|
hsdp = next(a for a in parser._actions if a.dest == "hsdp_shard_size")
|
|
max_num_seqs = next(a for a in parser._actions if a.dest == "max_num_seqs")
|
|
assert hsdp.default == -1
|
|
assert max_num_seqs.default is None
|
|
|
|
|
|
def test_help_text_preserves_default_after_nullify():
|
|
# Real defaults must stay visible in --help even though parser stores None.
|
|
import argparse
|
|
|
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--max-num-seqs", type=int, default=42, help="Example knob.")
|
|
nullify_stage_engine_defaults(parser)
|
|
|
|
action = next(a for a in parser._actions if a.dest == "max_num_seqs")
|
|
assert action.default is None
|
|
assert "(default: 42)" in action.help
|
|
|
|
|
|
_OMNIENGINEARGS_USER_INPUT_FIELDS = frozenset(
|
|
{
|
|
"model_stage",
|
|
"model_arch",
|
|
"engine_output_type",
|
|
"hf_config_name",
|
|
"custom_process_next_stage_input_func",
|
|
"subtalker_sampling_params",
|
|
"async_chunk",
|
|
"omni_kv_config",
|
|
"quantization_config",
|
|
"worker_type",
|
|
"task_type",
|
|
"worker_cls",
|
|
"enable_sleep_mode",
|
|
"omni_master_address",
|
|
"omni_master_port",
|
|
"stage_configs_path",
|
|
"output_modalities",
|
|
"log_stats",
|
|
"custom_pipeline_args",
|
|
}
|
|
)
|
|
|
|
|
|
def test_omniengineargs_user_input_fields_default_to_none():
|
|
try:
|
|
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
|
except Exception as exc:
|
|
pytest.skip(f"OmniEngineArgs not importable: {exc}")
|
|
|
|
offenders = [
|
|
(f.name, f.default)
|
|
for f in fields(OmniEngineArgs)
|
|
if f.name in _OMNIENGINEARGS_USER_INPUT_FIELDS
|
|
and f.default is not dataclasses.MISSING
|
|
and f.default is not None
|
|
]
|
|
assert not offenders, f"User-input fields with non-None defaults: {offenders}"
|
|
|
|
|
|
def test_omniengineargs_create_tracks_explicit_fields():
|
|
try:
|
|
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
|
except Exception as exc:
|
|
pytest.skip(f"OmniEngineArgs not importable: {exc}")
|
|
|
|
ea = OmniEngineArgs.create(model="x", gpu_memory_utilization=0.5)
|
|
assert ea._explicit_fields == frozenset({"model", "gpu_memory_utilization"})
|
|
assert ea.explicit_kwargs() == {"model": "x", "gpu_memory_utilization": 0.5}
|
|
|
|
|
|
def test_omniengineargs_bare_constructor_has_no_explicit_tracking():
|
|
try:
|
|
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
|
except Exception as exc:
|
|
pytest.skip(f"OmniEngineArgs not importable: {exc}")
|
|
|
|
ea = OmniEngineArgs(model="x")
|
|
assert not hasattr(ea, "_explicit_fields")
|
|
assert "model" in ea.explicit_kwargs()
|
|
|
|
|
|
# dataclasses already imported via ``from dataclasses import dataclass, fields``
|
|
import dataclasses # noqa: E402 -- needed for MISSING sentinel above
|