[Core] Retain structured configs through runtime startup (#6849)

Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
This commit is contained in:
maithilijoshi20 authored and GitHub committed 2026-09-16 20:47:38 -07:00
1 parent becfefd440
commit f1a6e7ce2a
49 files changed
+1856 -331

No files matched your search

+14 -13
View File
@@ -1048,6 +1048,7 @@ def run_headless(args: TrackingNamespace) -> None:
)
from vllm_omni.engine.stage_init_utils import (
build_engine_args_dict,
build_engine_args_dict_from_omni_stage_config,
build_vllm_config,
get_stage_connector_spec,
inject_omni_kv_connector_config,
@@ -1141,21 +1142,21 @@ def run_headless(args: TrackingNamespace) -> None:
stage_connector_spec = get_stage_connector_spec(
omni_transfer_config=omni_transfer_config,
stage_id=stage_id,
async_chunk=bool(stage_cfg.engine_args.get("async_chunk", False)),
async_chunk=bool(
getattr(getattr(stage_cfg, "connector_config", None), "async_chunk", None)
if hasattr(stage_cfg, "connector_config")
else stage_cfg.engine_args.get("async_chunk", False)
),
)
# ``runtime_cfg`` is mostly inherited from the parent's
# CUDA_VISIBLE_DEVICES; when ``--omni-dp-size-local > 1`` we additionally
# bracket each replica's spawn below with setup_stage_devices so they
# don't all stack on cuda:0 (see ``per_replica_devices`` above).
# Headless startup still supplies the legacy OmegaConf stage shape through
# the stable adapter entry point. The implementation switches only when
# RFC #4021 threads structured stage configs through the launch plan.
engine_args_dict = build_engine_args_dict(
stage_cfg,
model,
stage_connector_spec=stage_connector_spec,
cli_tokenizer=getattr(args, "tokenizer", None),
engine_args_dict = (
build_engine_args_dict_from_omni_stage_config(
stage_cfg, model, stage_connector_spec=stage_connector_spec, cli_tokenizer=getattr(args, "tokenizer", None)
)
if hasattr(stage_cfg, "connector_config")
else build_engine_args_dict(
stage_cfg, model, stage_connector_spec=stage_connector_spec, cli_tokenizer=getattr(args, "tokenizer", None)
)
)
inject_omni_kv_connector_config(engine_args_dict, omni_kv_connector, stage_id)
+21 -5
View File
@@ -248,10 +248,27 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
return obj.get(key, default)
return getattr(obj, key, default)
@classmethod
def _stage_model_metadata(cls, stage: Any) -> tuple[str | None, str | None]:
"""Return ``(model_arch, model_stage)`` for typed or legacy stages."""
model_config = cls._stage_get(stage, "model_config")
model_arch = cls._stage_get(model_config, "model_arch")
model_stage = cls._stage_get(stage, "model_stage")
topology = cls._stage_get(stage, "stage_pipeline_config")
if model_stage is None:
model_stage = cls._stage_get(topology, "model_stage")
engine_args = cls._stage_get(stage, "engine_args")
if model_arch is None:
model_arch = cls._stage_get(engine_args, "model_arch")
if model_stage is None:
model_stage = cls._stage_get(engine_args, "model_stage")
return model_arch, model_stage
def _has_minicpmo45_stage(self) -> bool:
for stage in getattr(self.engine_client, "stage_configs", []) or []:
engine_args = self._stage_get(stage, "engine_args")
model_arch = self._stage_get(engine_args, "model_arch")
model_arch, _ = self._stage_model_metadata(stage)
if model_arch == "MiniCPMO45OmniForConditionalGeneration":
return True
return False
@@ -324,11 +341,10 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
if not hasattr(sp, "output_kind"):
continue
engine_args = self._stage_get(stage, "engine_args")
if self._stage_get(engine_args, "model_arch") != "MiniCPMO45OmniForConditionalGeneration":
model_arch, model_stage = self._stage_model_metadata(stage)
if model_arch != "MiniCPMO45OmniForConditionalGeneration":
continue
model_stage = self._stage_get(engine_args, "model_stage")
if model_stage == "llm":
sp.output_kind = RequestOutputKind.FINAL_ONLY
elif model_stage == "tts":
+33 -13
View File
@@ -36,6 +36,7 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils import random_uuid
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.config.stage_config import StagePipelineConfig
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler
from vllm_omni.entrypoints.openai.protocol.audio import (
AudioResponse,
@@ -76,6 +77,24 @@ _SPEECH_USAGE_TOTAL_TOKENS_HEADER = "X-VLLM-OMNI-TOTAL-TOKENS"
_SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TEXT-TOKENS"
_SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS"
def _stage_speech_metadata(stage: Any) -> tuple[str | None, str | None, str | None]:
"""Return speech routing metadata from typed or legacy stage configs."""
topology = getattr(stage, "stage_pipeline_config", None)
if isinstance(topology, StagePipelineConfig):
return (
getattr(topology, "model_stage", None),
getattr(getattr(stage, "model_config", None), "model_arch", None),
getattr(stage, "worker_type", None),
)
engine_args = getattr(stage, "engine_args", None)
return (
getattr(engine_args, "model_stage", None),
getattr(engine_args, "model_arch", None),
getattr(engine_args, "worker_type", None),
)
# TTS Configuration
#
# The stage-key -> model-type mapping is NOT declared here: it is derived from
@@ -472,12 +491,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""Find and return the TTS stage config, or None if not found."""
tts_stage_keys = all_tts_stage_keys()
entry_stage_archs = tts_entry_stage_archs()
all_stages = frozenset(stage.engine_args.model_stage for stage in self.engine_client.stage_configs)
all_stages = frozenset(
model_stage
for stage in self.engine_client.stage_configs
if (model_stage := _stage_speech_metadata(stage)[0]) is not None
)
for stage in self.engine_client.stage_configs:
engine_args = stage.engine_args
model_stage = engine_args.model_stage
model_arch = getattr(engine_args, "model_arch", None)
worker_type = getattr(engine_args, "worker_type", None)
model_stage, model_arch, worker_type = _stage_speech_metadata(stage)
if model_stage in tts_stage_keys:
# Owning the stage key is not always enough: a model may be
# speech-capable only in some deployment topologies. Ask the
@@ -501,10 +521,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""
if self._tts_stage is None:
return None
return detect_tts_model_type(
getattr(self._tts_stage.engine_args, "model_stage", None),
getattr(self._tts_stage.engine_args, "model_arch", None),
)
return detect_tts_model_type(*_stage_speech_metadata(self._tts_stage)[:2])
def _compute_max_instructions_length(self) -> int:
"""Compute max instructions length with precedence: CLI > stage config > default.
@@ -518,7 +535,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# 2. Try to get from TTS stage config
if self._tts_stage is not None:
tts_args = getattr(self._tts_stage, "tts_args", {})
topology = getattr(self._tts_stage, "stage_pipeline_config", None)
tts_args = (
getattr(topology, "extras", {}).get("tts_args", {})
if isinstance(topology, StagePipelineConfig)
else getattr(self._tts_stage, "tts_args", {})
)
if "max_instructions_length" in tts_args:
return tts_args["max_instructions_length"]
@@ -1896,9 +1918,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# chat-template markers (im_start_token_id 151644). A raw-text
# prompt produces a 1-token placeholder that crashes the talker's
# prefill/decode handoff. Reject early with an actionable message.
stage_names = {
getattr(getattr(s, "engine_args", None), "model_stage", None) for s in self.engine_client.stage_configs
}
stage_names = {_stage_speech_metadata(s)[0] for s in self.engine_client.stage_configs}
if "talker" in stage_names:
raise ValueError(
"The /v1/audio/speech endpoint is only supported for "
+26 -24
View File
@@ -53,6 +53,30 @@ logger = init_logger(__name__)
_VIDEO_RESPONSE_FRAME_CONVERSION_WORKERS = 8
def _config_value(config: Any, key: str, default: Any = None) -> Any:
if isinstance(config, Mapping):
return config.get(key, default)
return getattr(config, key, default)
def _stage_diffusion_model_class_name(stage_config: Any) -> str | None:
"""Resolve a stage's diffusion class across typed and legacy shapes."""
diffusion_config = _config_value(stage_config, "diffusion_config")
model_class_name = _config_value(diffusion_config, "model_class_name")
if model_class_name:
return str(model_class_name)
model_config = _config_value(stage_config, "model_config")
model_arch = _config_value(model_config, "model_arch") or _config_value(stage_config, "model_arch")
if model_arch:
return str(model_arch)
engine_args = _config_value(stage_config, "engine_args", {})
model_class_name = _config_value(engine_args, "model_class_name")
return str(model_class_name) if model_class_name else None
if TYPE_CHECKING:
from vllm_omni.diffusion.data import OmniDiffusionConfig
@@ -195,18 +219,7 @@ class OmniOpenAIServingVideo:
model_class_name = getattr(od_config, "model_class_name", None)
model_archs = [model_class_name]
for stage_config in self.stage_configs or ():
stage_get = (
stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None)
)
engine_args = stage_get("engine_args") or {}
model_archs.extend(
(
stage_get("model_arch"),
engine_args.get("model_class_name")
if isinstance(engine_args, Mapping)
else getattr(engine_args, "model_class_name", None),
)
)
model_archs.append(_stage_diffusion_model_class_name(stage_config))
metadata_capability = any(
get_diffusion_model_metadata(model_arch).supports_mixed_reference_inputs for model_arch in model_archs
)
@@ -223,18 +236,7 @@ class OmniOpenAIServingVideo:
od_config = self._resolve_diffusion_od_config()
model_archs = [None if od_config is None else getattr(od_config, "model_class_name", None)]
for stage_config in self.stage_configs or ():
stage_get = (
stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None)
)
engine_args = stage_get("engine_args") or {}
model_archs.extend(
(
stage_get("model_arch"),
engine_args.get("model_class_name")
if isinstance(engine_args, Mapping)
else getattr(engine_args, "model_class_name", None),
)
)
model_archs.append(_stage_diffusion_model_class_name(stage_config))
supported: set[str] = set()
for model_arch in model_archs:
@@ -42,7 +42,11 @@ class AudexAdapter(ARTTSAdapter):
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
stage = getattr(getattr(self.ctx.server._tts_stage, "engine_args", None), "model_stage", None)
stage_config = self.ctx.server._tts_stage
topology = getattr(stage_config, "stage_pipeline_config", None)
stage = getattr(topology, "model_stage", None)
if stage is None:
stage = getattr(getattr(stage_config, "engine_args", None), "model_stage", None)
profile, folder = (
("full", "checkpoint_folder_full") if stage == "audex_omni" else ("tts", "checkpoint_folder_audiogen")
)
@@ -39,6 +39,21 @@ DEFAULT_TTS_LANGUAGES = frozenset(
)
def resolve_stage_model_path(engine_client: Any) -> str | None:
"""Prefer typed stage model overrides, then legacy and served-model values."""
stages = getattr(engine_client, "stage_configs", ()) or ()
for stage in stages:
model_path = getattr(getattr(stage, "model_config", None), "model", None)
if model_path:
return str(model_path)
for stage in stages:
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
if model_path:
return str(model_path)
model_path = getattr(engine_client, "model", None)
return str(model_path) if model_path else None
def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: dict | None = None) -> str:
"""Stable hash of the real Stage 0 conditioning for the prefix cache.
@@ -9,7 +9,12 @@ import numpy as np
from vllm.inputs import tokens_input
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
from vllm_omni.entrypoints.openai.tts_adapters.base import (
ARTTSAdapter,
PreparedRequest,
apply_max_new_tokens,
resolve_stage_model_path,
)
if TYPE_CHECKING:
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
@@ -81,17 +86,7 @@ class HiggsAudioV2Adapter(ARTTSAdapter):
from transformers import AutoProcessor
model_path = None
for stage in self.engine_client.stage_configs:
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
if model_path:
break
if model_path is None:
# Fallback: the orchestrator stores the served model id on the engine
# itself (set by AsyncOmniEngine.__init__). Stage-level engine_args
# may not surface ``model`` when the deploy yaml doesn't set it per
# stage (the CLI-passed model id is the single source of truth).
model_path = getattr(self.engine_client, "model", None)
model_path = resolve_stage_model_path(self.engine_client)
if model_path is None:
raise RuntimeError("higgs_audio_v2 serving could not resolve the model path from the engine stage configs")
processor = AutoProcessor.from_pretrained(model_path)
@@ -16,6 +16,7 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import (
PreparedRequest,
apply_max_new_tokens,
conditioning_cache_salt,
resolve_stage_model_path,
)
_REF_CODE_CACHE_MAX_ENTRIES = 256
@@ -170,13 +171,7 @@ class HiggsAudioV3Adapter(ARTTSAdapter):
HiggsAudioV3TokenizerAdapter,
)
model_path = None
for stage in self.engine_client.stage_configs:
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
if model_path:
break
if model_path is None:
model_path = getattr(self.engine_client, "model", None)
model_path = resolve_stage_model_path(self.engine_client)
if model_path is None:
raise RuntimeError("higgs_audio_v3 serving could not resolve model path")
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
@@ -58,6 +58,7 @@ from vllm_omni.entrypoints.openai.serving_video import (
ReferenceAudio,
ReferenceImage,
ReferenceVideo,
_stage_diffusion_model_class_name,
)
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE
@@ -136,10 +137,6 @@ def _config_get(config: Any, key: str, default: Any = None) -> Any:
return getattr(config, key, default)
def _stage_engine_args(stage_cfg: Any) -> Any:
return _config_get(stage_cfg, "engine_args", {}) or {}
def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
if not stage_configs:
return []
@@ -150,7 +147,7 @@ def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
for stage_cfg in stage_configs:
if get_stage_type(stage_cfg) != "diffusion":
continue
model_class_name = _config_get(_stage_engine_args(stage_cfg), "model_class_name")
model_class_name = _stage_diffusion_model_class_name(stage_cfg)
if not model_class_name:
continue
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
+8
View File
@@ -103,6 +103,14 @@ class ServingRealtimeRobotOpenPI:
for stage_config in getattr(engine_client, "stage_configs", []) or []:
if getattr(stage_config, "stage_type", None) != "diffusion":
continue
# Typed diffusion stages keep model-owned OpenPI handshake
# metadata in diffusion_config.model_config. The out-of-process
# head only has this stage view because full od_config lives in
# the worker.
diffusion_config = getattr(stage_config, "diffusion_config", None)
model_config = getattr(diffusion_config, "model_config", None)
if model_config is not None:
break
engine_args = getattr(stage_config, "engine_args", None)
model_config = getattr(engine_args, "model_config", None)
if model_config is not None:
+9 -7
View File
@@ -4,18 +4,20 @@
def _should_enable_profiler_endpoints(stage_configs: list | None) -> bool:
"""Check if any stage has profiler_config set in its engine_args."""
"""Check whether any typed or legacy stage enables a profiler."""
if not stage_configs:
return False
for stage in stage_configs:
engine_args = stage.get("engine_args") if isinstance(stage, dict) else getattr(stage, "engine_args", None)
if engine_args is None:
continue
profiler_config = (
engine_args.get("profiler_config")
if isinstance(engine_args, dict)
else getattr(engine_args, "profiler_config", None)
stage.get("profiler_config") if isinstance(stage, dict) else getattr(stage, "profiler_config", None)
)
if profiler_config is None:
engine_args = stage.get("engine_args") if isinstance(stage, dict) else getattr(stage, "engine_args", None)
profiler_config = (
engine_args.get("profiler_config")
if isinstance(engine_args, dict)
else getattr(engine_args, "profiler_config", None)
)
if profiler_config is not None:
profiler = (
profiler_config.get("profiler")
+6
View File
@@ -1,7 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import logging
import os
from dataclasses import asdict, is_dataclass
from multiprocessing import shared_memory as _shm
from typing import Any
@@ -274,6 +278,8 @@ def _to_dict(x: Any) -> dict[str, Any]:
try:
if isinstance(x, dict):
return dict(x)
if is_dataclass(x) and not isinstance(x, type):
return asdict(x)
return _omega_to_dict(x)
except Exception:
try:
+7
View File
@@ -18,6 +18,13 @@ logger = init_logger(__name__)
def inject_omni_kv_config(stage: Any, omni_conn_cfg: dict[str, Any], omni_from: str, omni_to: str) -> None:
"""Inject connector configuration into stage engine arguments."""
typed_connector_config = getattr(stage, "connector_config", None)
if typed_connector_config is not None:
omni_conf_dict = dict(typed_connector_config.omni_kv_config or {})
omni_conf_dict.update(connector_config=omni_conn_cfg, omni_from_stage=omni_from, omni_to_stage=omni_to)
typed_connector_config.omni_kv_config = omni_conf_dict
return
# Prepare omni_kv_config dict
omni_conf_dict = {}
try: