[Core] Retain structured configs through runtime startup (#6849)
Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
This commit is contained in:
1 parent
becfefd440
commit
f1a6e7ce2a
49 files changed
+1856
-331
No files matched your search
@@ -1048,6 +1048,7 @@ def run_headless(args: TrackingNamespace) -> None:
|
||||
)
|
||||
from vllm_omni.engine.stage_init_utils import (
|
||||
build_engine_args_dict,
|
||||
build_engine_args_dict_from_omni_stage_config,
|
||||
build_vllm_config,
|
||||
get_stage_connector_spec,
|
||||
inject_omni_kv_connector_config,
|
||||
@@ -1141,21 +1142,21 @@ def run_headless(args: TrackingNamespace) -> None:
|
||||
stage_connector_spec = get_stage_connector_spec(
|
||||
omni_transfer_config=omni_transfer_config,
|
||||
stage_id=stage_id,
|
||||
async_chunk=bool(stage_cfg.engine_args.get("async_chunk", False)),
|
||||
async_chunk=bool(
|
||||
getattr(getattr(stage_cfg, "connector_config", None), "async_chunk", None)
|
||||
if hasattr(stage_cfg, "connector_config")
|
||||
else stage_cfg.engine_args.get("async_chunk", False)
|
||||
),
|
||||
)
|
||||
|
||||
# ``runtime_cfg`` is mostly inherited from the parent's
|
||||
# CUDA_VISIBLE_DEVICES; when ``--omni-dp-size-local > 1`` we additionally
|
||||
# bracket each replica's spawn below with setup_stage_devices so they
|
||||
# don't all stack on cuda:0 (see ``per_replica_devices`` above).
|
||||
# Headless startup still supplies the legacy OmegaConf stage shape through
|
||||
# the stable adapter entry point. The implementation switches only when
|
||||
# RFC #4021 threads structured stage configs through the launch plan.
|
||||
engine_args_dict = build_engine_args_dict(
|
||||
stage_cfg,
|
||||
model,
|
||||
stage_connector_spec=stage_connector_spec,
|
||||
cli_tokenizer=getattr(args, "tokenizer", None),
|
||||
engine_args_dict = (
|
||||
build_engine_args_dict_from_omni_stage_config(
|
||||
stage_cfg, model, stage_connector_spec=stage_connector_spec, cli_tokenizer=getattr(args, "tokenizer", None)
|
||||
)
|
||||
if hasattr(stage_cfg, "connector_config")
|
||||
else build_engine_args_dict(
|
||||
stage_cfg, model, stage_connector_spec=stage_connector_spec, cli_tokenizer=getattr(args, "tokenizer", None)
|
||||
)
|
||||
)
|
||||
|
||||
inject_omni_kv_connector_config(engine_args_dict, omni_kv_connector, stage_id)
|
||||
|
||||
@@ -248,10 +248,27 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
return obj.get(key, default)
|
||||
return getattr(obj, key, default)
|
||||
|
||||
@classmethod
|
||||
def _stage_model_metadata(cls, stage: Any) -> tuple[str | None, str | None]:
|
||||
"""Return ``(model_arch, model_stage)`` for typed or legacy stages."""
|
||||
model_config = cls._stage_get(stage, "model_config")
|
||||
model_arch = cls._stage_get(model_config, "model_arch")
|
||||
model_stage = cls._stage_get(stage, "model_stage")
|
||||
|
||||
topology = cls._stage_get(stage, "stage_pipeline_config")
|
||||
if model_stage is None:
|
||||
model_stage = cls._stage_get(topology, "model_stage")
|
||||
|
||||
engine_args = cls._stage_get(stage, "engine_args")
|
||||
if model_arch is None:
|
||||
model_arch = cls._stage_get(engine_args, "model_arch")
|
||||
if model_stage is None:
|
||||
model_stage = cls._stage_get(engine_args, "model_stage")
|
||||
return model_arch, model_stage
|
||||
|
||||
def _has_minicpmo45_stage(self) -> bool:
|
||||
for stage in getattr(self.engine_client, "stage_configs", []) or []:
|
||||
engine_args = self._stage_get(stage, "engine_args")
|
||||
model_arch = self._stage_get(engine_args, "model_arch")
|
||||
model_arch, _ = self._stage_model_metadata(stage)
|
||||
if model_arch == "MiniCPMO45OmniForConditionalGeneration":
|
||||
return True
|
||||
return False
|
||||
@@ -324,11 +341,10 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin):
|
||||
if not hasattr(sp, "output_kind"):
|
||||
continue
|
||||
|
||||
engine_args = self._stage_get(stage, "engine_args")
|
||||
if self._stage_get(engine_args, "model_arch") != "MiniCPMO45OmniForConditionalGeneration":
|
||||
model_arch, model_stage = self._stage_model_metadata(stage)
|
||||
if model_arch != "MiniCPMO45OmniForConditionalGeneration":
|
||||
continue
|
||||
|
||||
model_stage = self._stage_get(engine_args, "model_stage")
|
||||
if model_stage == "llm":
|
||||
sp.output_kind = RequestOutputKind.FINAL_ONLY
|
||||
elif model_stage == "tts":
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
|
||||
from vllm_omni.config.stage_config import StagePipelineConfig
|
||||
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import (
|
||||
AudioResponse,
|
||||
@@ -76,6 +77,24 @@ _SPEECH_USAGE_TOTAL_TOKENS_HEADER = "X-VLLM-OMNI-TOTAL-TOKENS"
|
||||
_SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TEXT-TOKENS"
|
||||
_SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS"
|
||||
|
||||
|
||||
def _stage_speech_metadata(stage: Any) -> tuple[str | None, str | None, str | None]:
|
||||
"""Return speech routing metadata from typed or legacy stage configs."""
|
||||
topology = getattr(stage, "stage_pipeline_config", None)
|
||||
if isinstance(topology, StagePipelineConfig):
|
||||
return (
|
||||
getattr(topology, "model_stage", None),
|
||||
getattr(getattr(stage, "model_config", None), "model_arch", None),
|
||||
getattr(stage, "worker_type", None),
|
||||
)
|
||||
engine_args = getattr(stage, "engine_args", None)
|
||||
return (
|
||||
getattr(engine_args, "model_stage", None),
|
||||
getattr(engine_args, "model_arch", None),
|
||||
getattr(engine_args, "worker_type", None),
|
||||
)
|
||||
|
||||
|
||||
# TTS Configuration
|
||||
#
|
||||
# The stage-key -> model-type mapping is NOT declared here: it is derived from
|
||||
@@ -472,12 +491,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
"""Find and return the TTS stage config, or None if not found."""
|
||||
tts_stage_keys = all_tts_stage_keys()
|
||||
entry_stage_archs = tts_entry_stage_archs()
|
||||
all_stages = frozenset(stage.engine_args.model_stage for stage in self.engine_client.stage_configs)
|
||||
all_stages = frozenset(
|
||||
model_stage
|
||||
for stage in self.engine_client.stage_configs
|
||||
if (model_stage := _stage_speech_metadata(stage)[0]) is not None
|
||||
)
|
||||
for stage in self.engine_client.stage_configs:
|
||||
engine_args = stage.engine_args
|
||||
model_stage = engine_args.model_stage
|
||||
model_arch = getattr(engine_args, "model_arch", None)
|
||||
worker_type = getattr(engine_args, "worker_type", None)
|
||||
model_stage, model_arch, worker_type = _stage_speech_metadata(stage)
|
||||
if model_stage in tts_stage_keys:
|
||||
# Owning the stage key is not always enough: a model may be
|
||||
# speech-capable only in some deployment topologies. Ask the
|
||||
@@ -501,10 +521,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
"""
|
||||
if self._tts_stage is None:
|
||||
return None
|
||||
return detect_tts_model_type(
|
||||
getattr(self._tts_stage.engine_args, "model_stage", None),
|
||||
getattr(self._tts_stage.engine_args, "model_arch", None),
|
||||
)
|
||||
return detect_tts_model_type(*_stage_speech_metadata(self._tts_stage)[:2])
|
||||
|
||||
def _compute_max_instructions_length(self) -> int:
|
||||
"""Compute max instructions length with precedence: CLI > stage config > default.
|
||||
@@ -518,7 +535,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
# 2. Try to get from TTS stage config
|
||||
if self._tts_stage is not None:
|
||||
tts_args = getattr(self._tts_stage, "tts_args", {})
|
||||
topology = getattr(self._tts_stage, "stage_pipeline_config", None)
|
||||
tts_args = (
|
||||
getattr(topology, "extras", {}).get("tts_args", {})
|
||||
if isinstance(topology, StagePipelineConfig)
|
||||
else getattr(self._tts_stage, "tts_args", {})
|
||||
)
|
||||
if "max_instructions_length" in tts_args:
|
||||
return tts_args["max_instructions_length"]
|
||||
|
||||
@@ -1896,9 +1918,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
# chat-template markers (im_start_token_id 151644). A raw-text
|
||||
# prompt produces a 1-token placeholder that crashes the talker's
|
||||
# prefill/decode handoff. Reject early with an actionable message.
|
||||
stage_names = {
|
||||
getattr(getattr(s, "engine_args", None), "model_stage", None) for s in self.engine_client.stage_configs
|
||||
}
|
||||
stage_names = {_stage_speech_metadata(s)[0] for s in self.engine_client.stage_configs}
|
||||
if "talker" in stage_names:
|
||||
raise ValueError(
|
||||
"The /v1/audio/speech endpoint is only supported for "
|
||||
|
||||
@@ -53,6 +53,30 @@ logger = init_logger(__name__)
|
||||
|
||||
_VIDEO_RESPONSE_FRAME_CONVERSION_WORKERS = 8
|
||||
|
||||
|
||||
def _config_value(config: Any, key: str, default: Any = None) -> Any:
|
||||
if isinstance(config, Mapping):
|
||||
return config.get(key, default)
|
||||
return getattr(config, key, default)
|
||||
|
||||
|
||||
def _stage_diffusion_model_class_name(stage_config: Any) -> str | None:
|
||||
"""Resolve a stage's diffusion class across typed and legacy shapes."""
|
||||
diffusion_config = _config_value(stage_config, "diffusion_config")
|
||||
model_class_name = _config_value(diffusion_config, "model_class_name")
|
||||
if model_class_name:
|
||||
return str(model_class_name)
|
||||
|
||||
model_config = _config_value(stage_config, "model_config")
|
||||
model_arch = _config_value(model_config, "model_arch") or _config_value(stage_config, "model_arch")
|
||||
if model_arch:
|
||||
return str(model_arch)
|
||||
|
||||
engine_args = _config_value(stage_config, "engine_args", {})
|
||||
model_class_name = _config_value(engine_args, "model_class_name")
|
||||
return str(model_class_name) if model_class_name else None
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
|
||||
@@ -195,18 +219,7 @@ class OmniOpenAIServingVideo:
|
||||
model_class_name = getattr(od_config, "model_class_name", None)
|
||||
model_archs = [model_class_name]
|
||||
for stage_config in self.stage_configs or ():
|
||||
stage_get = (
|
||||
stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None)
|
||||
)
|
||||
engine_args = stage_get("engine_args") or {}
|
||||
model_archs.extend(
|
||||
(
|
||||
stage_get("model_arch"),
|
||||
engine_args.get("model_class_name")
|
||||
if isinstance(engine_args, Mapping)
|
||||
else getattr(engine_args, "model_class_name", None),
|
||||
)
|
||||
)
|
||||
model_archs.append(_stage_diffusion_model_class_name(stage_config))
|
||||
metadata_capability = any(
|
||||
get_diffusion_model_metadata(model_arch).supports_mixed_reference_inputs for model_arch in model_archs
|
||||
)
|
||||
@@ -223,18 +236,7 @@ class OmniOpenAIServingVideo:
|
||||
od_config = self._resolve_diffusion_od_config()
|
||||
model_archs = [None if od_config is None else getattr(od_config, "model_class_name", None)]
|
||||
for stage_config in self.stage_configs or ():
|
||||
stage_get = (
|
||||
stage_config.get if isinstance(stage_config, Mapping) else lambda key: getattr(stage_config, key, None)
|
||||
)
|
||||
engine_args = stage_get("engine_args") or {}
|
||||
model_archs.extend(
|
||||
(
|
||||
stage_get("model_arch"),
|
||||
engine_args.get("model_class_name")
|
||||
if isinstance(engine_args, Mapping)
|
||||
else getattr(engine_args, "model_class_name", None),
|
||||
)
|
||||
)
|
||||
model_archs.append(_stage_diffusion_model_class_name(stage_config))
|
||||
|
||||
supported: set[str] = set()
|
||||
for model_arch in model_archs:
|
||||
|
||||
@@ -42,7 +42,11 @@ class AudexAdapter(ARTTSAdapter):
|
||||
|
||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||
|
||||
stage = getattr(getattr(self.ctx.server._tts_stage, "engine_args", None), "model_stage", None)
|
||||
stage_config = self.ctx.server._tts_stage
|
||||
topology = getattr(stage_config, "stage_pipeline_config", None)
|
||||
stage = getattr(topology, "model_stage", None)
|
||||
if stage is None:
|
||||
stage = getattr(getattr(stage_config, "engine_args", None), "model_stage", None)
|
||||
profile, folder = (
|
||||
("full", "checkpoint_folder_full") if stage == "audex_omni" else ("tts", "checkpoint_folder_audiogen")
|
||||
)
|
||||
|
||||
@@ -39,6 +39,21 @@ DEFAULT_TTS_LANGUAGES = frozenset(
|
||||
)
|
||||
|
||||
|
||||
def resolve_stage_model_path(engine_client: Any) -> str | None:
|
||||
"""Prefer typed stage model overrides, then legacy and served-model values."""
|
||||
stages = getattr(engine_client, "stage_configs", ()) or ()
|
||||
for stage in stages:
|
||||
model_path = getattr(getattr(stage, "model_config", None), "model", None)
|
||||
if model_path:
|
||||
return str(model_path)
|
||||
for stage in stages:
|
||||
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
|
||||
if model_path:
|
||||
return str(model_path)
|
||||
model_path = getattr(engine_client, "model", None)
|
||||
return str(model_path) if model_path else None
|
||||
|
||||
|
||||
def conditioning_cache_salt(request: "OpenAICreateSpeechRequest", tts_params: dict | None = None) -> str:
|
||||
"""Stable hash of the real Stage 0 conditioning for the prefix cache.
|
||||
|
||||
|
||||
@@ -9,7 +9,12 @@ import numpy as np
|
||||
from vllm.inputs import tokens_input
|
||||
|
||||
from vllm_omni.entrypoints.openai.tts_adapters import register_tts_adapter
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import ARTTSAdapter, PreparedRequest, apply_max_new_tokens
|
||||
from vllm_omni.entrypoints.openai.tts_adapters.base import (
|
||||
ARTTSAdapter,
|
||||
PreparedRequest,
|
||||
apply_max_new_tokens,
|
||||
resolve_stage_model_path,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
@@ -81,17 +86,7 @@ class HiggsAudioV2Adapter(ARTTSAdapter):
|
||||
|
||||
from transformers import AutoProcessor
|
||||
|
||||
model_path = None
|
||||
for stage in self.engine_client.stage_configs:
|
||||
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
|
||||
if model_path:
|
||||
break
|
||||
if model_path is None:
|
||||
# Fallback: the orchestrator stores the served model id on the engine
|
||||
# itself (set by AsyncOmniEngine.__init__). Stage-level engine_args
|
||||
# may not surface ``model`` when the deploy yaml doesn't set it per
|
||||
# stage (the CLI-passed model id is the single source of truth).
|
||||
model_path = getattr(self.engine_client, "model", None)
|
||||
model_path = resolve_stage_model_path(self.engine_client)
|
||||
if model_path is None:
|
||||
raise RuntimeError("higgs_audio_v2 serving could not resolve the model path from the engine stage configs")
|
||||
processor = AutoProcessor.from_pretrained(model_path)
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm_omni.entrypoints.openai.tts_adapters.base import (
|
||||
PreparedRequest,
|
||||
apply_max_new_tokens,
|
||||
conditioning_cache_salt,
|
||||
resolve_stage_model_path,
|
||||
)
|
||||
|
||||
_REF_CODE_CACHE_MAX_ENTRIES = 256
|
||||
@@ -170,13 +171,7 @@ class HiggsAudioV3Adapter(ARTTSAdapter):
|
||||
HiggsAudioV3TokenizerAdapter,
|
||||
)
|
||||
|
||||
model_path = None
|
||||
for stage in self.engine_client.stage_configs:
|
||||
model_path = getattr(getattr(stage, "engine_args", None), "model", None)
|
||||
if model_path:
|
||||
break
|
||||
if model_path is None:
|
||||
model_path = getattr(self.engine_client, "model", None)
|
||||
model_path = resolve_stage_model_path(self.engine_client)
|
||||
if model_path is None:
|
||||
raise RuntimeError("higgs_audio_v3 serving could not resolve model path")
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm_omni.entrypoints.openai.serving_video import (
|
||||
ReferenceAudio,
|
||||
ReferenceImage,
|
||||
ReferenceVideo,
|
||||
_stage_diffusion_model_class_name,
|
||||
)
|
||||
from vllm_omni.entrypoints.openai.storage import STORAGE_MANAGER
|
||||
from vllm_omni.entrypoints.openai.stores import VIDEO_STORE
|
||||
@@ -136,10 +137,6 @@ def _config_get(config: Any, key: str, default: Any = None) -> Any:
|
||||
return getattr(config, key, default)
|
||||
|
||||
|
||||
def _stage_engine_args(stage_cfg: Any) -> Any:
|
||||
return _config_get(stage_cfg, "engine_args", {}) or {}
|
||||
|
||||
|
||||
def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
|
||||
if not stage_configs:
|
||||
return []
|
||||
@@ -150,7 +147,7 @@ def _diffusion_model_classes(stage_configs: list[Any] | None) -> list[type]:
|
||||
for stage_cfg in stage_configs:
|
||||
if get_stage_type(stage_cfg) != "diffusion":
|
||||
continue
|
||||
model_class_name = _config_get(_stage_engine_args(stage_cfg), "model_class_name")
|
||||
model_class_name = _stage_diffusion_model_class_name(stage_cfg)
|
||||
if not model_class_name:
|
||||
continue
|
||||
model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name)
|
||||
|
||||
@@ -103,6 +103,14 @@ class ServingRealtimeRobotOpenPI:
|
||||
for stage_config in getattr(engine_client, "stage_configs", []) or []:
|
||||
if getattr(stage_config, "stage_type", None) != "diffusion":
|
||||
continue
|
||||
# Typed diffusion stages keep model-owned OpenPI handshake
|
||||
# metadata in diffusion_config.model_config. The out-of-process
|
||||
# head only has this stage view because full od_config lives in
|
||||
# the worker.
|
||||
diffusion_config = getattr(stage_config, "diffusion_config", None)
|
||||
model_config = getattr(diffusion_config, "model_config", None)
|
||||
if model_config is not None:
|
||||
break
|
||||
engine_args = getattr(stage_config, "engine_args", None)
|
||||
model_config = getattr(engine_args, "model_config", None)
|
||||
if model_config is not None:
|
||||
|
||||
@@ -4,18 +4,20 @@
|
||||
|
||||
|
||||
def _should_enable_profiler_endpoints(stage_configs: list | None) -> bool:
|
||||
"""Check if any stage has profiler_config set in its engine_args."""
|
||||
"""Check whether any typed or legacy stage enables a profiler."""
|
||||
if not stage_configs:
|
||||
return False
|
||||
for stage in stage_configs:
|
||||
engine_args = stage.get("engine_args") if isinstance(stage, dict) else getattr(stage, "engine_args", None)
|
||||
if engine_args is None:
|
||||
continue
|
||||
profiler_config = (
|
||||
engine_args.get("profiler_config")
|
||||
if isinstance(engine_args, dict)
|
||||
else getattr(engine_args, "profiler_config", None)
|
||||
stage.get("profiler_config") if isinstance(stage, dict) else getattr(stage, "profiler_config", None)
|
||||
)
|
||||
if profiler_config is None:
|
||||
engine_args = stage.get("engine_args") if isinstance(stage, dict) else getattr(stage, "engine_args", None)
|
||||
profiler_config = (
|
||||
engine_args.get("profiler_config")
|
||||
if isinstance(engine_args, dict)
|
||||
else getattr(engine_args, "profiler_config", None)
|
||||
)
|
||||
if profiler_config is not None:
|
||||
profiler = (
|
||||
profiler_config.get("profiler")
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from multiprocessing import shared_memory as _shm
|
||||
from typing import Any
|
||||
|
||||
@@ -274,6 +278,8 @@ def _to_dict(x: Any) -> dict[str, Any]:
|
||||
try:
|
||||
if isinstance(x, dict):
|
||||
return dict(x)
|
||||
if is_dataclass(x) and not isinstance(x, type):
|
||||
return asdict(x)
|
||||
return _omega_to_dict(x)
|
||||
except Exception:
|
||||
try:
|
||||
|
||||
@@ -18,6 +18,13 @@ logger = init_logger(__name__)
|
||||
|
||||
def inject_omni_kv_config(stage: Any, omni_conn_cfg: dict[str, Any], omni_from: str, omni_to: str) -> None:
|
||||
"""Inject connector configuration into stage engine arguments."""
|
||||
typed_connector_config = getattr(stage, "connector_config", None)
|
||||
if typed_connector_config is not None:
|
||||
omni_conf_dict = dict(typed_connector_config.omni_kv_config or {})
|
||||
omni_conf_dict.update(connector_config=omni_conn_cfg, omni_from_stage=omni_from, omni_to_stage=omni_to)
|
||||
typed_connector_config.omni_kv_config = omni_conf_dict
|
||||
return
|
||||
|
||||
# Prepare omni_kv_config dict
|
||||
omni_conf_dict = {}
|
||||
try:
|
||||
|
||||
Reference in new issue
Block a user