[Core] Retain structured configs through runtime startup (#6849)

Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com>
This commit is contained in:
maithilijoshi20 authored and GitHub committed 2026-09-16 20:47:38 -07:00
1 parent becfefd440
commit f1a6e7ce2a
49 files changed
+1856 -331

No files matched your search

+33 -13
View File
@@ -36,6 +36,7 @@ from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils import random_uuid
from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
from vllm_omni.config.stage_config import StagePipelineConfig
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin, StreamingAudioResampler
from vllm_omni.entrypoints.openai.protocol.audio import (
AudioResponse,
@@ -76,6 +77,24 @@ _SPEECH_USAGE_TOTAL_TOKENS_HEADER = "X-VLLM-OMNI-TOTAL-TOKENS"
_SPEECH_USAGE_INPUT_TEXT_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-TEXT-TOKENS"
_SPEECH_USAGE_INPUT_AUDIO_TOKENS_HEADER = "X-VLLM-OMNI-INPUT-AUDIO-TOKENS"
def _stage_speech_metadata(stage: Any) -> tuple[str | None, str | None, str | None]:
"""Return speech routing metadata from typed or legacy stage configs."""
topology = getattr(stage, "stage_pipeline_config", None)
if isinstance(topology, StagePipelineConfig):
return (
getattr(topology, "model_stage", None),
getattr(getattr(stage, "model_config", None), "model_arch", None),
getattr(stage, "worker_type", None),
)
engine_args = getattr(stage, "engine_args", None)
return (
getattr(engine_args, "model_stage", None),
getattr(engine_args, "model_arch", None),
getattr(engine_args, "worker_type", None),
)
# TTS Configuration
#
# The stage-key -> model-type mapping is NOT declared here: it is derived from
@@ -472,12 +491,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""Find and return the TTS stage config, or None if not found."""
tts_stage_keys = all_tts_stage_keys()
entry_stage_archs = tts_entry_stage_archs()
all_stages = frozenset(stage.engine_args.model_stage for stage in self.engine_client.stage_configs)
all_stages = frozenset(
model_stage
for stage in self.engine_client.stage_configs
if (model_stage := _stage_speech_metadata(stage)[0]) is not None
)
for stage in self.engine_client.stage_configs:
engine_args = stage.engine_args
model_stage = engine_args.model_stage
model_arch = getattr(engine_args, "model_arch", None)
worker_type = getattr(engine_args, "worker_type", None)
model_stage, model_arch, worker_type = _stage_speech_metadata(stage)
if model_stage in tts_stage_keys:
# Owning the stage key is not always enough: a model may be
# speech-capable only in some deployment topologies. Ask the
@@ -501,10 +521,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""
if self._tts_stage is None:
return None
return detect_tts_model_type(
getattr(self._tts_stage.engine_args, "model_stage", None),
getattr(self._tts_stage.engine_args, "model_arch", None),
)
return detect_tts_model_type(*_stage_speech_metadata(self._tts_stage)[:2])
def _compute_max_instructions_length(self) -> int:
"""Compute max instructions length with precedence: CLI > stage config > default.
@@ -518,7 +535,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# 2. Try to get from TTS stage config
if self._tts_stage is not None:
tts_args = getattr(self._tts_stage, "tts_args", {})
topology = getattr(self._tts_stage, "stage_pipeline_config", None)
tts_args = (
getattr(topology, "extras", {}).get("tts_args", {})
if isinstance(topology, StagePipelineConfig)
else getattr(self._tts_stage, "tts_args", {})
)
if "max_instructions_length" in tts_args:
return tts_args["max_instructions_length"]
@@ -1896,9 +1918,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
# chat-template markers (im_start_token_id 151644). A raw-text
# prompt produces a 1-token placeholder that crashes the talker's
# prefill/decode handoff. Reject early with an actionable message.
stage_names = {
getattr(getattr(s, "engine_args", None), "model_stage", None) for s in self.engine_client.stage_configs
}
stage_names = {_stage_speech_metadata(s)[0] for s in self.engine_client.stage_configs}
if "talker" in stage_names:
raise ValueError(
"The /v1/audio/speech endpoint is only supported for "