[Metrics] Add audio SLOs + cross-stage transfer families + per-(stage, replica) wrap for upstream vllm:* (#3576)

Signed-off-by: vraiti <vraiti@redhat.com>
Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com>
Co-authored-by: vraiti <vraiti@redhat.com>
This commit is contained in:
LHXuuuandvraiti authored and GitHub committed 2026-06-01 20:20:48 +08:00
1 parent ee33601571
commit 41a795b5d6
29 files changed
+3377 -25

No files matched your search

+186 -3
View File
@@ -29,6 +29,7 @@ from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingParams
from vllm.v1.engine import EngineCoreOutputs
from vllm.v1.engine.exceptions import EngineDeadError
from vllm.v1.metrics.stats import IterationStats
from vllm_omni.distributed.omni_coordinator import (
LoadBalancer,
@@ -54,6 +55,8 @@ from vllm_omni.engine.messages import (
)
from vllm_omni.engine.serialization import serialize_additional_information
from vllm_omni.engine.stage_pool import StagePool
from vllm_omni.metrics.prometheus import OmniRequestCounter
from vllm_omni.metrics.stat_logger import OmniPrometheusStatLogger
from vllm_omni.outputs import OmniRequestOutput
# Factory signature for building a head-side stage client for a
@@ -155,6 +158,12 @@ class Orchestrator:
_WATCH_REPLICA_INTERVAL_S: float = 0.5
_WATCH_REPLICA_IDLE_INTERVAL_S: float = 1.0
# Class-level defaults so tests that bypass __init__ via object.__new__
# don't AttributeError when transfer / counter emit paths access them.
_running_counter: OmniRequestCounter | None = None
_transfer_emitter: Any = None
_stat_logger: OmniPrometheusStatLogger | None = None
def __init__(
self,
request_async_queue: janus.AsyncQueue[EngineQueueMessage],
@@ -164,9 +173,12 @@ class Orchestrator:
*,
async_chunk: bool = False,
pd_config: dict[str, Any] | None = None,
running_counter: OmniRequestCounter | None = None,
coordinator_pub_address: str | None = None,
load_balancer_factory: Callable[[], LoadBalancer] | None = None,
remote_replica_factory: RemoteReplicaFactory | None = None,
transfer_emitter: Any = None,
log_stats: bool = False,
) -> None:
self.request_async_queue = request_async_queue
self.output_async_queue = output_async_queue
@@ -186,6 +198,8 @@ class Orchestrator:
self._pd_bootstrap_addr = pd_config.get("bootstrap_addr")
self._pd_prefill_engine_id = pd_config.get("prefill_engine_id")
self.request_states: dict[str, OrchestratorRequestState] = {}
self._init_metrics_state(stage_pools, running_counter, transfer_emitter, log_stats=log_stats)
self._cfg_tracker = CfgCompanionTracker()
self._shutdown_event = asyncio.Event()
@@ -215,6 +229,71 @@ class Orchestrator:
pool.attach_hub(self._hub)
pool.attach_load_balancer(factory())
def _init_metrics_state(
self,
stage_pools: list[StagePool],
running_counter: OmniRequestCounter | None,
transfer_emitter: Any,
log_stats: bool = False,
) -> None:
"""Wire up all metric-related orchestrator state.
Sets ``self._running_counter`` and ``self._transfer_emitter``
(both optional, used by request-add / forward paths), builds the
``(stage_id, replica_id) ↔ engine_idx`` lookup used at record() time,
and best-effort constructs the ``OmniPrometheusStatLogger`` wrap
that exposes ~37 upstream ``vllm:*`` families with per-(stage,
replica) labels. Failure to build the wrap is logged and metrics
are simply disabled — orchestrator construction continues so unit
tests with a minimal ``vllm_config`` still pass.
``log_stats=False`` short-circuits the wrap entirely so the
~65 upstream ``vllm:*`` families are not registered in the
Prometheus default registry at all. The per-step record() path
already no-ops on ``scheduler_stats is None`` (which is what
the upstream scheduler returns when its own log_stats is False),
so this gate is mainly to keep the ``/metrics`` surface clean
when the user did not request stats.
"""
self._running_counter = running_counter
self._transfer_emitter = transfer_emitter
# Flat engine_idx ↔ (stage, replica) maps. The reverse map is
# consulted at record() time to translate the orchestrator's
# (stage_id, replica_id) loop variables into an engine_idx the
# underlying PrometheusStatLogger can address.
stage_replica_map: dict[int, tuple[str, str]] = {}
self._stage_replica_to_engine_idx: dict[tuple[int, int], int] = {}
flat_idx = 0
for stage_id, pool in enumerate(stage_pools):
for replica_id in range(pool.num_replicas):
stage_replica_map[flat_idx] = (str(stage_id), str(replica_id))
self._stage_replica_to_engine_idx[(stage_id, replica_id)] = flat_idx
flat_idx += 1
if not log_stats:
self._stat_logger = None
return
vllm_config_for_stats = next(
(p.stage_vllm_config for p in stage_pools if p.stage_vllm_config is not None),
None,
)
if vllm_config_for_stats is None:
self._stat_logger = None
return
try:
self._stat_logger = OmniPrometheusStatLogger(
vllm_config=vllm_config_for_stats,
stage_replica_map=stage_replica_map,
)
except Exception:
# Minimal vllm_config in unit-test contexts can lack fields the
# upstream PrometheusStatLogger expects. Skip wrap rather than
# break orchestrator construction.
logger.exception("[Orchestrator] OmniPrometheusStatLogger init failed; metrics wrap disabled")
self._stat_logger = None
async def run(self) -> None:
"""Main entry point for the Orchestrator event loop."""
logger.info("[Orchestrator] Starting event loop")
@@ -393,6 +472,8 @@ class Orchestrator:
mm_features=getattr(prompt, "mm_features", None),
)
self.request_states[request_id] = req_state
if self._running_counter is not None:
self._running_counter.increment()
req_state.streaming.enabled = bool(getattr(prompt, "resumable", False))
req_state.stage_submit_ts[stage_id] = _time.time()
enqueue_ts = msg.enqueue_ts
@@ -603,7 +684,24 @@ class Orchestrator:
"new_prompt_len_snapshot",
None,
)
raw_output = await pool.process_llm_raw_outputs(replica_id, raw_outputs)
# OmniSchedulerMixin.make_stats() already throttles
# per-scheduler at 1 Hz, so raw_outputs.scheduler_stats
# being non-None means this replica passed its own gate.
# A second global throttle here would drop stats for
# other (stage, replica) pairs in the same 1s window.
record_stats = self._stat_logger is not None and raw_outputs.scheduler_stats is not None
iteration_stats = IterationStats() if record_stats else None
raw_output = await pool.process_llm_raw_outputs(
replica_id,
raw_outputs,
iteration_stats=iteration_stats,
)
if record_stats:
self._stat_logger.record(
raw_outputs.scheduler_stats,
iteration_stats,
engine_idx=self._stage_replica_to_engine_idx[(stage_id, replica_id)],
)
except asyncio.CancelledError:
raise
except EngineDeadError as e:
@@ -681,7 +779,7 @@ class Orchestrator:
)
stage_metrics.pipeline_timings = dict(req_state.pipeline_timings)
await self._route_output(stage_id, output, req_state, stage_metrics)
await self._route_output(stage_id, replica_id, output, req_state, stage_metrics)
async def _handle_stage_error(self, stage_id: int, output: Any) -> None:
"""Emit a frontend-visible error and clean up request state."""
@@ -713,7 +811,8 @@ class Orchestrator:
self._release_request_bindings(request_ids)
for request_id in request_ids:
self._pd_kv_params.pop(request_id, None)
self.request_states.pop(request_id, None)
if self.request_states.pop(request_id, None) is not None and self._running_counter is not None:
self._running_counter.decrement()
def _maybe_clone_diffusion_params_for_cfg(self, request_id: str, params: Any) -> Any:
"""Attach CFG companion ids to diffusion sampling params when needed."""
@@ -735,6 +834,7 @@ class Orchestrator:
async def _route_output(
self,
stage_id: int,
replica_id: int,
output: Any,
req_state: OrchestratorRequestState,
stage_metrics: Any,
@@ -757,6 +857,7 @@ class Orchestrator:
OutputMessage(
request_id=req_id,
stage_id=stage_id,
replica_id=replica_id,
engine_outputs=output,
metrics=stage_metrics,
finished=finished and stage_id == req_state.final_stage_id,
@@ -768,6 +869,7 @@ class Orchestrator:
StageMetricsMessage(
request_id=req_id,
stage_id=stage_id,
replica_id=replica_id,
metrics=stage_metrics,
stage_submit_ts=submit_ts,
)
@@ -797,6 +899,7 @@ class Orchestrator:
stage_id,
output,
req_state,
src_replica_id=replica_id,
is_streaming_session=req_state.streaming.enabled,
is_final_update=False,
)
@@ -807,6 +910,7 @@ class Orchestrator:
stage_id,
output,
req_state,
src_replica_id=replica_id,
is_streaming_session=True,
is_final_update=True,
)
@@ -918,6 +1022,41 @@ class Orchestrator:
)
return sp
def _emit_tx_edge(
self,
*,
from_stage: int,
from_replica: int,
to_stage: int,
to_pool: StagePool,
request_id: str,
tx_ms: float,
) -> None:
"""Emit per-edge transfer_tx_s + transfer_size_bytes histograms.
``tx_ms`` is the orchestrator-side wall-clock spent in ``next_pool.
submit_*`` (serialize + queue submit to the receiving worker). Best-
effort size_bytes left at 0 — orchestrator doesn't have a cheap handle
on the serialized payload size; a follow-up can plumb that from the
connector adapter.
"""
if self._transfer_emitter is None:
return
to_replica = to_pool.get_bound_replica_id(request_id)
if to_replica is None:
return
try:
self._transfer_emitter.observe_size(from_stage, from_replica, to_stage, to_replica, 0)
self._transfer_emitter.observe_tx_time(from_stage, from_replica, to_stage, to_replica, tx_ms / 1000.0)
except Exception:
logger.debug(
"[Orchestrator] transfer_tx emit failed for edge %d->%d req=%s",
from_stage,
to_stage,
request_id,
exc_info=True,
)
async def _forward_to_next_stage(
self,
req_id: str,
@@ -925,6 +1064,7 @@ class Orchestrator:
output: Any,
req_state: OrchestratorRequestState,
*,
src_replica_id: int | None = None,
is_streaming_session: bool = False,
is_final_update: bool = False,
) -> None:
@@ -937,6 +1077,7 @@ class Orchestrator:
next_stage_resumable = is_streaming_session and not is_final_update
already_submitted = self._next_stage_already_submitted(src_stage_id, req_state)
requires_multimodal_data = getattr(next_client, "requires_multimodal_data", False)
_t_submit_start = _time.perf_counter()
if next_pool.stage_type == "diffusion":
companion_outputs = self._cfg_tracker.pop_companion_outputs(req_id)
@@ -1024,6 +1165,15 @@ class Orchestrator:
params_override=self._maybe_clone_diffusion_params_for_cfg(req_id, params),
)
req_state.stage_submit_ts[next_logical] = _time.time()
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
self._emit_tx_edge(
from_stage=src_stage_id,
from_replica=src_replica_id if src_replica_id is not None else 0,
to_stage=next_logical,
to_pool=next_pool,
request_id=req_id,
tx_ms=_tx_ms,
)
return
# PD disaggregation: prefill → decode routing uses original prompt + KV transfer params
@@ -1063,6 +1213,15 @@ class Orchestrator:
await next_pool.submit_initial(req_id, req_state, request, prompt_text=None)
req_state.stage_submit_ts[next_logical] = _time.time()
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
self._emit_tx_edge(
from_stage=src_stage_id,
from_replica=src_replica_id if src_replica_id is not None else 0,
to_stage=next_logical,
to_pool=next_pool,
request_id=req_id,
tx_ms=_tx_ms,
)
return
if req_state.pd_prefill_multimodal_output is not None:
@@ -1107,6 +1266,15 @@ class Orchestrator:
await next_pool.submit_initial(req_id, req_state, request, prompt_text=None)
req_state.stage_submit_ts[next_logical] = _time.time()
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
self._emit_tx_edge(
from_stage=src_stage_id,
from_replica=src_replica_id if src_replica_id is not None else 0,
to_stage=next_logical,
to_pool=next_pool,
request_id=req_id,
tx_ms=_tx_ms,
)
async def _prewarm_async_chunk_stages(
self,
@@ -1131,6 +1299,7 @@ class Orchestrator:
params = req_state.sampling_params_list[next_stage_id]
req_state.stage_submit_ts[next_stage_id] = _time.time()
_t_submit_start = _time.perf_counter()
if next_pool.stage_type == "diffusion":
await next_pool.submit_initial(
@@ -1174,6 +1343,20 @@ class Orchestrator:
request.external_req_id = request.request_id
await next_pool.submit_initial(request_id, req_state, request, prompt_text=None)
# async_chunk pre-submit fires per stage edge (N-1 -> N). Source
# replica is stage 0's bound replica (single-replica thinker in
# all current configs); fall back to 0 if unknown.
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
src_replica = self.stage_pools[next_stage_id - 1].get_bound_replica_id(request_id)
self._emit_tx_edge(
from_stage=next_stage_id - 1,
from_replica=src_replica if src_replica is not None else 0,
to_stage=next_stage_id,
to_pool=next_pool,
request_id=request_id,
tx_ms=_tx_ms,
)
def _build_kv_sender_info(
self,
sender_stage_ids: list[int],