[Metrics] Add audio SLOs + cross-stage transfer families + per-(stage, replica) wrap for upstream vllm:* (#3576)
Signed-off-by: vraiti <vraiti@redhat.com> Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: vraiti <vraiti@redhat.com>
This commit is contained in:
1 parent
ee33601571
commit
41a795b5d6
29 files changed
+3377
-25
No files matched your search
@@ -29,6 +29,7 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreOutputs
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
from vllm_omni.distributed.omni_coordinator import (
|
||||
LoadBalancer,
|
||||
@@ -54,6 +55,8 @@ from vllm_omni.engine.messages import (
|
||||
)
|
||||
from vllm_omni.engine.serialization import serialize_additional_information
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.metrics.prometheus import OmniRequestCounter
|
||||
from vllm_omni.metrics.stat_logger import OmniPrometheusStatLogger
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
# Factory signature for building a head-side stage client for a
|
||||
@@ -155,6 +158,12 @@ class Orchestrator:
|
||||
_WATCH_REPLICA_INTERVAL_S: float = 0.5
|
||||
_WATCH_REPLICA_IDLE_INTERVAL_S: float = 1.0
|
||||
|
||||
# Class-level defaults so tests that bypass __init__ via object.__new__
|
||||
# don't AttributeError when transfer / counter emit paths access them.
|
||||
_running_counter: OmniRequestCounter | None = None
|
||||
_transfer_emitter: Any = None
|
||||
_stat_logger: OmniPrometheusStatLogger | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
request_async_queue: janus.AsyncQueue[EngineQueueMessage],
|
||||
@@ -164,9 +173,12 @@ class Orchestrator:
|
||||
*,
|
||||
async_chunk: bool = False,
|
||||
pd_config: dict[str, Any] | None = None,
|
||||
running_counter: OmniRequestCounter | None = None,
|
||||
coordinator_pub_address: str | None = None,
|
||||
load_balancer_factory: Callable[[], LoadBalancer] | None = None,
|
||||
remote_replica_factory: RemoteReplicaFactory | None = None,
|
||||
transfer_emitter: Any = None,
|
||||
log_stats: bool = False,
|
||||
) -> None:
|
||||
self.request_async_queue = request_async_queue
|
||||
self.output_async_queue = output_async_queue
|
||||
@@ -186,6 +198,8 @@ class Orchestrator:
|
||||
self._pd_bootstrap_addr = pd_config.get("bootstrap_addr")
|
||||
self._pd_prefill_engine_id = pd_config.get("prefill_engine_id")
|
||||
self.request_states: dict[str, OrchestratorRequestState] = {}
|
||||
self._init_metrics_state(stage_pools, running_counter, transfer_emitter, log_stats=log_stats)
|
||||
|
||||
self._cfg_tracker = CfgCompanionTracker()
|
||||
|
||||
self._shutdown_event = asyncio.Event()
|
||||
@@ -215,6 +229,71 @@ class Orchestrator:
|
||||
pool.attach_hub(self._hub)
|
||||
pool.attach_load_balancer(factory())
|
||||
|
||||
def _init_metrics_state(
|
||||
self,
|
||||
stage_pools: list[StagePool],
|
||||
running_counter: OmniRequestCounter | None,
|
||||
transfer_emitter: Any,
|
||||
log_stats: bool = False,
|
||||
) -> None:
|
||||
"""Wire up all metric-related orchestrator state.
|
||||
|
||||
Sets ``self._running_counter`` and ``self._transfer_emitter``
|
||||
(both optional, used by request-add / forward paths), builds the
|
||||
``(stage_id, replica_id) ↔ engine_idx`` lookup used at record() time,
|
||||
and best-effort constructs the ``OmniPrometheusStatLogger`` wrap
|
||||
that exposes ~37 upstream ``vllm:*`` families with per-(stage,
|
||||
replica) labels. Failure to build the wrap is logged and metrics
|
||||
are simply disabled — orchestrator construction continues so unit
|
||||
tests with a minimal ``vllm_config`` still pass.
|
||||
|
||||
``log_stats=False`` short-circuits the wrap entirely so the
|
||||
~65 upstream ``vllm:*`` families are not registered in the
|
||||
Prometheus default registry at all. The per-step record() path
|
||||
already no-ops on ``scheduler_stats is None`` (which is what
|
||||
the upstream scheduler returns when its own log_stats is False),
|
||||
so this gate is mainly to keep the ``/metrics`` surface clean
|
||||
when the user did not request stats.
|
||||
"""
|
||||
self._running_counter = running_counter
|
||||
self._transfer_emitter = transfer_emitter
|
||||
|
||||
# Flat engine_idx ↔ (stage, replica) maps. The reverse map is
|
||||
# consulted at record() time to translate the orchestrator's
|
||||
# (stage_id, replica_id) loop variables into an engine_idx the
|
||||
# underlying PrometheusStatLogger can address.
|
||||
stage_replica_map: dict[int, tuple[str, str]] = {}
|
||||
self._stage_replica_to_engine_idx: dict[tuple[int, int], int] = {}
|
||||
flat_idx = 0
|
||||
for stage_id, pool in enumerate(stage_pools):
|
||||
for replica_id in range(pool.num_replicas):
|
||||
stage_replica_map[flat_idx] = (str(stage_id), str(replica_id))
|
||||
self._stage_replica_to_engine_idx[(stage_id, replica_id)] = flat_idx
|
||||
flat_idx += 1
|
||||
|
||||
if not log_stats:
|
||||
self._stat_logger = None
|
||||
return
|
||||
|
||||
vllm_config_for_stats = next(
|
||||
(p.stage_vllm_config for p in stage_pools if p.stage_vllm_config is not None),
|
||||
None,
|
||||
)
|
||||
if vllm_config_for_stats is None:
|
||||
self._stat_logger = None
|
||||
return
|
||||
try:
|
||||
self._stat_logger = OmniPrometheusStatLogger(
|
||||
vllm_config=vllm_config_for_stats,
|
||||
stage_replica_map=stage_replica_map,
|
||||
)
|
||||
except Exception:
|
||||
# Minimal vllm_config in unit-test contexts can lack fields the
|
||||
# upstream PrometheusStatLogger expects. Skip wrap rather than
|
||||
# break orchestrator construction.
|
||||
logger.exception("[Orchestrator] OmniPrometheusStatLogger init failed; metrics wrap disabled")
|
||||
self._stat_logger = None
|
||||
|
||||
async def run(self) -> None:
|
||||
"""Main entry point for the Orchestrator event loop."""
|
||||
logger.info("[Orchestrator] Starting event loop")
|
||||
@@ -393,6 +472,8 @@ class Orchestrator:
|
||||
mm_features=getattr(prompt, "mm_features", None),
|
||||
)
|
||||
self.request_states[request_id] = req_state
|
||||
if self._running_counter is not None:
|
||||
self._running_counter.increment()
|
||||
req_state.streaming.enabled = bool(getattr(prompt, "resumable", False))
|
||||
req_state.stage_submit_ts[stage_id] = _time.time()
|
||||
enqueue_ts = msg.enqueue_ts
|
||||
@@ -603,7 +684,24 @@ class Orchestrator:
|
||||
"new_prompt_len_snapshot",
|
||||
None,
|
||||
)
|
||||
raw_output = await pool.process_llm_raw_outputs(replica_id, raw_outputs)
|
||||
# OmniSchedulerMixin.make_stats() already throttles
|
||||
# per-scheduler at 1 Hz, so raw_outputs.scheduler_stats
|
||||
# being non-None means this replica passed its own gate.
|
||||
# A second global throttle here would drop stats for
|
||||
# other (stage, replica) pairs in the same 1s window.
|
||||
record_stats = self._stat_logger is not None and raw_outputs.scheduler_stats is not None
|
||||
iteration_stats = IterationStats() if record_stats else None
|
||||
raw_output = await pool.process_llm_raw_outputs(
|
||||
replica_id,
|
||||
raw_outputs,
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
if record_stats:
|
||||
self._stat_logger.record(
|
||||
raw_outputs.scheduler_stats,
|
||||
iteration_stats,
|
||||
engine_idx=self._stage_replica_to_engine_idx[(stage_id, replica_id)],
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except EngineDeadError as e:
|
||||
@@ -681,7 +779,7 @@ class Orchestrator:
|
||||
)
|
||||
stage_metrics.pipeline_timings = dict(req_state.pipeline_timings)
|
||||
|
||||
await self._route_output(stage_id, output, req_state, stage_metrics)
|
||||
await self._route_output(stage_id, replica_id, output, req_state, stage_metrics)
|
||||
|
||||
async def _handle_stage_error(self, stage_id: int, output: Any) -> None:
|
||||
"""Emit a frontend-visible error and clean up request state."""
|
||||
@@ -713,7 +811,8 @@ class Orchestrator:
|
||||
self._release_request_bindings(request_ids)
|
||||
for request_id in request_ids:
|
||||
self._pd_kv_params.pop(request_id, None)
|
||||
self.request_states.pop(request_id, None)
|
||||
if self.request_states.pop(request_id, None) is not None and self._running_counter is not None:
|
||||
self._running_counter.decrement()
|
||||
|
||||
def _maybe_clone_diffusion_params_for_cfg(self, request_id: str, params: Any) -> Any:
|
||||
"""Attach CFG companion ids to diffusion sampling params when needed."""
|
||||
@@ -735,6 +834,7 @@ class Orchestrator:
|
||||
async def _route_output(
|
||||
self,
|
||||
stage_id: int,
|
||||
replica_id: int,
|
||||
output: Any,
|
||||
req_state: OrchestratorRequestState,
|
||||
stage_metrics: Any,
|
||||
@@ -757,6 +857,7 @@ class Orchestrator:
|
||||
OutputMessage(
|
||||
request_id=req_id,
|
||||
stage_id=stage_id,
|
||||
replica_id=replica_id,
|
||||
engine_outputs=output,
|
||||
metrics=stage_metrics,
|
||||
finished=finished and stage_id == req_state.final_stage_id,
|
||||
@@ -768,6 +869,7 @@ class Orchestrator:
|
||||
StageMetricsMessage(
|
||||
request_id=req_id,
|
||||
stage_id=stage_id,
|
||||
replica_id=replica_id,
|
||||
metrics=stage_metrics,
|
||||
stage_submit_ts=submit_ts,
|
||||
)
|
||||
@@ -797,6 +899,7 @@ class Orchestrator:
|
||||
stage_id,
|
||||
output,
|
||||
req_state,
|
||||
src_replica_id=replica_id,
|
||||
is_streaming_session=req_state.streaming.enabled,
|
||||
is_final_update=False,
|
||||
)
|
||||
@@ -807,6 +910,7 @@ class Orchestrator:
|
||||
stage_id,
|
||||
output,
|
||||
req_state,
|
||||
src_replica_id=replica_id,
|
||||
is_streaming_session=True,
|
||||
is_final_update=True,
|
||||
)
|
||||
@@ -918,6 +1022,41 @@ class Orchestrator:
|
||||
)
|
||||
return sp
|
||||
|
||||
def _emit_tx_edge(
|
||||
self,
|
||||
*,
|
||||
from_stage: int,
|
||||
from_replica: int,
|
||||
to_stage: int,
|
||||
to_pool: StagePool,
|
||||
request_id: str,
|
||||
tx_ms: float,
|
||||
) -> None:
|
||||
"""Emit per-edge transfer_tx_s + transfer_size_bytes histograms.
|
||||
|
||||
``tx_ms`` is the orchestrator-side wall-clock spent in ``next_pool.
|
||||
submit_*`` (serialize + queue submit to the receiving worker). Best-
|
||||
effort size_bytes left at 0 — orchestrator doesn't have a cheap handle
|
||||
on the serialized payload size; a follow-up can plumb that from the
|
||||
connector adapter.
|
||||
"""
|
||||
if self._transfer_emitter is None:
|
||||
return
|
||||
to_replica = to_pool.get_bound_replica_id(request_id)
|
||||
if to_replica is None:
|
||||
return
|
||||
try:
|
||||
self._transfer_emitter.observe_size(from_stage, from_replica, to_stage, to_replica, 0)
|
||||
self._transfer_emitter.observe_tx_time(from_stage, from_replica, to_stage, to_replica, tx_ms / 1000.0)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"[Orchestrator] transfer_tx emit failed for edge %d->%d req=%s",
|
||||
from_stage,
|
||||
to_stage,
|
||||
request_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _forward_to_next_stage(
|
||||
self,
|
||||
req_id: str,
|
||||
@@ -925,6 +1064,7 @@ class Orchestrator:
|
||||
output: Any,
|
||||
req_state: OrchestratorRequestState,
|
||||
*,
|
||||
src_replica_id: int | None = None,
|
||||
is_streaming_session: bool = False,
|
||||
is_final_update: bool = False,
|
||||
) -> None:
|
||||
@@ -937,6 +1077,7 @@ class Orchestrator:
|
||||
next_stage_resumable = is_streaming_session and not is_final_update
|
||||
already_submitted = self._next_stage_already_submitted(src_stage_id, req_state)
|
||||
requires_multimodal_data = getattr(next_client, "requires_multimodal_data", False)
|
||||
_t_submit_start = _time.perf_counter()
|
||||
|
||||
if next_pool.stage_type == "diffusion":
|
||||
companion_outputs = self._cfg_tracker.pop_companion_outputs(req_id)
|
||||
@@ -1024,6 +1165,15 @@ class Orchestrator:
|
||||
params_override=self._maybe_clone_diffusion_params_for_cfg(req_id, params),
|
||||
)
|
||||
req_state.stage_submit_ts[next_logical] = _time.time()
|
||||
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
|
||||
self._emit_tx_edge(
|
||||
from_stage=src_stage_id,
|
||||
from_replica=src_replica_id if src_replica_id is not None else 0,
|
||||
to_stage=next_logical,
|
||||
to_pool=next_pool,
|
||||
request_id=req_id,
|
||||
tx_ms=_tx_ms,
|
||||
)
|
||||
return
|
||||
|
||||
# PD disaggregation: prefill → decode routing uses original prompt + KV transfer params
|
||||
@@ -1063,6 +1213,15 @@ class Orchestrator:
|
||||
await next_pool.submit_initial(req_id, req_state, request, prompt_text=None)
|
||||
|
||||
req_state.stage_submit_ts[next_logical] = _time.time()
|
||||
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
|
||||
self._emit_tx_edge(
|
||||
from_stage=src_stage_id,
|
||||
from_replica=src_replica_id if src_replica_id is not None else 0,
|
||||
to_stage=next_logical,
|
||||
to_pool=next_pool,
|
||||
request_id=req_id,
|
||||
tx_ms=_tx_ms,
|
||||
)
|
||||
return
|
||||
|
||||
if req_state.pd_prefill_multimodal_output is not None:
|
||||
@@ -1107,6 +1266,15 @@ class Orchestrator:
|
||||
await next_pool.submit_initial(req_id, req_state, request, prompt_text=None)
|
||||
|
||||
req_state.stage_submit_ts[next_logical] = _time.time()
|
||||
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
|
||||
self._emit_tx_edge(
|
||||
from_stage=src_stage_id,
|
||||
from_replica=src_replica_id if src_replica_id is not None else 0,
|
||||
to_stage=next_logical,
|
||||
to_pool=next_pool,
|
||||
request_id=req_id,
|
||||
tx_ms=_tx_ms,
|
||||
)
|
||||
|
||||
async def _prewarm_async_chunk_stages(
|
||||
self,
|
||||
@@ -1131,6 +1299,7 @@ class Orchestrator:
|
||||
params = req_state.sampling_params_list[next_stage_id]
|
||||
|
||||
req_state.stage_submit_ts[next_stage_id] = _time.time()
|
||||
_t_submit_start = _time.perf_counter()
|
||||
|
||||
if next_pool.stage_type == "diffusion":
|
||||
await next_pool.submit_initial(
|
||||
@@ -1174,6 +1343,20 @@ class Orchestrator:
|
||||
request.external_req_id = request.request_id
|
||||
await next_pool.submit_initial(request_id, req_state, request, prompt_text=None)
|
||||
|
||||
# async_chunk pre-submit fires per stage edge (N-1 -> N). Source
|
||||
# replica is stage 0's bound replica (single-replica thinker in
|
||||
# all current configs); fall back to 0 if unknown.
|
||||
_tx_ms = (_time.perf_counter() - _t_submit_start) * 1000.0
|
||||
src_replica = self.stage_pools[next_stage_id - 1].get_bound_replica_id(request_id)
|
||||
self._emit_tx_edge(
|
||||
from_stage=next_stage_id - 1,
|
||||
from_replica=src_replica if src_replica is not None else 0,
|
||||
to_stage=next_stage_id,
|
||||
to_pool=next_pool,
|
||||
request_id=request_id,
|
||||
tx_ms=_tx_ms,
|
||||
)
|
||||
|
||||
def _build_kv_sender_info(
|
||||
self,
|
||||
sender_stage_ids: list[int],
|
||||
|
||||
Reference in new issue
Block a user