[BugFix][Metrics] Fix per-stage time and token stats in metrics API (#4974)

Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
akshatvishu authored and GitHub committed 2026-08-01 03:54:17 +00:00
1 parent 8cb79308c6
commit cd5daaea6a
18 files changed
+618 -78

No files matched your search

+1
View File
@@ -331,6 +331,7 @@ class AsyncOmniEngine:
omni_heartbeat_timeout=self._omni_heartbeat_timeout,
omni_lb_policy=self._omni_lb_policy,
request_queue=self.request_queue,
log_stats=self._log_stats,
)
self._runtime.initialize()
+11 -9
View File
@@ -362,6 +362,7 @@ class Orchestrator:
self.async_chunk = bool(async_chunk)
self.num_stages = len(stage_pools)
self.stage_pools: list[StagePool] = stage_pools
self.log_stats = log_stats
self._orch_monitor = create_orch_monitor(
enabled=enable_orch_monitor,
replica_sampler=self._sample_replica_metrics,
@@ -446,7 +447,10 @@ class Orchestrator:
already no-ops on ``scheduler_stats is None`` (which is what
the upstream scheduler returns when its own log_stats is False),
so this gate is mainly to keep the ``/metrics`` surface clean
when the user did not request stats.
when the user did not request stats. When stats are enabled, record()
must still receive IterationStats even on ticks where scheduler_stats
is None; upstream PrometheusStatLogger records token/request metrics
from iteration_stats independently of scheduler gauges.
"""
self._running_counter = running_counter
self._transfer_emitter = transfer_emitter
@@ -937,19 +941,17 @@ class Orchestrator:
)
if req_state.streaming.enabled:
await self._apply_raw_terminal_stage_finish(stage_id, eco, req_state)
# OmniSchedulerMixin.make_stats() already throttles
# per-scheduler at 1 Hz, so raw_outputs.scheduler_stats
# being non-None means this replica passed its own gate.
# A second global throttle here would drop stats for
# other (stage, replica) pairs in the same 1s window.
record_stats = self._stat_logger is not None and raw_outputs.scheduler_stats is not None
iteration_stats = IterationStats() if record_stats else None
iteration_stats = (
IterationStats() if (self._stat_logger is not None and raw_outputs.outputs) else None
)
raw_output = await pool.process_llm_raw_outputs(
replica_id,
raw_outputs,
iteration_stats=iteration_stats,
)
if record_stats:
if self._stat_logger is not None and (
raw_outputs.scheduler_stats is not None or iteration_stats is not None
):
self._stat_logger.record(
raw_outputs.scheduler_stats,
iteration_stats,
+5 -2
View File
@@ -1081,7 +1081,11 @@ class StagePool:
async def _poll_stage_raw(self, client: StagePoolLLMClient) -> EngineCoreOutputs | None:
"""Pull raw EngineCoreOutputs from a stage replica without processing."""
outputs = await client.get_output_async()
if not outputs.outputs:
# Keep scheduler-only / finished-only batches. Omni schedulers mirror
# upstream by emitting SchedulerStats on throttled ticks even when no
# request output is produced, and dropping those batches loses KV/queue
# gauges for that interval.
if not outputs.outputs and outputs.scheduler_stats is None and not outputs.finished_requests:
return None
return outputs
@@ -1097,7 +1101,6 @@ class StagePool:
return []
client = cast(StagePoolLLMClient, raw_client)
processor = self.output_processor
iteration_stats = IterationStats()
processed = processor.process_outputs(
raw_outputs.outputs,
raw_outputs.timestamp,
+16 -3
View File
@@ -126,6 +126,7 @@ class StageRuntime:
diffusion_batch_size: int,
async_chunk: bool,
tokenizer: str | None = None,
log_stats: bool = False,
) -> None:
self._stage_configs = stage_configs
self._model = model
@@ -134,6 +135,7 @@ class StageRuntime:
self._diffusion_batch_size = diffusion_batch_size
self._async_chunk = async_chunk
self._tokenizer = tokenizer
self._log_stats = log_stats
self._num_stages = len(stage_configs)
# Populated by initialize()
@@ -575,7 +577,7 @@ class StageRuntime:
with launch_stage_replica(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=False,
log_stats=self._log_stats,
stage_id=plan.metadata.stage_id,
replica_id=plan.replica_id,
stage_config=plan.stage_cfg,
@@ -594,6 +596,7 @@ class StageRuntime:
stage_client = StageEngineCoreClientBase.make_async_mp_client(
vllm_config=vllm_config,
executor_class=executor_class,
log_stats=self._log_stats,
metadata=plan.metadata,
client_addresses=self._client_addresses_from_zmq(resources.addresses),
engine_manager=resources.manager,
@@ -704,7 +707,11 @@ class StageRuntime:
stage_vllm_config = plan.replicas[0].stage_vllm_config
if stage_vllm_config is None:
raise RuntimeError(f"Stage {plan.stage_id} is missing vllm_config")
output_processor = build_llm_stage_output_processor(plan, stage_vllm_config)
output_processor = build_llm_stage_output_processor(
plan,
stage_vllm_config,
log_stats=self._log_stats,
)
stage_pools.append(
StagePool(
@@ -742,10 +749,11 @@ class DistStageRuntime(StageRuntime):
stage_init_timeout: int,
diffusion_batch_size: int,
async_chunk: bool,
tokenizer: str | None = None,
single_stage_id_filter: int | None,
omni_master_address: str,
omni_master_port: int,
tokenizer: str | None = None,
log_stats: bool = False,
omni_dp_size_local: int = 1,
omni_heartbeat_timeout: float = 30.0,
omni_lb_policy: str = "random",
@@ -759,6 +767,7 @@ class DistStageRuntime(StageRuntime):
diffusion_batch_size=diffusion_batch_size,
async_chunk=async_chunk,
tokenizer=tokenizer,
log_stats=log_stats,
)
self._single_stage_id_filter = single_stage_id_filter
self._omni_master_address = omni_master_address
@@ -1051,6 +1060,7 @@ class DistStageRuntime(StageRuntime):
client = StageEngineCoreClientBase.make_async_mp_client(
vllm_config=vllm_config,
executor_class=ctx.executor_class,
log_stats=self._log_stats,
metadata=metadata,
client_addresses=client_addresses,
engine_manager=resources.manager,
@@ -1089,6 +1099,7 @@ def create_stage_runtime(
omni_heartbeat_timeout: float = 30.0,
omni_lb_policy: str = "random",
request_queue: janus.Queue[EngineQueueMessage] | None = None,
log_stats: bool = False,
) -> StageRuntime:
"""Factory: select StageRuntime or DistStageRuntime."""
if single_stage_mode:
@@ -1102,6 +1113,7 @@ def create_stage_runtime(
diffusion_batch_size=diffusion_batch_size,
async_chunk=async_chunk,
tokenizer=tokenizer,
log_stats=log_stats,
single_stage_id_filter=single_stage_id_filter,
omni_master_address=omni_master_address,
omni_master_port=omni_master_port,
@@ -1118,4 +1130,5 @@ def create_stage_runtime(
diffusion_batch_size=diffusion_batch_size,
async_chunk=async_chunk,
tokenizer=tokenizer,
log_stats=log_stats,
)