[BugFix][Metrics] Fix per-stage time and token stats in metrics API (#4974)
Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
This commit is contained in:
1 parent
8cb79308c6
commit
cd5daaea6a
18 files changed
+618
-78
No files matched your search
@@ -331,6 +331,7 @@ class AsyncOmniEngine:
|
||||
omni_heartbeat_timeout=self._omni_heartbeat_timeout,
|
||||
omni_lb_policy=self._omni_lb_policy,
|
||||
request_queue=self.request_queue,
|
||||
log_stats=self._log_stats,
|
||||
)
|
||||
self._runtime.initialize()
|
||||
|
||||
|
||||
@@ -362,6 +362,7 @@ class Orchestrator:
|
||||
self.async_chunk = bool(async_chunk)
|
||||
self.num_stages = len(stage_pools)
|
||||
self.stage_pools: list[StagePool] = stage_pools
|
||||
self.log_stats = log_stats
|
||||
self._orch_monitor = create_orch_monitor(
|
||||
enabled=enable_orch_monitor,
|
||||
replica_sampler=self._sample_replica_metrics,
|
||||
@@ -446,7 +447,10 @@ class Orchestrator:
|
||||
already no-ops on ``scheduler_stats is None`` (which is what
|
||||
the upstream scheduler returns when its own log_stats is False),
|
||||
so this gate is mainly to keep the ``/metrics`` surface clean
|
||||
when the user did not request stats.
|
||||
when the user did not request stats. When stats are enabled, record()
|
||||
must still receive IterationStats even on ticks where scheduler_stats
|
||||
is None; upstream PrometheusStatLogger records token/request metrics
|
||||
from iteration_stats independently of scheduler gauges.
|
||||
"""
|
||||
self._running_counter = running_counter
|
||||
self._transfer_emitter = transfer_emitter
|
||||
@@ -937,19 +941,17 @@ class Orchestrator:
|
||||
)
|
||||
if req_state.streaming.enabled:
|
||||
await self._apply_raw_terminal_stage_finish(stage_id, eco, req_state)
|
||||
# OmniSchedulerMixin.make_stats() already throttles
|
||||
# per-scheduler at 1 Hz, so raw_outputs.scheduler_stats
|
||||
# being non-None means this replica passed its own gate.
|
||||
# A second global throttle here would drop stats for
|
||||
# other (stage, replica) pairs in the same 1s window.
|
||||
record_stats = self._stat_logger is not None and raw_outputs.scheduler_stats is not None
|
||||
iteration_stats = IterationStats() if record_stats else None
|
||||
iteration_stats = (
|
||||
IterationStats() if (self._stat_logger is not None and raw_outputs.outputs) else None
|
||||
)
|
||||
raw_output = await pool.process_llm_raw_outputs(
|
||||
replica_id,
|
||||
raw_outputs,
|
||||
iteration_stats=iteration_stats,
|
||||
)
|
||||
if record_stats:
|
||||
if self._stat_logger is not None and (
|
||||
raw_outputs.scheduler_stats is not None or iteration_stats is not None
|
||||
):
|
||||
self._stat_logger.record(
|
||||
raw_outputs.scheduler_stats,
|
||||
iteration_stats,
|
||||
|
||||
@@ -1081,7 +1081,11 @@ class StagePool:
|
||||
async def _poll_stage_raw(self, client: StagePoolLLMClient) -> EngineCoreOutputs | None:
|
||||
"""Pull raw EngineCoreOutputs from a stage replica without processing."""
|
||||
outputs = await client.get_output_async()
|
||||
if not outputs.outputs:
|
||||
# Keep scheduler-only / finished-only batches. Omni schedulers mirror
|
||||
# upstream by emitting SchedulerStats on throttled ticks even when no
|
||||
# request output is produced, and dropping those batches loses KV/queue
|
||||
# gauges for that interval.
|
||||
if not outputs.outputs and outputs.scheduler_stats is None and not outputs.finished_requests:
|
||||
return None
|
||||
return outputs
|
||||
|
||||
@@ -1097,7 +1101,6 @@ class StagePool:
|
||||
return []
|
||||
client = cast(StagePoolLLMClient, raw_client)
|
||||
processor = self.output_processor
|
||||
iteration_stats = IterationStats()
|
||||
processed = processor.process_outputs(
|
||||
raw_outputs.outputs,
|
||||
raw_outputs.timestamp,
|
||||
|
||||
@@ -126,6 +126,7 @@ class StageRuntime:
|
||||
diffusion_batch_size: int,
|
||||
async_chunk: bool,
|
||||
tokenizer: str | None = None,
|
||||
log_stats: bool = False,
|
||||
) -> None:
|
||||
self._stage_configs = stage_configs
|
||||
self._model = model
|
||||
@@ -134,6 +135,7 @@ class StageRuntime:
|
||||
self._diffusion_batch_size = diffusion_batch_size
|
||||
self._async_chunk = async_chunk
|
||||
self._tokenizer = tokenizer
|
||||
self._log_stats = log_stats
|
||||
self._num_stages = len(stage_configs)
|
||||
|
||||
# Populated by initialize()
|
||||
@@ -575,7 +577,7 @@ class StageRuntime:
|
||||
with launch_stage_replica(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
log_stats=self._log_stats,
|
||||
stage_id=plan.metadata.stage_id,
|
||||
replica_id=plan.replica_id,
|
||||
stage_config=plan.stage_cfg,
|
||||
@@ -594,6 +596,7 @@ class StageRuntime:
|
||||
stage_client = StageEngineCoreClientBase.make_async_mp_client(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=self._log_stats,
|
||||
metadata=plan.metadata,
|
||||
client_addresses=self._client_addresses_from_zmq(resources.addresses),
|
||||
engine_manager=resources.manager,
|
||||
@@ -704,7 +707,11 @@ class StageRuntime:
|
||||
stage_vllm_config = plan.replicas[0].stage_vllm_config
|
||||
if stage_vllm_config is None:
|
||||
raise RuntimeError(f"Stage {plan.stage_id} is missing vllm_config")
|
||||
output_processor = build_llm_stage_output_processor(plan, stage_vllm_config)
|
||||
output_processor = build_llm_stage_output_processor(
|
||||
plan,
|
||||
stage_vllm_config,
|
||||
log_stats=self._log_stats,
|
||||
)
|
||||
|
||||
stage_pools.append(
|
||||
StagePool(
|
||||
@@ -742,10 +749,11 @@ class DistStageRuntime(StageRuntime):
|
||||
stage_init_timeout: int,
|
||||
diffusion_batch_size: int,
|
||||
async_chunk: bool,
|
||||
tokenizer: str | None = None,
|
||||
single_stage_id_filter: int | None,
|
||||
omni_master_address: str,
|
||||
omni_master_port: int,
|
||||
tokenizer: str | None = None,
|
||||
log_stats: bool = False,
|
||||
omni_dp_size_local: int = 1,
|
||||
omni_heartbeat_timeout: float = 30.0,
|
||||
omni_lb_policy: str = "random",
|
||||
@@ -759,6 +767,7 @@ class DistStageRuntime(StageRuntime):
|
||||
diffusion_batch_size=diffusion_batch_size,
|
||||
async_chunk=async_chunk,
|
||||
tokenizer=tokenizer,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
self._single_stage_id_filter = single_stage_id_filter
|
||||
self._omni_master_address = omni_master_address
|
||||
@@ -1051,6 +1060,7 @@ class DistStageRuntime(StageRuntime):
|
||||
client = StageEngineCoreClientBase.make_async_mp_client(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=ctx.executor_class,
|
||||
log_stats=self._log_stats,
|
||||
metadata=metadata,
|
||||
client_addresses=client_addresses,
|
||||
engine_manager=resources.manager,
|
||||
@@ -1089,6 +1099,7 @@ def create_stage_runtime(
|
||||
omni_heartbeat_timeout: float = 30.0,
|
||||
omni_lb_policy: str = "random",
|
||||
request_queue: janus.Queue[EngineQueueMessage] | None = None,
|
||||
log_stats: bool = False,
|
||||
) -> StageRuntime:
|
||||
"""Factory: select StageRuntime or DistStageRuntime."""
|
||||
if single_stage_mode:
|
||||
@@ -1102,6 +1113,7 @@ def create_stage_runtime(
|
||||
diffusion_batch_size=diffusion_batch_size,
|
||||
async_chunk=async_chunk,
|
||||
tokenizer=tokenizer,
|
||||
log_stats=log_stats,
|
||||
single_stage_id_filter=single_stage_id_filter,
|
||||
omni_master_address=omni_master_address,
|
||||
omni_master_port=omni_master_port,
|
||||
@@ -1118,4 +1130,5 @@ def create_stage_runtime(
|
||||
diffusion_batch_size=diffusion_batch_size,
|
||||
async_chunk=async_chunk,
|
||||
tokenizer=tokenizer,
|
||||
log_stats=log_stats,
|
||||
)
|
||||
Reference in new issue
Block a user