[Metrics] Add audio SLOs + cross-stage transfer families + per-(stage, replica) wrap for upstream vllm:* (#3576)
Signed-off-by: vraiti <vraiti@redhat.com> Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: vraiti <vraiti@redhat.com>
This commit is contained in:
1 parent
ee33601571
commit
41a795b5d6
29 files changed
+3377
-25
No files matched your search
@@ -110,6 +110,7 @@ from vllm_omni.entrypoints.utils import (
|
||||
load_and_resolve_stage_configs,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniSamplingParams
|
||||
from vllm_omni.metrics.prometheus import OmniRequestCounter
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -259,6 +260,12 @@ class AsyncOmniEngine:
|
||||
**kwargs: Additional arguments
|
||||
"""
|
||||
|
||||
# Class-level defaults so tests that bypass __init__ via object.__new__
|
||||
# don't AttributeError when stage-init / forward paths touch these attrs.
|
||||
_log_stats: bool = False
|
||||
_coordinator_runtime: Any = None
|
||||
_transfer_emitter: Any = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
@@ -267,11 +274,23 @@ class AsyncOmniEngine:
|
||||
init_timeout: int = 600,
|
||||
diffusion_batch_size: int = 1,
|
||||
single_stage_mode: bool = False,
|
||||
transfer_emitter: Any = None,
|
||||
log_stats: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.diffusion_batch_size = diffusion_batch_size
|
||||
startup_timeout = int(init_timeout)
|
||||
# Forwarded into Orchestrator so its _forward_to_next_stage path can
|
||||
# emit per-edge transfer_tx_s / transfer_size_bytes histograms.
|
||||
# Optional: when None, Orchestrator silently skips TX emit (existing
|
||||
# RX path still works via OrchestratorAggregator).
|
||||
self._transfer_emitter = transfer_emitter
|
||||
# Drives upstream EngineCore + scheduler stats production. When False
|
||||
# the engine skips SchedulerStats / IterationStats; the per-(stage,
|
||||
# replica) vllm:* wrap stays registered but reads zero. Respects the
|
||||
# --log-stats CLI flag set by the user via OmniBase.
|
||||
self._log_stats = log_stats
|
||||
|
||||
logger.info(f"[AsyncOmniEngine] Initializing with model {model}")
|
||||
|
||||
@@ -357,6 +376,7 @@ class AsyncOmniEngine:
|
||||
self._stage_init_executor: concurrent.futures.ThreadPoolExecutor | None = None
|
||||
self._weak_finalizer: weakref.finalize | None = None
|
||||
self._rpc_lock = threading.Lock()
|
||||
self._running_counter = OmniRequestCounter()
|
||||
|
||||
logger.info(f"[AsyncOmniEngine] Launching Orchestrator thread with {self.num_stages} stages")
|
||||
|
||||
@@ -905,7 +925,7 @@ class AsyncOmniEngine:
|
||||
launch_omni_core_engines(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
log_stats=self._log_stats,
|
||||
omni_master_server=self._omni_master_server,
|
||||
stage_id=plan.metadata.stage_id,
|
||||
stage_config=stage_cfg,
|
||||
@@ -917,7 +937,7 @@ class AsyncOmniEngine:
|
||||
addresses, proc, handshake_address = spawn_stage_core(
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=False,
|
||||
log_stats=self._log_stats,
|
||||
)
|
||||
logger.info(
|
||||
"[AsyncOmniEngine] Stage %s engine launch started",
|
||||
@@ -1229,7 +1249,7 @@ class AsyncOmniEngine:
|
||||
if plan.replicas[0].metadata.stage_type != "diffusion":
|
||||
stage_vllm_config = plan.replicas[0].stage_vllm_config
|
||||
assert stage_vllm_config is not None
|
||||
output_processor = build_llm_stage_output_processor(plan, stage_vllm_config)
|
||||
output_processor = build_llm_stage_output_processor(plan, stage_vllm_config, log_stats=self._log_stats)
|
||||
|
||||
stage_pools.append(
|
||||
StagePool(
|
||||
@@ -1374,9 +1394,12 @@ class AsyncOmniEngine:
|
||||
stage_pools=self.stage_pools,
|
||||
async_chunk=self.async_chunk,
|
||||
pd_config=pd_config,
|
||||
running_counter=self._running_counter,
|
||||
coordinator_pub_address=coordinator_pub_address,
|
||||
load_balancer_factory=load_balancer_factory,
|
||||
remote_replica_factory=remote_replica_factory,
|
||||
transfer_emitter=self._transfer_emitter,
|
||||
log_stats=self._log_stats,
|
||||
)
|
||||
if not startup_future.done():
|
||||
startup_future.set_result(asyncio.get_running_loop())
|
||||
|
||||
Reference in new issue
Block a user