diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index d4fad4206..1f2b07a27 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -904,21 +904,22 @@ Use `/v1/audio/voices` to list available voices for the loaded model. Multi-stage omni deployments route stage outputs through a single orchestrator loop. By default that loop polls every stage replica on a 1 ms cadence. An -opt-in event-driven mode replaces the poll with one reader task per live stage +the event-driven mode replaces the poll with one reader task per live stage replica awaiting its client directly, and switches the serving-side -final-output drain to a condition-variable wakeup at the same time. +final-output drain to a condition-variable wakeup at the same time. Qwen3-TTS +uses this mode by default; other pipelines keep the legacy poll unless enabled. **Configuration (environment variables):** | Variable | Default | Description | | --- | --- | --- | -| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. | +| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off), except Qwen3-TTS defaults to on | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. An explicit value wins; otherwise the pipeline default computed at engine initialization is used. Values are matched case-insensitively after surrounding whitespace is stripped; unrecognized values select the legacy poll loop. | -Set it on the process that runs the orchestrator (stage 0 of an omni -deployment) before starting the server: +Set it on the process that runs the orchestrator (stage 0 of an omni deployment) +before starting the server when overriding the pipeline default: ```bash -export VLLM_OMNI_EVENT_DRIVEN_ORCH=1 +VLLM_OMNI_EVENT_DRIVEN_ORCH=1 \ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \ --omni \ --port 8091 diff --git a/vllm_omni/engine/orchestrator.py b/vllm_omni/engine/orchestrator.py index b047bfd82..831cdf947 100644 --- a/vllm_omni/engine/orchestrator.py +++ b/vllm_omni/engine/orchestrator.py @@ -1246,7 +1246,8 @@ class Orchestrator: async def _orchestration_loop_event_driven(self) -> None: """Event-driven variant of ``_orchestration_loop``. - Selected by ``VLLM_OMNI_EVENT_DRIVEN_ORCH=1``. One reader task per + Selected by the explicit ``VLLM_OMNI_EVENT_DRIVEN_ORCH`` value or the + pipeline default computed at engine initialization. One reader task per available LLM stage replica awaits ``client.get_output_async()`` directly — the same pattern vLLM's own ``AsyncLLM`` output handler uses — and feeds a single dispatch queue. This coroutine consumes that queue @@ -1417,6 +1418,9 @@ class Orchestrator: continue kind, stage_id, replica_id, payload = pending_get.result() pending_get = None + # Record the queue after dequeue so the gauge reflects work + # still waiting for dispatch rather than producer-side depth. + self._orch_monitor.set_dispatch_queue_size(ready_q.qsize()) if kind == "error": # replica_id < 0 means the failure was raised by a poller diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index f9c880c9f..e1d5d5239 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -60,7 +60,8 @@ if TYPE_CHECKING: logger = init_logger(__name__) _FINAL_OUTPUT_IDLE_SLEEP_S = 0.001 # Blocking-wait interval for the event-driven final-output drain -# (VLLM_OMNI_EVENT_DRIVEN_ORCH=1): a message wakes the drain immediately via +# (selected by VLLM_OMNI_EVENT_DRIVEN_ORCH or the engine pipeline default): a +# message wakes the drain immediately via # the janus queue's condition variable; this timeout only bounds how often the # orchestrator liveness check runs while the pipeline is idle. _FINAL_OUTPUT_BLOCKING_WAIT_S = 1.0 @@ -937,7 +938,7 @@ class AsyncOmni(EngineClient, OmniBase): engine = self.engine - # Event-driven drain (VLLM_OMNI_EVENT_DRIVEN_ORCH=1): block on the + # Event-driven drain (explicit env value or the engine pipeline default): block on the # queue's condition variable in a dedicated thread instead of the # get_nowait + 1 ms sleep cadence. Same flag as the orchestrator-side # event-driven loop (vllm_omni/engine/orchestrator.py).