[Core][Frontend] Unified Full-duplex Framework (#7413)
Signed-off-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
chickeytonandClaude Opus 5 authored and GitHub committed
1 parent
9005d78903
commit
99ff4f3070
182 files changed
+26781
-34573
No files matched your search
@@ -38,9 +38,13 @@ x-minicpmo-4-5: &minicpmo_4_5
|
||||
x-minicpmo-4-5-duplex: &minicpmo_4_5_duplex
|
||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||
- vllm_omni/engine/duplex/
|
||||
- vllm_omni/engine/duplex_omni_engine.py
|
||||
- vllm_omni/engine/duplex_orchestrator.py
|
||||
- vllm_omni/engine/omni_engine_base.py
|
||||
- vllm_omni/entrypoints/duplex/
|
||||
- vllm_omni/entrypoints/duplex_omni.py
|
||||
- vllm_omni/entrypoints/async_omni_base.py
|
||||
- vllm_omni/clients/
|
||||
- vllm_omni/entrypoints/duplex_request_client.py
|
||||
- vllm_omni/outputs/duplex.py
|
||||
- vllm_omni/model_executor/duplex_sampling.py
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
@@ -249,7 +253,11 @@ source_file_dependencies:
|
||||
- tests/assets/minicpmo_4_5/
|
||||
- tests/clients/
|
||||
- tests/engine/duplex/
|
||||
- tests/engine/test_duplex_omni_engine.py
|
||||
- tests/engine/test_duplex_orchestrator.py
|
||||
- tests/entrypoints/duplex/
|
||||
- tests/entrypoints/test_duplex_omni.py
|
||||
- tests/entrypoints/openai_api/test_duplex_api_server.py
|
||||
- tests/model_executor/models/minicpmo_4_5/duplex/
|
||||
|
||||
omni_minicpmo_4_5_duplex_perf:
|
||||
|
||||
@@ -66,7 +66,9 @@ steps:
|
||||
# 0.26 startup regression is understood and fixed.
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
# Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5
|
||||
# (see the P0 thread on #7413); the test stays in the tree and runs again there.
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Entrypoint Test with L4"
|
||||
|
||||
+6
-1
@@ -5,6 +5,9 @@
|
||||
Main entry points for vLLM-Omni inference and serving.
|
||||
|
||||
- [vllm_omni.entrypoints.async_omni.AsyncOmni][]
|
||||
- [vllm_omni.entrypoints.async_omni_base.AsyncOmniBase][]
|
||||
- [vllm_omni.entrypoints.duplex_omni.DuplexOmni][]
|
||||
- [vllm_omni.entrypoints.duplex_omni.DuplexSessionHandle][]
|
||||
- [vllm_omni.engine.cfg_companion_tracker.CfgCompanionTracker][]
|
||||
- [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
|
||||
- [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]
|
||||
@@ -17,12 +20,13 @@ Main entry points for vLLM-Omni inference and serving.
|
||||
|
||||
## Clients
|
||||
|
||||
Client libraries that talk to a running vLLM-Omni server (the full-duplex Realtime client and its per-model session presets).
|
||||
Client libraries for full-duplex sessions: the Realtime WebSocket client, the in-process client over `DuplexOmni`, and the per-model session presets.
|
||||
|
||||
- [vllm_omni.clients.duplex.AudioDelta][]
|
||||
- [vllm_omni.clients.duplex.AudioFormat][]
|
||||
- [vllm_omni.clients.duplex.ConnectionResumed][]
|
||||
- [vllm_omni.clients.duplex.DuplexClient][]
|
||||
- [vllm_omni.clients.duplex.DuplexClientBase][]
|
||||
- [vllm_omni.clients.duplex.DuplexClientError][]
|
||||
- [vllm_omni.clients.duplex.DuplexConnectionError][]
|
||||
- [vllm_omni.clients.duplex.DuplexEvent][]
|
||||
@@ -44,6 +48,7 @@ Client libraries that talk to a running vLLM-Omni server (the full-duplex Realti
|
||||
- [vllm_omni.clients.duplex.TextDelta][]
|
||||
- [vllm_omni.clients.duplex.TranscriptDelta][]
|
||||
- [vllm_omni.clients.duplex.WebSocketTransport][]
|
||||
- [vllm_omni.clients.inline_duplex.InlineDuplexClient][]
|
||||
|
||||
## Inputs
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ Common `PipelineConfig` fields include:
|
||||
| ------- | ------------- |
|
||||
| `model_type` | Pipeline identifier used during model and config resolution. |
|
||||
| `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. |
|
||||
| `duplex_plugin` | Dotted path of the model's `DuplexModelPlugin`. Set only for full-duplex models; it makes `vllm-omni serve` run the model through `DuplexOmni` (a server whose every surface runs on a duplex session) and the engine host a `DuplexOrchestrator` with the plugin loaded. |
|
||||
| `model_arch` | Default Hugging Face architecture for the pipeline. |
|
||||
| `hf_architectures` | Architecture names used to identify checkpoints whose `model_type` is shared. |
|
||||
| `hf_config_predicate` | Optional predicate used to select between pipelines with otherwise identical HF metadata. |
|
||||
@@ -52,7 +53,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
|
||||
| ------- | ------ | ---------- | --------- | ------------- |
|
||||
| `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
|
||||
| `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
|
||||
| `session_mode` | str | optional | `"turn"` | Session behavior. MiniCPM-o 4.5 deploy YAMLs set `"duplex"` so `/v1/realtime` and chat share the same profile. |
|
||||
| `session_mode` | str | optional | `"turn"` | Session behavior. Duplex models (pipelines with a `duplex_plugin`, e.g. MiniCPM-o 4.5) set `"duplex"`; `vllm-omni serve` then serves it over `/v1/realtime?duplex=1` and `/v1/chat/completions`, both on duplex sessions. |
|
||||
| `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
|
||||
| `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
|
||||
| `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
|
||||
|
||||
+359
-1056
File diff suppressed because it is too large.
Load diff
@@ -16,6 +16,7 @@ required_reviewers:
|
||||
- "@yenuo26"
|
||||
- "@NickCao"
|
||||
primary_code_paths:
|
||||
- vllm_omni/engine/omni_engine_base.py
|
||||
- vllm_omni/engine/async_omni_engine.py
|
||||
- vllm_omni/engine/async_engine_utils.py
|
||||
- vllm_omni/engine/orchestrator.py
|
||||
@@ -28,8 +29,12 @@ related_code_paths:
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
- vllm_omni/outputs/output_processor.py
|
||||
- vllm_omni/distributed/omni_coordinator/**
|
||||
# Duplex kernel design is owned by ../fullduplex.md; listed here because the
|
||||
# orchestrator hosts the duplex control plane.
|
||||
# The duplex siblings (DuplexOmniEngine / DuplexOrchestrator) and the
|
||||
# engine-resident session framework are owned by ../fullduplex.md; listed
|
||||
# here because they fill the template seams of OmniEngineBase and
|
||||
# OrchestratorBase.
|
||||
- vllm_omni/engine/duplex_omni_engine.py
|
||||
- vllm_omni/engine/duplex_orchestrator.py
|
||||
- vllm_omni/engine/duplex/**
|
||||
depends_on:
|
||||
- input_output_modality_contracts.md
|
||||
@@ -48,6 +53,8 @@ validation_paths:
|
||||
- tests/engine/test_cfg_companion_lifecycle.py
|
||||
- tests/engine/test_rpc_result_router.py
|
||||
- tests/engine/duplex/**
|
||||
- tests/engine/test_duplex_orchestrator.py
|
||||
- tests/engine/test_duplex_omni_engine.py
|
||||
- tests/engine/test_duplex_import_boundary.py
|
||||
upstream_refs:
|
||||
- vllm.v1.engine.EngineCoreRequest
|
||||
|
||||
@@ -19,7 +19,9 @@ required_reviewers:
|
||||
primary_code_paths:
|
||||
- vllm_omni/entrypoints/omni.py
|
||||
- vllm_omni/entrypoints/async_omni.py
|
||||
- vllm_omni/entrypoints/async_omni_base.py
|
||||
- vllm_omni/entrypoints/omni_base.py
|
||||
- vllm_omni/entrypoints/duplex_omni.py
|
||||
- vllm_omni/entrypoints/cli/**
|
||||
- vllm_omni/entrypoints/openai/**
|
||||
- vllm_omni/entrypoints/openpi/**
|
||||
@@ -27,14 +29,13 @@ primary_code_paths:
|
||||
- vllm_omni/entrypoints/stage_utils.py
|
||||
- vllm_omni/entrypoints/utils.py
|
||||
- vllm_omni/entrypoints/duplex/**
|
||||
- vllm_omni/entrypoints/duplex_request_client.py
|
||||
- vllm_omni/clients/**
|
||||
primary_path_exceptions:
|
||||
- path: vllm_omni/entrypoints/openai/errors.py
|
||||
owner: error_contracts.md
|
||||
- path: vllm_omni/entrypoints/duplex/**
|
||||
owner: ../fullduplex.md
|
||||
- path: vllm_omni/entrypoints/duplex_request_client.py
|
||||
- path: vllm_omni/entrypoints/duplex_omni.py
|
||||
owner: ../fullduplex.md
|
||||
- path: vllm_omni/clients/**
|
||||
owner: ../fullduplex.md
|
||||
@@ -42,7 +43,9 @@ related_code_paths:
|
||||
- vllm_omni/errors.py
|
||||
- vllm_omni/inputs/**
|
||||
- vllm_omni/outputs/**
|
||||
- vllm_omni/engine/omni_engine_base.py
|
||||
- vllm_omni/engine/async_omni_engine.py
|
||||
- vllm_omni/engine/duplex_omni_engine.py
|
||||
- vllm_omni/config/**
|
||||
- vllm_omni/deploy/**
|
||||
depends_on:
|
||||
@@ -53,7 +56,9 @@ validation_paths:
|
||||
- tests/entrypoints/test_omni_entrypoints.py
|
||||
- tests/entrypoints/test_async_omni.py
|
||||
- tests/entrypoints/test_async_omni_pause_sleep_routing.py
|
||||
- tests/entrypoints/test_async_omni_duplex.py
|
||||
- tests/entrypoints/test_duplex_omni.py
|
||||
- tests/entrypoints/duplex/**
|
||||
- tests/entrypoints/openai_api/test_duplex_api_server.py
|
||||
- tests/entrypoints/test_serve.py
|
||||
- tests/entrypoints/test_stream_finish_reason.py
|
||||
- tests/entrypoints/openai/**
|
||||
|
||||
@@ -25,8 +25,8 @@ CATEGORIES = {
|
||||
"clients": {
|
||||
"name": "Clients",
|
||||
"description": (
|
||||
"Client libraries that talk to a running vLLM-Omni server "
|
||||
"(the full-duplex Realtime client and its per-model session presets)."
|
||||
"Client libraries for full-duplex sessions: the Realtime WebSocket client, "
|
||||
"the in-process client over `DuplexOmni`, and the per-model session presets."
|
||||
),
|
||||
},
|
||||
"inputs": {
|
||||
|
||||
@@ -107,7 +107,7 @@ th {
|
||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
|
||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
|
||||
| `DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
|
||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
|
||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/OpenBMB/MiniCPM-o-4_5.md) |
|
||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
|
||||
| `HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
|
||||
|
||||
+12
-5
@@ -145,15 +145,22 @@ interchangeably.
|
||||
| Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
|
||||
| Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
|
||||
| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
|
||||
| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` or `WS /v1/duplex` | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
|
||||
| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` (alias `WS /v1/duplex`) | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
|
||||
| Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
|
||||
| Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
|
||||
|
||||
All six routes are model- or configuration-dependent. In particular,
|
||||
`/v1/realtime` is not full duplex unless the client sets `duplex=1` and the
|
||||
deployment explicitly enables duplex sessions. Clients should also verify the
|
||||
duplex capability payload because the query-parameter form falls back to the
|
||||
ordinary realtime handler when duplex is unavailable.
|
||||
`/v1/realtime` is full duplex when the model is a duplex model (its pipeline
|
||||
declares a `duplex_plugin` and the deploy configuration sets
|
||||
`session_mode: duplex`); a stock Realtime client needs no vendor query
|
||||
parameter, and `duplex=0` is the explicit opt-out. Such a server serves the
|
||||
websocket route plus `POST /v1/chat/completions`, and no other turn-based
|
||||
HTTP route. On a duplex server that chat route is not the turn-based path:
|
||||
each request runs on a short-lived duplex session, so it holds one of
|
||||
`duplex_session.max_sessions` for its lifetime and answers at the model's
|
||||
real-time pace -- see [Full Duplex](full_duplex_api.md). Clients should
|
||||
also verify the duplex capability payload because the query-parameter form
|
||||
falls back to the ordinary realtime handler when duplex is unavailable.
|
||||
|
||||
## Related Endpoints
|
||||
|
||||
|
||||
@@ -15,40 +15,52 @@ capability gates, see the [Realtime Duplex API guide](realtime_duplex_api.md).
|
||||
|
||||
| Endpoint | Protocol | Recommended use |
|
||||
| --- | --- | --- |
|
||||
| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style event projection | Applications and browser clients |
|
||||
| `WS /v1/duplex` | Native vLLM-Omni duplex events | Runtime integration and low-level testing |
|
||||
|
||||
Both endpoints use the same duplex engine and require the same model-side
|
||||
adapter. Prefer the Realtime projection unless the native lifecycle events are
|
||||
specifically required.
|
||||
| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style events (the normative contract) | Applications and browser clients |
|
||||
| `WS /v1/duplex` | Alias of `/v1/realtime?duplex=1` (same protocol, same handler) | Clients that prefer a dedicated path |
|
||||
| Python: `DuplexOmni` / `InlineDuplexClient` | Typed commands and events in-process | Embedding the model without a server |
|
||||
|
||||
## Enable Full Duplex
|
||||
|
||||
The route becomes usable only when the deployment configuration explicitly
|
||||
sets:
|
||||
A model is served full duplex when its registered pipeline declares a
|
||||
`duplex_plugin` (the model's `DuplexModelPlugin`). That declaration alone
|
||||
decides it: `vllm-omni serve` constructs `DuplexOmni` instead of `AsyncOmni`,
|
||||
and every surface the server exposes is backed by a duplex session. It serves
|
||||
`/v1/realtime?duplex=1` (and its alias `/v1/duplex`),
|
||||
`POST /v1/chat/completions`, `/v1/models` and `/health`; every other
|
||||
turn-based HTTP route (speech, batch, embeddings, video, ...) answers "not
|
||||
available". Turn-based use of such a model stays available offline through the
|
||||
Python API (`Omni` / `AsyncOmni`).
|
||||
|
||||
`/v1/chat/completions` is the ordinary chat service running on the duplex
|
||||
engine: a request is a turn-based generation on the same stages, served
|
||||
alongside the live websocket sessions, with the request options the chat
|
||||
service supports. It is wired only when the model's plugin declares
|
||||
`DuplexCapabilities.supports_chat_completions`; on any other duplex model the
|
||||
route reports "not available". A model that should not serve the route at all
|
||||
lists it in the deploy configuration's `endpoint_restrictions`.
|
||||
|
||||
The deploy configuration of such a model must agree:
|
||||
|
||||
```yaml
|
||||
session_mode: duplex
|
||||
```
|
||||
|
||||
The selected model pipeline must also provide a duplex serving adapter.
|
||||
Model-native deployments configure the corresponding engine runtime extension
|
||||
and control plane as part of their registered pipeline.
|
||||
A duplex model started with a deploy configuration that does not set it fails
|
||||
at startup rather than falling back to turn-based serving.
|
||||
|
||||
!!! warning
|
||||
|
||||
`WS /v1/duplex` fails with `Duplex API is not available` when duplex is not
|
||||
enabled. By contrast, `/v1/realtime?duplex=1` falls back to the ordinary
|
||||
turn-based realtime handler when no duplex handler exists. Confirm that
|
||||
On a deployment that is *not* duplex, `WS /v1/duplex` fails with
|
||||
`Duplex API is not available` and `/v1/realtime?duplex=1` falls back to the
|
||||
ordinary turn-based realtime handler. Confirm that
|
||||
`session.created.session.capabilities` is present before treating the
|
||||
connection as full duplex.
|
||||
|
||||
The current unified-runtime integrations are:
|
||||
|
||||
- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5.yaml`
|
||||
(`session_mode: duplex`);
|
||||
- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex;
|
||||
- Nemotron VoiceChat, via its registered duplex plugin package.
|
||||
**MiniCPM-o 4.5** (`vllm_omni/deploy/minicpmo_4_5.yaml`) is the only model
|
||||
served over this endpoint today. PersonaPlex and Nemotron VoiceChat still carry
|
||||
their pre-framework duplex code: their pipelines declare no `duplex_plugin`, so
|
||||
they run turn-based until the follow-up PRs port them to the plugin contract
|
||||
(RFC [vllm-omni#7181](https://github.com/vllm-project/vllm-omni/issues/7181)).
|
||||
|
||||
JoyVL is a separate HTTP interaction orchestrator and does not use these
|
||||
WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md).
|
||||
@@ -101,22 +113,23 @@ deferred, treated as a short acknowledgement, or used to interrupt output.
|
||||
The `session.created` payload includes capability fields such as
|
||||
`supports_barge_in`, `supports_playback_ack`, `supports_multi_session`,
|
||||
`supports_session_resume`, and `chunk_period_ms`. Treat this payload as the
|
||||
runtime contract.
|
||||
runtime contract and branch on the flags, never on the model name: a model
|
||||
that supports native overlapping speech may still advertise
|
||||
`supports_barge_in=false` when destructive output interruption and model-state
|
||||
rewind are not validated for it. Capacity and session-resume behavior also
|
||||
depend on the selected deployment configuration. The per-model table lives in
|
||||
the [Realtime Duplex API guide](realtime_duplex_api.md#capability-negotiation-by-model).
|
||||
|
||||
For example, PersonaPlex supports native overlapping speech but currently
|
||||
advertises `supports_barge_in=false`; destructive output interruption and
|
||||
model-state rewind have not been validated for that integration. Capacity and
|
||||
session-resume behavior also depend on the selected deployment configuration.
|
||||
## Python API
|
||||
|
||||
## Native Protocol
|
||||
`vllm_omni.entrypoints.duplex_omni.DuplexOmni` runs the same engine-resident
|
||||
sessions in-process: `open_session()` returns a `DuplexSessionHandle` whose
|
||||
`submit()` takes typed `DuplexCommand` objects and whose `events()` yields
|
||||
typed `DuplexEvent` objects (each with a `to_realtime()` wire rendering).
|
||||
`vllm_omni.clients.inline_duplex.InlineDuplexClient` exposes that handle
|
||||
behind the `DuplexClient` API. See the
|
||||
[Realtime Duplex API guide](realtime_duplex_api.md#using-the-python-api).
|
||||
|
||||
`WS /v1/duplex` exposes lower-level lifecycle names including
|
||||
`session.create`, `input_audio_buffer.append`, `turn.signal`, `playback.ack`,
|
||||
and `session.close`. It returns native session, input, response, overlap, and
|
||||
error events. This protocol is experimental and may evolve with the runtime;
|
||||
applications should use the provided Realtime client where possible.
|
||||
|
||||
See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo),
|
||||
[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex),
|
||||
See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo)
|
||||
and the [full-duplex runtime design](../design/fullduplex.md)
|
||||
for model-specific validation and architecture details.
|
||||
+214
-135
@@ -1,17 +1,18 @@
|
||||
# Realtime Duplex API
|
||||
|
||||
vLLM-Omni serves full-duplex speech models over a WebSocket endpoint,
|
||||
`/v1/realtime`, that speaks the OpenAI Realtime vocabulary plus a
|
||||
`/v1/realtime?duplex=1`, that speaks the OpenAI Realtime vocabulary plus a
|
||||
set of duplex extensions: the model decides when to listen and when to speak,
|
||||
the user can talk over the assistant, playback progress is reported back so
|
||||
history stays honest, and a dropped connection can resume the same session.
|
||||
This page covers how to run a duplex deployment, how to drive it from Python
|
||||
with `vllm_omni.clients.duplex.DuplexClient`, and the complete wire contract.
|
||||
|
||||
The handler is enabled by either a model-native `session_mode: duplex`
|
||||
deployment or an explicit `duplex_session` configuration for turn-based
|
||||
Server VAD. The runtime architecture is described in
|
||||
[Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
|
||||
The endpoint is served for models that ship a duplex plugin — currently
|
||||
MiniCPM-o 4.5 only — and just for deploy configurations that declare
|
||||
`session_mode: duplex`. PersonaPlex and Nemotron VoiceChat are ported to the
|
||||
plugin contract in follow-up PRs and are not served over this endpoint yet. The runtime architecture
|
||||
is described in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -26,11 +27,38 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
|
||||
```
|
||||
|
||||
`vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
|
||||
`duplex_session.max_sessions: 4`, which mounts the WebSocket routes
|
||||
`ws://<host>:8099/v1/realtime` (this page) and `ws://<host>:8099/v1/duplex`
|
||||
(the native dialect for raw-protocol clients) next to the usual HTTP API.
|
||||
PersonaPlex and Nemotron VoiceChat use `vllm_omni/deploy/personaplex.yaml` and
|
||||
`vllm_omni/deploy/nemotron_labs_voicechat_duplex.yaml` in the same way.
|
||||
`duplex_session.max_sessions: 4`. Because the MiniCPM-o 4.5 pipeline declares a
|
||||
`duplex_plugin`, `vllm-omni serve` runs it through `DuplexOmni`: the server
|
||||
mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
|
||||
is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and
|
||||
`/health`, and nothing else. The remaining turn-based HTTP routes (speech,
|
||||
batch, embeddings, video, ...) are not served by a duplex model.
|
||||
|
||||
`/v1/chat/completions` is not a turn-based path on a duplex server: each
|
||||
request opens a short-lived duplex session and reads the answer off it. Audio
|
||||
content becomes an ordinary committed turn; text is seeded as the session's
|
||||
opening turn, because a model-native model takes a turn only when it hears
|
||||
speech. A model that cannot be seeded that way declares so, and a text request
|
||||
to it is refused with 400. Two consequences are worth sizing for:
|
||||
|
||||
- **Every request holds an admission slot** for its lifetime, so
|
||||
`duplex_session.max_sessions` caps HTTP concurrency as well as websocket
|
||||
sessions. Beyond it a request is refused with HTTP 503.
|
||||
- **Latency follows the model's own clock.** A model-native session generates
|
||||
per audio unit, so even a short answer costs what that many units cost --
|
||||
noticeably more than the same model would take turn-based.
|
||||
|
||||
`n > 1`, `logprobs`, `tools`, and image or video content parts have no
|
||||
representation in one duplex turn and are refused with HTTP 400 -- a Realtime
|
||||
conversation item carries text and audio only. `temperature`,
|
||||
`max_completion_tokens`, `modalities` and
|
||||
`chat_template_kwargs.use_tts_template` apply; the other
|
||||
`chat_template_kwargs` keys are logged as ignored. `extra_body` is passed to
|
||||
the session unchanged, exactly as a websocket client's session `extra_body`
|
||||
is, which is how a request asking for `modalities: ["text", "audio"]` supplies
|
||||
the `ref_audio` MiniCPM-o requires for audio output. A model that should not
|
||||
serve the route lists it in the deploy config's `endpoint_restrictions`,
|
||||
exactly as a turn-based model does.
|
||||
|
||||
### Run the Example Client
|
||||
|
||||
@@ -81,25 +109,24 @@ ships a preset:
|
||||
|
||||
| Preset | Input / output audio | What it sets |
|
||||
| --- | --- | --- |
|
||||
| `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config(ref_audio=...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | `extra_body.native_duplex=True` (opt in to the model-native lane), `force_listen_count=0`, `overlap_policy="listen_only"`, `playback_commit_policy="ack_only"`; `ref_audio` is the assistant voice clip |
|
||||
| `vllm_omni.clients.minicpmo_4_5.create_duplex_session_config(ref_audio=...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | `force_listen_count=0`, `overlap_policy="listen_only"`, `playback_commit_policy="ack_only"`; `ref_audio` is the assistant voice clip |
|
||||
| `vllm_omni.clients.personaplex.create_duplex_session_config(voice="NATF2.pt", persona="")` | 24 kHz `pcm_f32le` / 24 kHz `pcm16` | bundled `.pt` voice prompt and the persona as `instructions` |
|
||||
| `SessionConfig(...)` | 16 kHz `pcm16` / 24 kHz `pcm16` | model-neutral defaults; pass `extra_body`, `turn_detection`, `overlap_policy`, `playback_commit_policy`, `instructions`, `voice`, `temperature` yourself |
|
||||
|
||||
Keyword overrides on a preset replace the corresponding `SessionConfig`
|
||||
field, e.g. `create_duplex_session_config(ref_audio=..., temperature=0.6)`.
|
||||
|
||||
Constructor options: `session_id` (names the session the client creates; it
|
||||
does not resume or take over an existing session — entering the client always
|
||||
performs the `session.update` handshake), `reconnect` (a `ReconnectPolicy`,
|
||||
default five attempts with 0.25–4 s jittered backoff; `None` disables
|
||||
auto-resume), `heartbeat_interval_s` (default 30 s; `None` disables the lease
|
||||
heartbeat), `handshake_timeout_s`, and `connect` (a custom transport factory,
|
||||
used by tests and benchmarks). Resume is supported only as automatic reconnect
|
||||
within the same `DuplexClient` instance (see
|
||||
[Reconnect and resume](#reconnect-and-resume)); re-attaching from a new client
|
||||
or process requires the wire-level `session.resume` handshake
|
||||
(`resume_token`, `incarnation`, `last_received_server_event_seq`), which this
|
||||
client does not expose.
|
||||
Constructor options: `reconnect` (a `ReconnectPolicy`, default five
|
||||
attempts with 0.25–4 s jittered backoff; `None` disables auto-resume),
|
||||
`heartbeat_interval_s` (default 30 s; `None` disables the lease heartbeat),
|
||||
`handshake_timeout_s`, and `connect` (a custom transport factory, used by
|
||||
tests and benchmarks). The session id is allocated by the server and becomes
|
||||
`client.session_id` once `session.created` arrives; a client cannot choose
|
||||
it. Resume is supported only as automatic reconnect within the same
|
||||
`DuplexClient` instance (see [Reconnect and resume](#reconnect-and-resume));
|
||||
re-attaching from a new client or process requires the wire-level
|
||||
`session.resume` handshake (`resume_token`,
|
||||
`last_received_server_event_seq`), which this client does not expose.
|
||||
|
||||
### Stream audio and commit a turn
|
||||
|
||||
@@ -195,7 +222,7 @@ the stamped `event_id`.
|
||||
On a transport drop the client reconnects with the `ReconnectPolicy` backoff
|
||||
and sends `session.resume` from the last acknowledged `server_event_seq`.
|
||||
Replayed events are deduplicated, consumers see one `ConnectionResumed`
|
||||
event, and `client.incarnation` / `client.resume_token` are refreshed. The
|
||||
event, and `client.resume_token` is refreshed. The
|
||||
server keeps a detached session alive for `disconnect_grace_s` (30 s by
|
||||
default); after that a resume fails with `session_resume_expired`. Resume is
|
||||
gated by `capabilities.supports_session_resume` (MiniCPM-o 4.5 and Nemotron
|
||||
@@ -238,9 +265,67 @@ definitions out of the dependency-free client. `audio_bytes(response_id)`, `coun
|
||||
`openai-realtime-duplex` benchmark backend (`vllm-omni bench serve`) is built
|
||||
on the same helpers.
|
||||
|
||||
## Using the Python API
|
||||
|
||||
`vllm_omni.entrypoints.duplex_omni.DuplexOmni` is the in-process Python API
|
||||
of a duplex model: sessions run inside the engine, and the class only opens,
|
||||
resumes and closes them and pipes typed commands in and typed events out
|
||||
through a `DuplexSessionHandle`.
|
||||
|
||||
```python
|
||||
from vllm_omni.engine.duplex.commands import AppendAudio, Commit
|
||||
from vllm_omni.engine.duplex.events import AudioDelta, SessionClosed
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
|
||||
omni = DuplexOmni(model="openbmb/MiniCPM-o-4_5", trust_remote_code=True)
|
||||
async with await omni.open_session({"modalities": ["audio", "text"], "ref_audio": ref_audio_data_url}) as session:
|
||||
async def consume():
|
||||
async for event in session.events(): # typed DuplexEvent objects
|
||||
if isinstance(event, AudioDelta):
|
||||
play(event.audio)
|
||||
elif isinstance(event, ResponseDone):
|
||||
break # one turn is enough here
|
||||
task = asyncio.create_task(consume())
|
||||
await session.submit(AppendAudio(audio=pcm16_bytes, format="pcm16", sample_rate_hz=16000))
|
||||
await session.submit(Commit())
|
||||
await task
|
||||
# leaving the block closes the session (the last event is SessionClosed)
|
||||
```
|
||||
|
||||
`open_session` takes a `DuplexSessionConfig` or the same Realtime `session`
|
||||
object the WebSocket handshake takes; the session id is always allocated by
|
||||
`DuplexOmni`. `DuplexSessionHandle` offers `submit(command)` plus one
|
||||
convenience wrapper per command (`append_audio`, `commit`, `create_response`,
|
||||
`cancel_response`, `barge_in`, `ack_playback`, `update`, `create_item`,
|
||||
`delete_item`, `truncate_item`, `heartbeat`, ...). `events()` is a
|
||||
single-consumer async iterator that ends after `session.closed` /
|
||||
`session.expired`; rejected commands come back as `ErrorEvent`s on it.
|
||||
Every event renders the wire JSON with `to_realtime()`, so anything written
|
||||
against the WebSocket protocol works unchanged on the typed stream.
|
||||
|
||||
`vllm_omni.clients.inline_duplex.InlineDuplexClient` wraps a `DuplexOmni`
|
||||
behind the `DuplexClient` API, so the same application code runs in-process
|
||||
(no server) or over the network:
|
||||
|
||||
```python
|
||||
from vllm_omni.clients.inline_duplex import InlineDuplexClient
|
||||
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
|
||||
|
||||
config = create_duplex_session_config(ref_audio=audio_data_url(Path("reference_voice.wav")))
|
||||
async with InlineDuplexClient(omni, model="openbmb/MiniCPM-o-4_5", config=config) as client:
|
||||
await client.stream_pcm(question)
|
||||
await client.commit()
|
||||
async for response in client.responses():
|
||||
...
|
||||
```
|
||||
|
||||
`examples/online_serving/barge_in_client.py --inline` drives the barge-in
|
||||
scenario this way. The inline client has no transport, so it has no
|
||||
reconnect, resume, heartbeat or event-acknowledgement machinery.
|
||||
|
||||
## API Reference
|
||||
|
||||
This section is the complete wire contract of `/v1/realtime`: how
|
||||
This section is the complete wire contract of `/v1/realtime?duplex=1`: how
|
||||
the dialect relates to the OpenAI Realtime protocol, which surfaces each
|
||||
model's capability flags gate, a catalogue of every client-to-server and
|
||||
server-to-client event, and a JSON example for each. `DuplexClient` speaks
|
||||
@@ -253,21 +338,26 @@ its protocol tests.
|
||||
|
||||
### Endpoint and transport
|
||||
|
||||
The endpoint is `ws(s)://<host>/v1/realtime`. Bare connections select the
|
||||
session handler when the deployment declares either `session_mode: duplex`
|
||||
or `duplex_session`. `?duplex=1` remains a compatibility alias, while
|
||||
`?duplex=0` selects the legacy Realtime handler. Optional query parameters
|
||||
are `model`, `session_id`,
|
||||
`autostart` (`0` means resume-only), `resume`, and `native_duplex`
|
||||
(`minicpmo45_native_duplex` is accepted as a deprecated alias and folded
|
||||
into the canonical name). Every message is one JSON object per WebSocket
|
||||
The endpoint is `ws(s)://<host>/v1/realtime`. On a deployment whose pipeline
|
||||
declares a `duplex_plugin` and whose deploy configuration declares
|
||||
`session_mode: duplex`, a bare connection is already a duplex session, so a
|
||||
stock Realtime client needs no vendor query parameter. `?duplex=1` (`1`,
|
||||
`true` or `on`) is the spelling this page uses and stays valid;
|
||||
`?duplex=0` (`0`, `false`, `off`) explicitly opts out and selects the
|
||||
turn-based Realtime handler, which a duplex server does not mount --
|
||||
that connection is answered with `Realtime API is not available`.
|
||||
`ws(s)://<host>/v1/duplex` is an alias that always selects the duplex
|
||||
handler. Optional query parameters are `model`, `autostart` (`0` means
|
||||
resume-only) and `resume`. Every message is one JSON object per WebSocket
|
||||
text frame, discriminated by `type`. Inbound events are applied in arrival
|
||||
order through one per-session mailbox; outbound events preserve that order
|
||||
and carry a monotonically increasing `server_event_seq`, which is the replay
|
||||
cursor for `session.resume`. Client events may carry an `event_id`, echoed in
|
||||
`error.error.event_id` when the event is rejected.
|
||||
|
||||
Session identity is `(session_id, incarnation)`; `attachment_generation`
|
||||
Session identity is the server-allocated `session_id` (`duplex-<uuid>`,
|
||||
announced in `session.created`; a `session_id` the client puts in the session
|
||||
object is ignored, and ids are never reused); `attachment_generation`
|
||||
names the current socket; `epoch` is the interruption fence that advances on
|
||||
every cancel, clear, or barge-in; `turn_id` counts committed model turns;
|
||||
`response_id` (item id `item_<response_id>`) names one assistant utterance.
|
||||
@@ -281,12 +371,13 @@ with no OpenAI counterpart.
|
||||
|
||||
The event vocabulary is uniform, but several surfaces are gated by the
|
||||
`capabilities` object the server returns in `session.created`; a client must
|
||||
branch on those flags rather than on the model name. The current model
|
||||
plugins advertise:
|
||||
branch on those flags rather than on the model name. MiniCPM-o 4.5 is the
|
||||
only model on the plugin contract today; the other two columns record what
|
||||
their integrations advertise once the follow-up PRs port them:
|
||||
|
||||
| Capability | MiniCPM-o 4.5 | PersonaPlex | Nemotron VoiceChat | Gated surface |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `implementation_level` | `model_native_duplex` (when `extra_body.native_duplex=true`) | `model_native_duplex` | `model_native_duplex` | model-owned `response.listen` / `response.speak`; the chat-fallback lane otherwise |
|
||||
| `implementation_level` | `model_native_duplex` | `model_native_duplex` | `model_native_duplex` | model-owned `response.listen` / `response.speak` (constant: every duplex model is model-native) |
|
||||
| `chunk_period_ms` | 1000 | 80 | 80 | the model unit that `response.listen` decisions and camera frames align to |
|
||||
| `supports_session_resume` | yes | no | yes | `session.resume`, `session.resumed`, `session.replaced`, replay after a transport drop |
|
||||
| `supports_barge_in` | yes | no | no | `barge_in`, `turn.signal{event:"barge_in"}`, `overlap_policy=barge_in_on_speech`, `turn_detection.server_vad` |
|
||||
@@ -316,14 +407,8 @@ work unmodified. The overlap falls into the three tiers used by the event catalo
|
||||
| Object shapes | `realtime.session`, `realtime.item`, `realtime.response`; item content parts `input_text` / `input_audio` / `output_text` / `output_audio`; `function_call` and `function_call_output` items; `previous_item_id` chaining; `output_index` / `content_index` addressing |
|
||||
| Sequencing | `response.created → output_item.added → content_part.added → deltas → *.done → content_part.done → output_item.done → response.done → rate_limits.updated`; `speech_started → speech_stopped → committed` |
|
||||
|
||||
Audio/transcript output uses only the current (non-beta) OpenAI event names
|
||||
(`response.output_audio.delta`, not the deprecated `response.audio.delta`),
|
||||
matching `openai.types.realtime` in the official `openai` Python SDK and what
|
||||
clients built against it (e.g. `livekit-plugins-openai`'s non-Azure code path)
|
||||
expect; a client still on the old beta SDK types would not recognize these
|
||||
events. Session fields and `conversation.item.*` events, in contrast, are
|
||||
deliberately dual: the server emits and accepts **both** the beta and the GA
|
||||
spellings at once (`output_modalities` alongside `modalities`,
|
||||
The server deliberately emits and accepts **both** the beta and the GA
|
||||
spellings at once (`response.output_audio.delta` alongside `output_modalities`,
|
||||
`conversation.item.added` alongside `conversation.item.created`) so either
|
||||
generation of OpenAI client parses the stream.
|
||||
|
||||
@@ -333,7 +418,7 @@ A stock client ignores the extra keys; the extensions are additive.
|
||||
|
||||
| Message | Extension |
|
||||
| --- | --- |
|
||||
| `session.created` / `session.updated` | top-level `incarnation`, `attachment_generation`, `resume_token`; inside `session`: `state`, `turn_state`, `epoch`, `turn_id`, `active_request_id`, `active_response_id`, `active_response_turn_id`, `overlap_policy`, `overlap_*_ms/rms`, `playback_commit_policy`, `playback`, `capabilities`, `ref_audio`, `extra_body`, `idle_timeout_s`, `response_format` |
|
||||
| `session.created` / `session.updated` | top-level `attachment_generation`, `resume_token`; inside `session`: `state`, `turn_state`, `epoch`, `turn_id`, `active_request_id`, `active_response_id`, `active_response_turn_id`, `overlap_policy`, `overlap_*_ms/rms`, `playback_commit_policy`, `playback`, `capabilities`, `ref_audio`, `extra_body`, `idle_timeout_s`, `response_format` |
|
||||
| `response.created` / `response.done` / `response.listen` | `response_id` at top level; the raw duplex event under `response.metadata` (`duplex_event`); `status_details.reason` uses vLLM reasons (`barge_in`, `client_cancelled`, `new_response`, …) |
|
||||
| `response.output_audio.delta` | `format`, `sample_rate_hz`, `metadata{session_id, epoch, model_speak, end_of_turn, audio_duration_ms, audio_text_marks, playback}` |
|
||||
| `response.speak` (inserted before the first delta) | not an OpenAI event, but rides the OpenAI response envelope (`response_id`, `item_id`, `output_index`, `content_index`) |
|
||||
@@ -349,14 +434,9 @@ Semantic divergences hidden behind shared names:
|
||||
before any commit; a commit may end in `response.listen` and no
|
||||
response, and the model may open a response with no commit at all.
|
||||
OpenAI: commit ⇒ item, `response.create` ⇒ exactly one response.
|
||||
- With `turn_detection={"type":"server_vad"}`, omitted `interrupt_response`
|
||||
defaults to `false` on turn-based models such as Qwen3-Omni, enabling
|
||||
endpointing and automatic responses without interruption. Explicit `true`
|
||||
is rejected there. Model-native duplex keeps the `true` default and rejects
|
||||
`false`; its VAD implies `overlap_policy=barge_in_on_speech`.
|
||||
This capability-dependent default differs from OpenAI's `true` default;
|
||||
the effective configuration is returned in `session.created` / `session.updated`.
|
||||
`semantic_vad` is unsupported.
|
||||
- `turn_detection.interrupt_response=false` is rejected;
|
||||
`create_response` is ignored; `semantic_vad` is unsupported; the VAD
|
||||
runs per session (Silero) and implies `overlap_policy=barge_in_on_speech`.
|
||||
- `rate_limits.updated` is always an empty list (compatibility only).
|
||||
- Cancellation never reuses a `response_id`; truncation is driven by
|
||||
`playback.ack` as well as `conversation.item.truncate`.
|
||||
@@ -365,11 +445,11 @@ Semantic divergences hidden behind shared names:
|
||||
|
||||
| Area | Messages / fields |
|
||||
| --- | --- |
|
||||
| Session lifetime | `session.heartbeat` / `session.heartbeat_ack`, `session.event_ack`, `session.close` / `session.closed`, `session.resume` / `session.resumed` / `session.replaced` / `session.expired` / `session.resync_required`; `server_event_seq`, `resume_token`, `incarnation`, `attachment_generation`; query params `?duplex=1`, `autostart`, `resume`, `native_duplex` |
|
||||
| Session lifetime | `session.heartbeat` / `session.heartbeat_ack`, `session.event_ack`, `session.close` / `session.closed`, `session.resume` / `session.resumed` / `session.replaced` / `session.expired` / `session.resync_required`; `server_event_seq`, `resume_token`, `attachment_generation`; query params `?duplex=1`, `autostart`, `resume` |
|
||||
| Turn-taking | `response.speak`, `response.listen`, `overlap.decision`, `overlap_policy`, `barge_in`, `turn.signal`, `input.cancel`, `input.text.append`, `epoch` / `turn_id` fencing, `force_listen` |
|
||||
| Playback truth | `playback.ack` / `playback.acknowledged`, `playback_commit_policy` (`ack_only` \| `commit_all_on_done`), `playback{generated_ms, sent_ms, played_ms, committed_ms}`, `history_committed`, `audio_text_marks` |
|
||||
| Model negotiation | `capabilities{implementation_level, supports_input_append, supports_barge_in, supports_session_resume, chunk_period_ms, input_modes, …}`, `ref_audio`, `extra_body` (`auto_response`, `native_duplex` — deprecated alias `minicpmo45_native_duplex` is folded into it, `force_listen_count`, `duplex_initial_user_text`) |
|
||||
| Diagnostics | `runtime.control`, `duplex.function_call.done`, the full error-code vocabulary (`REALTIME_ERROR_TYPES_BY_CODE` in `vllm_omni/entrypoints/duplex/realtime_state.py`) |
|
||||
| Model negotiation | `capabilities{implementation_level, supports_input_append, supports_barge_in, supports_session_resume, chunk_period_ms, input_modes, …}`, `ref_audio`, `extra_body` (`auto_response`, `force_listen_count`, `duplex_initial_user_text`) |
|
||||
| Diagnostics | `duplex.function_call.done`, the full error-code vocabulary (`REALTIME_ERROR_TYPES_BY_CODE` in `vllm_omni/engine/duplex/events.py`) |
|
||||
|
||||
#### Consequences for clients
|
||||
|
||||
@@ -433,11 +513,15 @@ remains an approximation, not word-level audio alignment.
|
||||
### Event catalogue
|
||||
|
||||
Every event type the Realtime route accepts or emits, with a worked example in the next section: 21
|
||||
client→server and 42 server→client. Aliases (`push_chunk`,
|
||||
`input.audio.append`, `input_text.append`, `push_text`, `signal_turn`,
|
||||
`audio.playback_ack`, `close_session`, `close`, `session_close`) share the
|
||||
payload of their canonical event and are not listed separately; the alias
|
||||
table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md).
|
||||
client→server and 42 server→client. Aliases (`input_text.append`,
|
||||
`push_text`, `signal_turn`, `audio.playback_ack`, `close_session`, `close`)
|
||||
share the payload of their canonical event and are not listed
|
||||
separately; the alias table is `translate_realtime_command` in
|
||||
`vllm_omni/engine/duplex/realtime_commands.py`. The pre-Realtime WAV-append
|
||||
aliases `push_chunk` and `input.audio.append` are **not** accepted on this
|
||||
route: send `input_audio_buffer.append` with one of the supported input
|
||||
formats (`pcm16`, `pcm_s16le`, `s16le`, `pcm_f32le`, `g711_ulaw`,
|
||||
`g711_alaw`).
|
||||
|
||||
#### Client to server
|
||||
|
||||
@@ -447,18 +531,18 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
|
||||
| `session.heartbeat` | 3 | Keepalive; refreshes the engine session lease. |
|
||||
| `session.event_ack` | 3 | Acknowledges received server events by `server_event_seq`; trims the resume replay journal. |
|
||||
| `session.close` | 3 | Graceful close; answered with `session.closed`. |
|
||||
| `session.resume` | 3 | Re-attach a live session on a new socket (`resume_token`, `incarnation`, `last_received_server_event_seq`). |
|
||||
| `session.resume` | 3 | Re-attach a live session on a new socket (`session_id`, `resume_token`, `last_received_server_event_seq`). |
|
||||
| `input_audio_buffer.append` | 2 | Append one audio chunk. OpenAI name; extended with `is_speech`, `video_frames`, `duration_ms`, `audio_end_ms`, per-event `format`/`sample_rate_hz`. |
|
||||
| `input_audio_buffer.commit` | 2 | Seal the buffered utterance into a user item. OpenAI name; extended with `final`, `response_create`, `is_speech:false`. |
|
||||
| `input.commit` | 3 | Alias of `input_audio_buffer.commit` accepted by the runner. |
|
||||
| `input_audio_buffer.clear` | 1 | Drop un-committed input audio. |
|
||||
| `input.cancel` | 3 | Cancel pending input; advances `epoch`. |
|
||||
| `input.text.append` | 3 | Append text to the open input item (chat-fallback lane only). |
|
||||
| `input.text.append` | 3 | Append text input; model-native sessions reject it with `native_text_append_unsupported`. |
|
||||
| `conversation.item.create` | 1 | Add a completed user item (text, audio, or `function_call_output`) to history. |
|
||||
| `conversation.item.retrieve` | 1 | Fetch a stored item by id. |
|
||||
| `conversation.item.delete` | 1 | Delete a stored item by id. |
|
||||
| `conversation.item.truncate` | 1 | Truncate an assistant item's audio/transcript at `audio_end_ms`. |
|
||||
| `response.create` | 1 | Explicitly request a response with per-response overrides. |
|
||||
| `response.create` | 1 | Explicitly request a response with per-response overrides. Requires committed audio input; otherwise `response_create_without_input`. |
|
||||
| `response.cancel` | 1 | Cancel the active (or named) response; advances `epoch`. |
|
||||
| `output_audio_buffer.clear` | 1 | Discard queued output audio; advances `epoch` (OpenAI: WebRTC-only, here also WebSocket). |
|
||||
| `barge_in` | 3 | Explicit hard interrupt; requires `capabilities.supports_barge_in`. |
|
||||
@@ -469,10 +553,10 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
|
||||
|
||||
| Event type | Tier | Description |
|
||||
| --- | --- | --- |
|
||||
| `session.created` | 2 | Session opened. OpenAI name; adds `incarnation`, `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`epoch`, `turn_id`, `playback`, `capabilities`, ...). |
|
||||
| `session.created` | 2 | Session opened. OpenAI name; adds `attachment_generation`, `resume_token` and vLLM-Omni keys inside `session` (`id` is the server-allocated session id, `epoch`, `turn_id`, `playback`, `capabilities`, ...). |
|
||||
| `session.updated` | 2 | Echo of the effective session config after every `session.update` (same extended `session` object). |
|
||||
| `session.heartbeat_ack` | 3 | Reply to `session.heartbeat`. |
|
||||
| `session.closed` | 3 | Last event on the socket; `reason` ∈ `session_close`, `timeout`, `disconnect`, ... |
|
||||
| `session.closed` | 3 | Last event on the socket, emitted once the engine released the session; `reason` ∈ `client_close`, `disconnect`, `timeout`, `transport_error`, `shutdown`, ... |
|
||||
| `session.resumed` | 3 | Resume accepted; carries the new `attachment_generation` and rotated `resume_token`; journaled events are replayed after it. |
|
||||
| `session.replaced` | 3 | Sent to the superseded socket when another socket resumes the session. |
|
||||
| `session.expired` | 3 | Engine lease reaped (`disconnect_grace_expired`, `idle_ttl_expired`); socket closes after it. |
|
||||
@@ -508,8 +592,8 @@ table lives in the normative contract section of [Full-Duplex Runtime (MiniCPM-o
|
||||
| `duplex.function_call.done` | 3 | Raw fallback when a model tool call is malformed. |
|
||||
| `output_audio_buffer.cleared` | 1 | Reply to `output_audio_buffer.clear`. |
|
||||
| `playback.acknowledged` | 3 | Reply to `playback.ack`; returns the playback ledger and `history_committed`. |
|
||||
| `overlap.decision` | 3 | How an append that overlapped an active response was handled (`drop` \| `listen` \| `barge_in`). |
|
||||
| `runtime.control` | 3 | Diagnostic: an engine control signal was partially unsupported. |
|
||||
| `overlap.decision` | 3 | How an append that overlapped an active response was handled (`drop` \| `listen` \| `barge_in`); details under `metadata`. |
|
||||
| `turn.event` | 3 | A turn-taking signal the session accepted (`event`, resulting `turn_state`). |
|
||||
| `error` | 2 | Error envelope. OpenAI shape and classes; vLLM-Omni error codes. |
|
||||
|
||||
### Message examples
|
||||
@@ -528,7 +612,6 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
"event_id": "evt_0001",
|
||||
"session": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"session_id": "sess_demo_01",
|
||||
"modalities": ["audio", "text"],
|
||||
"instructions": "You are a concise voice assistant.",
|
||||
"input_audio_format": "pcm16",
|
||||
@@ -541,7 +624,6 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
"idle_timeout_s": 300,
|
||||
"extra_body": {
|
||||
"auto_response": true,
|
||||
"native_duplex": true,
|
||||
"force_listen_count": 0
|
||||
}
|
||||
}
|
||||
@@ -553,13 +635,12 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
```json
|
||||
{
|
||||
"type": "session.created",
|
||||
"incarnation": 0,
|
||||
"attachment_generation": 1,
|
||||
"resume_token": "rt_3f9c…",
|
||||
"session": {
|
||||
"object": "realtime.session",
|
||||
"type": "realtime",
|
||||
"id": "sess_demo_01",
|
||||
"id": "duplex-5f2a…",
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"state": "open",
|
||||
"turn_state": "idle",
|
||||
@@ -612,7 +693,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
`session.created` and every later `session.update`)
|
||||
|
||||
```json
|
||||
{"type": "session.updated", "session": {"object": "realtime.session", "id": "sess_demo_01", "…": "…"}}
|
||||
{"type": "session.updated", "session": {"object": "realtime.session", "id": "duplex-5f2a…", "…": "…"}}
|
||||
```
|
||||
|
||||
`C→S session.heartbeat` / `S→C session.heartbeat_ack`
|
||||
@@ -622,7 +703,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
```
|
||||
|
||||
```json
|
||||
{"type": "session.heartbeat_ack", "session_id": "sess_demo_01"}
|
||||
{"type": "session.heartbeat_ack", "session_id": "duplex-5f2a…"}
|
||||
```
|
||||
|
||||
`C→S session.event_ack`
|
||||
@@ -638,7 +719,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
```
|
||||
|
||||
```json
|
||||
{"type": "session.closed", "session_id": "sess_demo_01", "reason": "session_close"}
|
||||
{"type": "session.closed", "session_id": "duplex-5f2a…", "epoch": 3, "reason": "client_close", "event": {"type": "session.closed", "reason": "client_close"}}
|
||||
```
|
||||
|
||||
`C→S session.resume` (new socket, `?duplex=1&autostart=0`)
|
||||
@@ -646,8 +727,7 @@ additionally carries `"server_event_seq": <int>` (omitted below for brevity).
|
||||
```json
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": "sess_demo_01",
|
||||
"incarnation": 0,
|
||||
"session_id": "duplex-5f2a…",
|
||||
"resume_token": "rt_3f9c…",
|
||||
"last_received_server_event_seq": 40
|
||||
}
|
||||
@@ -659,8 +739,8 @@ events with `server_event_seq > 40` are replayed in order)
|
||||
```json
|
||||
{
|
||||
"type": "session.resumed",
|
||||
"session_id": "sess_demo_01",
|
||||
"incarnation": 0,
|
||||
"session_id": "duplex-5f2a…",
|
||||
"session": {"id": "duplex-5f2a…", "state": "open", "epoch": 3, "turn_id": 2, "...": "the engine's current session object"},
|
||||
"attachment_generation": 2,
|
||||
"resume_token": "rt_a71e…"
|
||||
}
|
||||
@@ -669,19 +749,19 @@ events with `server_event_seq > 40` are replayed in order)
|
||||
`S→C session.replaced` (sent to the superseded socket, which is then closed)
|
||||
|
||||
```json
|
||||
{"type": "session.replaced", "session_id": "sess_demo_01", "attachment_generation": 1}
|
||||
{"type": "session.replaced", "session_id": "duplex-5f2a…", "attachment_generation": 1}
|
||||
```
|
||||
|
||||
`S→C session.expired`
|
||||
|
||||
```json
|
||||
{"type": "session.expired", "session_id": "sess_demo_01", "incarnation": 0, "reason": "disconnect_grace_expired"}
|
||||
{"type": "session.expired", "session_id": "duplex-5f2a…", "reason": "disconnect_grace_expired"}
|
||||
```
|
||||
|
||||
`S→C session.resync_required`
|
||||
|
||||
```json
|
||||
{"type": "session.resync_required", "session_id": "sess_demo_01", "reason": "journal_gap"}
|
||||
{"type": "session.resync_required", "session_id": "duplex-5f2a…", "reason": "journal_gap"}
|
||||
```
|
||||
|
||||
#### Input
|
||||
@@ -772,7 +852,7 @@ silence/noise declaration (no runtime append; answered with `response.listen`)
|
||||
"item_id": "item_9b2d…",
|
||||
"event": {
|
||||
"type": "input.committed",
|
||||
"session_id": "sess_demo_01",
|
||||
"session_id": "duplex-5f2a…",
|
||||
"turn_id": 1,
|
||||
"epoch": 0,
|
||||
"history_len": 1,
|
||||
@@ -802,7 +882,7 @@ deferred commit during an active response (`event.response_create_deferred`)
|
||||
"type": "input_audio_buffer.committed",
|
||||
"previous_item_id": "item_9b2d…",
|
||||
"item_id": "item_c044…",
|
||||
"event": {"type": "input.committed", "session_id": "sess_demo_01", "turn_id": 2, "epoch": 0, "history_len": 3, "response_create_deferred": true, "message": {"…": "…"}}
|
||||
"event": {"type": "input.committed", "session_id": "duplex-5f2a…", "turn_id": 2, "epoch": 0, "history_len": 3, "response_create_deferred": true, "message": {"…": "…"}}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -911,7 +991,7 @@ deferred commit during an active response (`event.response_create_deferred`)
|
||||
"status_details": null,
|
||||
"output": [],
|
||||
"modalities": ["audio", "text"],
|
||||
"metadata": {"duplex_event": {"type": "response.created", "session_id": "sess_demo_01", "response_id": "resp_01", "epoch": 0, "turn_id": 1}}
|
||||
"metadata": {"duplex_event": {"type": "response.created", "session_id": "duplex-5f2a…", "response_id": "resp_01", "epoch": 0, "turn_id": 1}}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -947,7 +1027,7 @@ deferred commit during an active response (`event.response_create_deferred`)
|
||||
"item_id": "item_resp_01",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"metadata": {"session_id": "sess_demo_01", "epoch": 0, "model_speak": true}
|
||||
"metadata": {"session_id": "duplex-5f2a…", "epoch": 0, "model_speak": true}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -964,7 +1044,7 @@ deferred commit during an active response (`event.response_create_deferred`)
|
||||
"format": "pcm16",
|
||||
"sample_rate_hz": 24000,
|
||||
"metadata": {
|
||||
"session_id": "sess_demo_01",
|
||||
"session_id": "duplex-5f2a…",
|
||||
"epoch": 0,
|
||||
"model_speak": true,
|
||||
"end_of_turn": false,
|
||||
@@ -1029,7 +1109,7 @@ deferred commit during an active response (`event.response_create_deferred`)
|
||||
"status": "completed",
|
||||
"status_details": null,
|
||||
"output": [{"id": "item_resp_01", "object": "realtime.item", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_audio", "transcript": "It is sunny, 24 degrees."}]}],
|
||||
"metadata": {"type": "response.done", "session_id": "sess_demo_01", "response_id": "resp_01", "epoch": 0, "committed": true, "playback": {"generated_ms": 2400, "sent_ms": 2400, "played_ms": 2400, "committed_ms": 2400}}
|
||||
"metadata": {"type": "response.done", "session_id": "duplex-5f2a…", "response_id": "resp_01", "epoch": 0, "committed": true, "playback": {"generated_ms": 2400, "sent_ms": 2400, "played_ms": 2400, "committed_ms": 2400}}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -1055,9 +1135,9 @@ deferred commit during an active response (`event.response_create_deferred`)
|
||||
```json
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sess_demo_01",
|
||||
"session_id": "duplex-5f2a…",
|
||||
"epoch": 0,
|
||||
"response": {"object": "realtime.response", "status": "listening", "metadata": {"type": "response.listen", "session_id": "sess_demo_01", "epoch": 0, "reason": "model_listen"}}
|
||||
"response": {"object": "realtime.response", "status": "listening", "metadata": {"type": "response.listen", "session_id": "duplex-5f2a…", "epoch": 0, "reason": "model_listen"}}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -1104,7 +1184,7 @@ function-call response (Nemotron VoiceChat tools)
|
||||
raw fallback when a model tool call is malformed (`call_id`/`name` missing)
|
||||
|
||||
```json
|
||||
{"type": "duplex.function_call.done", "event": {"type": "function_call.done", "session_id": "sess_demo_01", "epoch": 0, "call_id": null, "name": null, "arguments": "{"}}
|
||||
{"type": "duplex.function_call.done", "event": {"type": "function_call.done", "session_id": "duplex-5f2a…", "epoch": 0, "call_id": null, "name": null, "arguments": "{"}}
|
||||
```
|
||||
|
||||
#### Playback
|
||||
@@ -1120,7 +1200,7 @@ raw fallback when a model tool call is malformed (`call_id`/`name` missing)
|
||||
"type": "playback.acknowledged",
|
||||
"event": {
|
||||
"type": "playback.acknowledged",
|
||||
"session_id": "sess_demo_01",
|
||||
"session_id": "duplex-5f2a…",
|
||||
"epoch": 0,
|
||||
"item_id": "item_resp_01",
|
||||
"played_ms": 1000,
|
||||
@@ -1145,20 +1225,17 @@ truncating ack after an interruption (what was actually heard)
|
||||
```json
|
||||
{
|
||||
"type": "overlap.decision",
|
||||
"session_id": "sess_demo_01",
|
||||
"session_id": "duplex-5f2a…",
|
||||
"epoch": 0,
|
||||
"policy": "listen_only",
|
||||
"action": "listen",
|
||||
"reason": "long_overlap_speech",
|
||||
"overlap_ms": 1400,
|
||||
"buffer_audio": true,
|
||||
"defer_runtime_append": true,
|
||||
"force_listen": true
|
||||
"metadata": {"overlap_ms": 1400, "buffer_audio": true, "defer_runtime_append": true, "force_listen": true}
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
{"type": "overlap.decision", "session_id": "sess_demo_01", "epoch": 0, "policy": "barge_in_on_speech", "action": "barge_in", "reason": "server_vad_utterance_active"}
|
||||
{"type": "overlap.decision", "session_id": "duplex-5f2a…", "epoch": 0, "policy": "barge_in_on_speech", "action": "barge_in", "reason": "server_vad_utterance_active"}
|
||||
```
|
||||
|
||||
server-VAD session (`session.update` fragment) and the annotated append the
|
||||
@@ -1172,12 +1249,6 @@ translator forwards internally (shown for reference; clients never see it)
|
||||
{"type": "input_audio_buffer.append", "audio": "…", "format": "pcm16", "sample_rate_hz": 16000, "is_speech": true, "force_listen": true, "vad": {"backend": "silero", "is_speech": true, "speech_active": true, "speech_started": true, "speech_stopped": false, "speech_probability": 0.93}}
|
||||
```
|
||||
|
||||
`S→C runtime.control` (only when an engine signal was partially unsupported)
|
||||
|
||||
```json
|
||||
{"type": "runtime.control", "session_id": "sess_demo_01", "epoch": 1, "result": {"…": "redacted engine control result"}}
|
||||
```
|
||||
|
||||
#### Errors
|
||||
|
||||
`S→C error` — Realtime shape
|
||||
@@ -1203,71 +1274,79 @@ translator forwards internally (shown for reference; clients never see it)
|
||||
```
|
||||
|
||||
```json
|
||||
{"type": "error", "error": {"type": "rate_limit_error", "code": "duplex_session_busy", "message": "duplex session capacity reached (max_sessions=4)"}}
|
||||
{"type": "error", "error": {"type": "rate_limit_error", "code": "resource_exhausted", "message": "duplex_session_capacity_exhausted: limit=4"}}
|
||||
```
|
||||
|
||||
Handshake-stage errors (before the Realtime projector is bound) use the
|
||||
flat native shape:
|
||||
|
||||
```json
|
||||
{"type": "error", "error": "Unknown or expired duplex session: sess_demo_01", "code": "session_resume_expired"}
|
||||
{"type": "error", "error": "Unknown or expired duplex session: duplex-5f2a…", "code": "session_resume_expired"}
|
||||
```
|
||||
|
||||
#### Coverage
|
||||
|
||||
The examples above cover every event type the Realtime route emits or
|
||||
accepts (verified by diffing the `type` literals in
|
||||
`vllm_omni/entrypoints/duplex/` against this document). Not shown, because
|
||||
they never reach a `/v1/realtime?duplex=1` client unprojected: the native
|
||||
`/v1/duplex` dialect (`session.create` / `open_session` / `session.config`,
|
||||
`input.committed`, `input.cancelled`, `audio.cancelled`,
|
||||
`response.output_audio.delta`, `response.text.delta`, `response.message`,
|
||||
`function_call.done`) — their Realtime projections are the examples above
|
||||
(see the name map at the end of this section) — and the runner-internal markers `__timeout__`,
|
||||
`__disconnect__`, `__replaced_attachment__`. Input aliases (`push_chunk`,
|
||||
`input.audio.append`, `input_text.append`, `push_text`, `signal_turn`,
|
||||
`audio.playback_ack`, `close_session`, `close`, `session_close`) share the
|
||||
payload of their canonical event (see the alias table in [Full-Duplex Runtime (MiniCPM-o 4.5)](../design/fullduplex.md)).
|
||||
`vllm_omni/engine/duplex/events.py` and `vllm_omni/engine/duplex/commands.py`
|
||||
against this document). Not shown, because they never reach a client
|
||||
unprojected: the session-internal events the runner produces
|
||||
(`input.committed`, `input.cancelled`, `audio.cancelled`,
|
||||
`response.output_audio.delta`, `response.text.delta`, `function_call.done`)
|
||||
— their Realtime projections are the examples above (see the name map at the
|
||||
end of this section). Every public event is a typed `DuplexEvent` whose
|
||||
`to_realtime()` is the wire object; every client event is parsed into a
|
||||
typed `DuplexCommand` by `command_from_realtime()`.
|
||||
|
||||
#### Native to Realtime name map
|
||||
#### Internal to Realtime name map
|
||||
|
||||
The native `/v1/duplex` dialect is what the session runner produces; the
|
||||
Realtime projector renames and fans out these events before they reach a
|
||||
`/v1/realtime?duplex=1` client.
|
||||
The session runner emits these session-internal events; the Realtime
|
||||
projector (`vllm_omni/engine/duplex/realtime_events.py`) renames and fans
|
||||
them out into typed events before they reach a client.
|
||||
|
||||
| Native / internal event | Realtime projection |
|
||||
| Internal event | Realtime projection |
|
||||
| --- | --- |
|
||||
| `session.create` (`open_session`, `session.config`) | first `session.update` |
|
||||
| `input.committed` | `conversation.item.added` / `conversation.item.created`, `input_audio_buffer.committed`, `conversation.item.input_audio_transcription.completed`, `conversation.item.done` |
|
||||
| `input.cancelled` | `input_audio_buffer.cleared` |
|
||||
| `response.output_audio.delta` (with `audio_transcript`) | optional `response.speak`, then `response.output_audio.delta` and `response.output_audio_transcript.delta` |
|
||||
| `response.output_audio.done` / `response.output_text.done` | `response.output_audio.done` / `response.output_text.done` |
|
||||
| `response.text.delta` | `response.output_text.delta` |
|
||||
| `response.message` (chat-fallback raw chunk) | passed through |
|
||||
| `audio.cancelled` | optional `output_audio_buffer.cleared`, then the cancelled terminal events and `response.done` with `status: "cancelled"` |
|
||||
| `function_call.done` | `response.created`, `response.output_item.added` (`function_call` item), `response.function_call_arguments.delta` / `.done`, `response.output_item.done`, `response.done` |
|
||||
| `playback.acknowledged` | `playback.acknowledged` (wrapped verbatim under `event`) |
|
||||
| `runtime.control`, `session.resumed`, `session.replaced`, `session.expired`, `session.resync_required`, `session.heartbeat_ack` | passed through unchanged |
|
||||
| `session.resumed`, `session.replaced`, `session.expired`, `session.resync_required`, `session.heartbeat_ack` | passed through unchanged |
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- Only MiniCPM-o 4.5 is served over this endpoint today; PersonaPlex and
|
||||
Nemotron VoiceChat arrive with the follow-up PRs that port them to the
|
||||
plugin contract.
|
||||
- Several surfaces are capability-gated per model (see *Capability
|
||||
negotiation by model* above): PersonaPlex does not support session resume,
|
||||
barge-in, or audio truncation; Nemotron VoiceChat does not support barge-in
|
||||
or audio truncation; camera frames are consumed only by MiniCPM-o 4.5; tool
|
||||
calls are produced only by Nemotron VoiceChat.
|
||||
- `turn_detection` supports `server_vad` or `null`, not `semantic_vad`.
|
||||
Turn-based models support endpointing without interruption, including
|
||||
`create_response=false` for explicit response creation. Model-native duplex
|
||||
requires `interrupt_response=true` and `create_response=true`.
|
||||
- A response is generated from committed **audio**, so the OpenAI text-prompt
|
||||
shape does not drive one: `conversation.item.create` with an `input_text`
|
||||
part adds the item to history, but a following `response.create` with no
|
||||
committed audio is rejected with `response_create_without_input`, and
|
||||
`input.text.append` is rejected with `native_text_append_unsupported`.
|
||||
Text-to-speech has its own shape here — put the text in
|
||||
`extra_body.duplex_initial_user_text` on `session.update`, then stream audio
|
||||
units (silence is enough). The seeded turn is what the model answers.
|
||||
- `turn_detection` supports only `server_vad` with `interrupt_response=true`;
|
||||
`semantic_vad`, `interrupt_response=false`, and `create_response` are not
|
||||
supported.
|
||||
- `input_audio_transcription` and `input_audio_noise_reduction` are accepted
|
||||
and echoed but no separate transcription or noise-reduction stage runs;
|
||||
transcripts come from the model.
|
||||
- `rate_limits.updated` is emitted for compatibility only and always carries
|
||||
an empty list.
|
||||
- Session capacity is bounded by `duplex_session.max_sessions` in the deploy
|
||||
configuration; admission beyond it fails with `duplex_session_busy` or
|
||||
`resource_exhausted`.
|
||||
- The native `/v1/duplex` route speaks the internal dialect and does not
|
||||
support `session.resume`; use `/v1/realtime?duplex=1` for the contract on
|
||||
this page.
|
||||
configuration; admission beyond it fails with `resource_exhausted`
|
||||
(`rate_limit_error`, retryable).
|
||||
- A duplex deployment serves the duplex session surfaces only: the websocket
|
||||
route and `/v1/chat/completions`, the latter on a session per request (see
|
||||
[Start the Server](#start-the-server)). The other turn-based HTTP routes
|
||||
report "not available" on a MiniCPM-o 4.5 server; turn-based use of the
|
||||
model stays available offline through `Omni` / `AsyncOmni`.
|
||||
@@ -14,7 +14,10 @@ Scenario (see ``barge_in_client_flow.md`` for the diagram)
|
||||
|
||||
The demo is model-neutral: it drives any duplex model through the public
|
||||
:class:`vllm_omni.clients.duplex.DuplexClient` over
|
||||
``/v1/realtime?duplex=1``. Model-specific session shape comes from the
|
||||
``/v1/realtime?duplex=1``, or — with ``--inline`` — through
|
||||
:class:`vllm_omni.clients.inline_duplex.InlineDuplexClient` on an in-process
|
||||
:class:`vllm_omni.entrypoints.duplex_omni.DuplexOmni` (no server needed; the
|
||||
model loads in this process). Model-specific session shape comes from the
|
||||
per-model presets under ``vllm_omni.clients.<model>`` selected with
|
||||
``--preset`` (default ``minicpmo_4_5``, whose model decides listen/speak on
|
||||
its own — no VAD).
|
||||
@@ -31,6 +34,15 @@ then:
|
||||
--interrupt-wav follow_up_16k.wav \
|
||||
--output-dir ./duplex_out
|
||||
|
||||
or, without a server, in-process (``--deploy-config`` is optional and defaults
|
||||
to the model's bundled duplex profile):
|
||||
|
||||
python barge_in_client.py --inline \
|
||||
--model /path/to/MiniCPM-o-4_5 \
|
||||
--ref-audio /path/to/reference_voice.wav \
|
||||
--question-wav question_16k.wav \
|
||||
--interrupt-wav follow_up_16k.wav
|
||||
|
||||
Input WAVs must be mono 16 kHz PCM16; when a preset's session takes a
|
||||
different capture format (the PersonaPlex preset streams 24 kHz float32),
|
||||
the audio is converted before streaming. ``--ref-audio`` is required by the
|
||||
@@ -55,6 +67,7 @@ if str(REPO_ROOT) not in sys.path:
|
||||
from vllm_omni.clients.duplex import ( # noqa: E402
|
||||
AudioFormat,
|
||||
DuplexClient,
|
||||
DuplexClientBase,
|
||||
EventCollector,
|
||||
SessionConfig,
|
||||
acknowledge_collected_playback,
|
||||
@@ -139,6 +152,39 @@ def _fold_responses(collector: EventCollector) -> dict[str, dict[str, object]]:
|
||||
return out
|
||||
|
||||
|
||||
def _make_client(args: argparse.Namespace, config: SessionConfig) -> DuplexClientBase:
|
||||
"""Websocket client by default; ``--inline`` drives an in-process DuplexOmni."""
|
||||
if not args.inline:
|
||||
return DuplexClient(
|
||||
args.url,
|
||||
model=args.model,
|
||||
config=config,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
)
|
||||
# Imported lazily: the in-process path loads the model (and its engine
|
||||
# dependencies) in this process, which the websocket path never needs.
|
||||
from vllm_omni.clients.inline_duplex import InlineDuplexClient
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
|
||||
# Models shipped with custom HF code (MiniCPM-o) need trust_remote_code to
|
||||
# resolve their pipeline; harmless for the others.
|
||||
omni_kwargs: dict[str, object] = {"model": args.model, "trust_remote_code": True}
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
omni = DuplexOmni(**omni_kwargs)
|
||||
client = InlineDuplexClient(
|
||||
omni,
|
||||
model=args.model,
|
||||
config=config,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
)
|
||||
# The client owns no engine; stop the in-process one when the run ends.
|
||||
client.owned_omni = omni # type: ignore[attr-defined]
|
||||
return client
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
config = _session_config(args.preset, ref_audio_path=args.ref_audio)
|
||||
question = _convert_to_session_format(read_pcm16_wav(Path(args.question_wav)), config.input_audio)
|
||||
@@ -148,15 +194,9 @@ async def run(args: argparse.Namespace) -> int:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
collector = EventCollector()
|
||||
client = DuplexClient(
|
||||
args.url,
|
||||
model=args.model,
|
||||
config=config,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
)
|
||||
client = _make_client(args, config)
|
||||
# GREET: open the voice session (handshake happens on enter).
|
||||
try:
|
||||
async with client:
|
||||
consume_task = asyncio.create_task(collector.consume(client))
|
||||
|
||||
@@ -240,13 +280,31 @@ async def run(args: argparse.Namespace) -> int:
|
||||
f"\n{len(summary)} responses ({cancelled} interrupted); outputs in {output_dir}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
finally:
|
||||
owned_omni = getattr(client, "owned_omni", None)
|
||||
if owned_omni is not None:
|
||||
owned_omni.shutdown()
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:8099/v1/realtime")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="openbmb/MiniCPM-o-4_5",
|
||||
help="served model name, or the model path with --inline",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--inline",
|
||||
action="store_true",
|
||||
help="run the model in this process through DuplexOmni + InlineDuplexClient instead of connecting to --url",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
default=None,
|
||||
help="deploy YAML for --inline (default: the model's bundled profile)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preset",
|
||||
choices=["minicpmo_4_5", "personaplex", "none"],
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
# `barge_in_client.py` — what happens, at a glance
|
||||
|
||||
The demo plays one everyday conversation pattern against the full-duplex
|
||||
server: ask something, get talked-over mid-answer, and see the assistant
|
||||
cope. High-level flow below — for the exact wire events, read the
|
||||
The demo plays one everyday conversation pattern against a full-duplex
|
||||
model: ask something, get talked-over mid-answer, and see the assistant
|
||||
cope. It runs either against a server over `/v1/realtime?duplex=1` or, with
|
||||
`--inline`, against an in-process `DuplexOmni` — the conversation below is the
|
||||
same in both modes. High-level flow — for the exact wire events, read the
|
||||
docstring in `barge_in_client.py`.
|
||||
|
||||
```mermaid
|
||||
@@ -48,7 +50,8 @@ answer is saved as `response_1_cancelled.wav`, a completed one as
|
||||
duration per answer.
|
||||
|
||||
Companion: the runtime architecture lives in
|
||||
[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client
|
||||
API is `vllm_omni.clients.duplex.DuplexClient`, documented with the wire
|
||||
protocol in
|
||||
[`docs/design/fullduplex.md`](../../docs/design/fullduplex.md); the client API is
|
||||
`vllm_omni.clients.duplex.DuplexClient` over the websocket and
|
||||
`vllm_omni.clients.inline_duplex.InlineDuplexClient` in-process — both are
|
||||
documented with the wire protocol in
|
||||
[`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md).
|
||||
@@ -5,10 +5,10 @@
|
||||
# Server side for barge_in_client.py: start MiniCPM-o 4.5 in native
|
||||
# full-duplex mode.
|
||||
#
|
||||
# The duplex deploy profile registers the WS /v1/duplex endpoint (with
|
||||
# /v1/realtime?duplex=1 projecting onto the same handler), enables the
|
||||
# engine duplex control plane, and bounds the Thinker/Talker/Code2Wav
|
||||
# stages to four live duplex sessions on one GPU. See
|
||||
# MiniCPM-o 4.5 declares a duplex plugin, so vllm-omni serve runs it through
|
||||
# DuplexOmni: the server is duplex-only (WS /v1/realtime?duplex=1, alias
|
||||
# /v1/duplex, plus /v1/models and /health) and bounds the
|
||||
# Thinker/Talker/Code2Wav stages to four live duplex sessions on one GPU. See
|
||||
# vllm_omni/deploy/minicpmo_4_5.yaml (session_mode: duplex) for the session
|
||||
# limits; every other duplex_session knob keeps its runtime default
|
||||
# (idle TTL 300s, disconnect grace 30s, 16MiB pending input per session).
|
||||
@@ -22,4 +22,4 @@ exec vllm-omni serve "$MODEL" \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \
|
||||
--trust-remote-code \
|
||||
--host 0.0.0.0 --port "$PORT"
|
||||
# duplex endpoints: ws://<host>:$PORT/v1/duplex and /v1/realtime?duplex=1
|
||||
# duplex endpoint: ws://<host>:$PORT/v1/realtime?duplex=1 (alias /v1/duplex)
|
||||
@@ -445,7 +445,6 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
ref_audio=_ref_audio_data_url(args.ref_audio),
|
||||
temperature=args.temperature,
|
||||
),
|
||||
session_id=args.session_id,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=args.timeout_s,
|
||||
@@ -638,7 +637,6 @@ def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--session-id")
|
||||
parser.add_argument(
|
||||
"--input-wav",
|
||||
help="Mono 16 kHz PCM16 clip to stream. Optional when --input-video carries the audio track.",
|
||||
|
||||
@@ -86,7 +86,6 @@
|
||||
url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
url.searchParams.set('duplex', '1');
|
||||
url.searchParams.set('model', config.model || 'openbmb/MiniCPM-o-4_5');
|
||||
url.searchParams.set('native_duplex', '1');
|
||||
url.searchParams.set('autostart', '0');
|
||||
return url.toString();
|
||||
}
|
||||
@@ -458,7 +457,6 @@
|
||||
settled = true;
|
||||
const extraBody = {
|
||||
auto_response: true,
|
||||
native_duplex: true,
|
||||
};
|
||||
const session = {
|
||||
modalities: ['audio', 'text'],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# MiniCPM-o 4.5
|
||||
|
||||
> Online serving and offline inference for omni multimodal chat
|
||||
> Online full-duplex and chat-completions serving, and offline inference, for omni multimodal chat
|
||||
> (text / image / audio / video → text + 24 kHz speech)
|
||||
|
||||
## Summary
|
||||
@@ -9,8 +9,12 @@
|
||||
- Model: [`openbmb/MiniCPM-o-4_5`](https://huggingface.co/openbmb/MiniCPM-o-4_5)
|
||||
- Task: Omni multimodal chat — accepts text / image / audio / video input;
|
||||
emits text and 24 kHz mono speech in the same response
|
||||
- Mode: Online serving via the OpenAI-compatible `/v1/chat/completions`
|
||||
API (plus Gradio demo), and offline inference via `Omni.generate`
|
||||
- Mode: Online full-duplex serving over the OpenAI Realtime protocol
|
||||
(`/v1/realtime?duplex=1`, alias `/v1/duplex`), the OpenAI-compatible
|
||||
`/v1/chat/completions` API (plus Gradio demo) on the same server, and
|
||||
offline turn-based inference via `Omni.generate`. Besides those the server
|
||||
exposes only `/v1/models` and `/health`; the other turn-based HTTP routes
|
||||
are not available on a duplex deployment.
|
||||
- Maintainer: [`@tc-mb`](https://github.com/tc-mb) (MiniCPM-V / MiniCPM-o team)
|
||||
|
||||
## When to use this recipe
|
||||
@@ -18,9 +22,10 @@
|
||||
Use this recipe as a known-good starting point for serving
|
||||
`openbmb/MiniCPM-o-4_5` on vLLM-Omni. MiniCPM-o 4.5 is the omni member
|
||||
of the MiniCPM-o family — it runs a multimodal thinker, a streaming
|
||||
MiniCPMTTS codec talker, and a separate batched Code2Wav stage so a single
|
||||
`/v1/chat/completions` call can return text and 24 kHz speech in one
|
||||
shot. The recommended batching deploy isolates the Thinker on GPU 0 and
|
||||
MiniCPMTTS codec talker, and a separate batched Code2Wav stage, so a live
|
||||
duplex session streams text and 24 kHz speech while it keeps listening, and a
|
||||
single `/v1/chat/completions` call returns both in one shot. The
|
||||
recommended batching deploy isolates the Thinker on GPU 0 and
|
||||
co-locates Talker and Code2Wav on GPU 1; 1-GPU, 3-GPU, and 8x4090 layouts are
|
||||
also provided.
|
||||
|
||||
@@ -36,8 +41,11 @@ also provided.
|
||||
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
||||
- 8x RTX 4090 layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
||||
- Online example + Gradio demo:
|
||||
- Online examples (Realtime CLI demo, browser client, barge-in client,
|
||||
chat-completions client, curl script, Gradio demo):
|
||||
[`examples/online_serving/minicpmo/`](../../examples/online_serving/minicpmo/)
|
||||
- Duplex API and Python client docs:
|
||||
[`docs/serving/realtime_duplex_api.md`](../../docs/serving/realtime_duplex_api.md)
|
||||
- Offline end-to-end example:
|
||||
[`examples/offline_inference/minicpmo/`](../../examples/offline_inference/minicpmo/)
|
||||
- Pipeline / talker source:
|
||||
@@ -179,6 +187,13 @@ as one aggregated chunk, so the table reports Stage 0 metrics.
|
||||
|
||||
#### Verification
|
||||
|
||||
**Health and model listing**:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8099/health
|
||||
curl http://localhost:8099/v1/models
|
||||
```
|
||||
|
||||
**Quick smoke test (text-only output)**:
|
||||
|
||||
```bash
|
||||
@@ -191,8 +206,31 @@ curl http://localhost:8099/v1/chat/completions \
|
||||
}'
|
||||
```
|
||||
|
||||
**Text + speech in one response** (the headline 4.5 feature). The model
|
||||
bridge conditions the Talker from the generated assistant span, so the
|
||||
**One duplex turn over the Realtime WebSocket** (text + speech stream back
|
||||
while the session keeps listening):
|
||||
|
||||
```bash
|
||||
python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--url ws://localhost:8099/v1/realtime?duplex=1 \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--input-wav /path/to/input_16k_mono_pcm16.wav \
|
||||
--ref-audio /path/to/MiniCPM-o-Demo/assets/ref_audio/ref_minicpm_signature.wav \
|
||||
--output-dir /tmp/minicpmo_realtime_duplex_demo
|
||||
```
|
||||
|
||||
The demo prints transcript deltas as they arrive and saves the streamed
|
||||
24 kHz audio to WAV files. The same session shape is available from Python
|
||||
through `vllm_omni.clients.duplex.DuplexClient` (over the server) or
|
||||
`vllm_omni.clients.inline_duplex.InlineDuplexClient` (in-process, no server;
|
||||
`examples/online_serving/barge_in_client.py --inline`).
|
||||
|
||||
**Barge-in**: `examples/online_serving/barge_in_client.py` plays a question,
|
||||
interrupts the answer with a second WAV and checks that the first response is
|
||||
cancelled and the second one is answered.
|
||||
|
||||
**Text + speech in one response over `/v1/chat/completions`** (the headline
|
||||
4.5 feature, served by the ordinary chat service on the duplex engine). The
|
||||
model bridge conditions the Talker from the generated assistant span, so the
|
||||
generic serving layer does not inject MiniCPM-specific template defaults.
|
||||
`use_tts_template=true` remains supported when explicitly requested:
|
||||
|
||||
@@ -243,6 +281,12 @@ python examples/online_serving/minicpmo/gradio_demo.py \
|
||||
Open `http://<host>:7862` and try a text prompt with the **"Generate
|
||||
speech output (TTS)"** checkbox on / off.
|
||||
|
||||
**Turn-based generation** is also available offline
|
||||
(`examples/offline_inference/minicpmo/`, `Omni.generate`).
|
||||
|
||||
The Daily-Omni numbers above were measured on the `/v1/chat/completions`
|
||||
route.
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.15, and
|
||||
@@ -326,9 +370,13 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
and defaults to deterministic seed 42. Stage-1 deploy sampling parameters
|
||||
control only vLLM's binary continue/stop token.
|
||||
|
||||
- **Output audio**: 24 kHz mono WAV inside the OpenAI-style
|
||||
`message.audio.data` (base64). The Gradio demo's WAV player decodes
|
||||
this automatically.
|
||||
- **Output audio**: 24 kHz mono. On `/v1/chat/completions` it is base64 WAV
|
||||
inside the OpenAI-style `message.audio.data` (the Gradio demo's WAV player
|
||||
decodes this automatically); on a duplex session it streams as base64
|
||||
`response.audio.delta` events; offline `Omni.generate` returns WAV.
|
||||
- **Response choices**: on `/v1/chat/completions` text and audio are separate
|
||||
choices. SDK clients should select the choice whose `message.audio.data` is
|
||||
populated rather than assuming `choices[0]` contains audio.
|
||||
|
||||
- **Routing**: MiniCPM-o 4.5 and 2.6 both ship `architectures=
|
||||
["MiniCPMO"]` in HF config; routing is disambiguated by
|
||||
@@ -340,6 +388,3 @@ vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
- **Async chunking**: enabled in all deploy configs. Talker sends
|
||||
25-code chunks with three-code left context to Code2Wav through
|
||||
`SharedMemoryConnector`; terminal chunks flush held lookahead state.
|
||||
- **Response choices**: text and audio are separate choices. SDK clients
|
||||
should select the choice whose `message.audio.data` is populated rather
|
||||
than assuming `choices[0]` contains audio.
|
||||
+1
-1
@@ -71,7 +71,7 @@ recipes/
|
||||
| [`cosmos3/Cosmos3-Nano.md`](./cosmos3/Cosmos3-Nano.md) | Text-to-image, text-to-video, image-to-video, video-to-video generation, text to video with sound, action policy | 1x H200 141GB / B300 |
|
||||
| [`cosmos3/Cosmos3-Edge.md`](./cosmos3/Cosmos3-Edge.md) | T2I / T2V / I2V + action (Physical-AI) generation on the Nemotron-based Edge transformer | 1x GPU (~8 GiB) |
|
||||
| [`cosmos3/Cosmos3-Super.md`](./cosmos3/Cosmos3-Super.md) | 64B T2I / T2V / I2V / V2V generation (+ optional audio) / Action policy | 8x H200/H100/A100 / 2x H200 / B300 |
|
||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online full-duplex serving and offline inference for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||
| [`OpenMOSS/MOSS-TTS.md`](./OpenMOSS/MOSS-TTS.md) | Online + offline multilingual TTS (MOSS-TTS family, 8B) | 1x H100 80GB |
|
||||
| [`OpenMOSS/MOSS-TTS-Nano-NPU.md`](./OpenMOSS/MOSS-TTS-Nano-NPU.md) | Online multilingual voice-cloned TTS | 1x Ascend 910B3 64GB |
|
||||
|
||||
@@ -135,6 +135,7 @@ class MockResponse:
|
||||
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
|
||||
class FakeRealtimeClient:
|
||||
last_instance = None
|
||||
instances: list = []
|
||||
|
||||
def __init__(self, url):
|
||||
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
|
||||
@@ -143,6 +144,9 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
|
||||
self.sent = []
|
||||
self.response_count = 0
|
||||
self.ack_count = 0
|
||||
self.silence_seconds = []
|
||||
self.closed = 0
|
||||
type(self).instances.append(self)
|
||||
type(self).last_instance = self
|
||||
|
||||
async def __aenter__(self):
|
||||
@@ -155,10 +159,22 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
|
||||
assert model == "openbmb/MiniCPM-o-4_5"
|
||||
self.configure_kwargs = kwargs
|
||||
|
||||
async def stream_silence(self, *, seconds, chunk_ms=200, until=None):
|
||||
# A model-native session speaks off its seeded context once audio
|
||||
# units arrive; the silence itself carries no content. The probe
|
||||
# stops the silence as soon as the turn settles.
|
||||
self.silence_seconds.append(seconds)
|
||||
self._emit_response()
|
||||
assert until is not None and until()
|
||||
return 1.0
|
||||
|
||||
async def send(self, event):
|
||||
self.sent.append(event)
|
||||
if event["type"] != "response.create":
|
||||
return
|
||||
self._emit_response()
|
||||
|
||||
def _emit_response(self):
|
||||
self.response_count += 1
|
||||
response_id = f"resp-{self.response_count}"
|
||||
now = time.monotonic()
|
||||
@@ -210,7 +226,7 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
|
||||
self.ack_count += 1
|
||||
|
||||
async def close_session(self, **_kwargs):
|
||||
return None
|
||||
self.closed += 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.patch.patch._RealtimeTTSProbe",
|
||||
@@ -239,30 +255,27 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
|
||||
session=None,
|
||||
)
|
||||
|
||||
client = FakeRealtimeClient.last_instance
|
||||
assert client.configure_kwargs["native_duplex"] is False
|
||||
# One session per utterance: a model-native duplex session takes its text
|
||||
# once, in the session context, so it cannot be re-seeded for a second one.
|
||||
sessions = FakeRealtimeClient.instances
|
||||
assert len(sessions) == 4
|
||||
for index, client in enumerate(sessions):
|
||||
assert "native_duplex" not in client.configure_kwargs
|
||||
assert client.configure_kwargs["extra_body"] == {
|
||||
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||
"return_stage_metrics": True,
|
||||
"duplex_initial_user_text": f"text {index}",
|
||||
"force_listen_count": 0,
|
||||
}
|
||||
assert client.sent == [
|
||||
event
|
||||
for index in range(4)
|
||||
for event in (
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": f"text {index}"}],
|
||||
},
|
||||
},
|
||||
{"type": "response.create"},
|
||||
)
|
||||
]
|
||||
assert client.ack_count == 4
|
||||
# The target text rides the session context, never a conversation item:
|
||||
# a text-only response.create is rejected by a model-native session.
|
||||
assert client.sent == []
|
||||
assert client.silence_seconds and all(seconds > 0 for seconds in client.silence_seconds)
|
||||
assert client.ack_count == 1
|
||||
assert client.closed == 1
|
||||
|
||||
assert output.success is True
|
||||
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
|
||||
assert output.generated_text == "turn 1 turn 1 turn 1 turn 1"
|
||||
assert output.audio_duration == pytest.approx(0.4)
|
||||
assert output.ttft > 0
|
||||
assert output.audio_ttfp > output.ttft
|
||||
@@ -281,11 +294,12 @@ async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch)
|
||||
"session_id": session_id,
|
||||
"request_index": index,
|
||||
"utterance_id": f"utt-{index}",
|
||||
"response_id": f"resp-{index + 1}",
|
||||
# Response ids are per session, and each utterance now gets its own.
|
||||
"response_id": "resp-1",
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||
"ttfp": "conversation.item.create client send to first audio packet",
|
||||
"ttft": "first silence append client send to first non-empty text delta",
|
||||
"ttfp": "first silence append client send to first audio packet",
|
||||
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||
"tpot": "Stage-0 engine mean time per output token",
|
||||
},
|
||||
|
||||
@@ -467,6 +467,27 @@ def test_tolerated_playback_ack_rejection_is_a_warning_not_a_failure():
|
||||
oi._raise_if_session_terminated(collector, 0, warnings=warnings)
|
||||
|
||||
|
||||
def test_our_own_close_is_expected_even_though_the_server_stamps_a_reason():
|
||||
"""``session.close`` is answered with ``session.closed`` carrying ``client_close``.
|
||||
|
||||
The guard exists to catch a session that ended for a reason we did not ask
|
||||
for. A close we requested is not that, whether or not the server names it.
|
||||
"""
|
||||
collector = _collector(({"type": "session.closed", "reason": "client_close"}, 1.0))
|
||||
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
|
||||
|
||||
collector = _collector(({"type": "session.closed", "event": {"reason": "client_close"}}, 1.0))
|
||||
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
|
||||
|
||||
collector = _collector(({"type": "session.closed", "reason": "disconnect"}, 1.0))
|
||||
with pytest.raises(RuntimeError, match="Unexpected session.closed: disconnect"):
|
||||
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
|
||||
|
||||
collector = _collector(({"type": "session.expired", "reason": "timeout"}, 1.0))
|
||||
with pytest.raises(RuntimeError, match="session.expired: timeout"):
|
||||
oi._raise_if_session_terminated(collector, 0, explicit_close_from=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event", "match"),
|
||||
[
|
||||
@@ -561,8 +582,8 @@ class _RealtimeClient(oi._RealtimeSession):
|
||||
|
||||
instances: list[_RealtimeClient] = []
|
||||
|
||||
def __init__(self, config: oi.OmniInteractBenchmarkConfig, session_id: str, reference_audio: str):
|
||||
super().__init__(config, session_id, reference_audio)
|
||||
def __init__(self, config: oi.OmniInteractBenchmarkConfig, reference_audio: str):
|
||||
super().__init__(config, reference_audio)
|
||||
self.acks: list[tuple[str, int]] = []
|
||||
self.instances.append(self)
|
||||
|
||||
@@ -621,7 +642,7 @@ async def test_public_runner_executes_one_prepared_session(tmp_path: Path, monke
|
||||
assert "autostart=0" in _RealtimeClient.instances[-1].url
|
||||
session_config = _RealtimeClient.instances[-1].session_config
|
||||
assert session_config.extra_body["custom"] == "value"
|
||||
assert session_config.extra_body["native_duplex"] is True
|
||||
assert "native_duplex" not in session_config.extra_body
|
||||
assert session_config.ref_audio == "data:audio/wav;base64,ref"
|
||||
acks = _RealtimeClient.instances[-1].acks
|
||||
# Cumulative acks for the one response: an optional 0 ms checkpoint the
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The Seed-TTS Realtime probe queues for a duplex session slot instead of failing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.benchmarks.patch import patch as bench_patch
|
||||
from vllm_omni.clients import duplex as duplex_client
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _AdmissionGate:
|
||||
"""Stands in for ``DuplexClient``: refuses admission ``refusals`` times, then admits."""
|
||||
|
||||
instances: list[_AdmissionGate] = []
|
||||
refusals = 0
|
||||
refusal_code = "resource_exhausted"
|
||||
|
||||
def __init__(self, url: str, **kwargs: object) -> None:
|
||||
del url, kwargs
|
||||
self.entered = False
|
||||
type(self).instances.append(self)
|
||||
|
||||
async def __aenter__(self) -> _AdmissionGate:
|
||||
cls = type(self)
|
||||
if cls.refusals > 0:
|
||||
cls.refusals -= 1
|
||||
raise duplex_client.DuplexProtocolError("duplex_session_capacity_exhausted: limit=4", code=cls.refusal_code)
|
||||
self.entered = True
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||
del exc_type, exc, tb
|
||||
|
||||
async def events(self): # the collector's consume loop ends at once
|
||||
return
|
||||
yield # pragma: no cover
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gate(monkeypatch: pytest.MonkeyPatch) -> type[_AdmissionGate]:
|
||||
_AdmissionGate.instances = []
|
||||
_AdmissionGate.refusals = 0
|
||||
_AdmissionGate.refusal_code = "resource_exhausted"
|
||||
monkeypatch.setattr(duplex_client, "DuplexClient", _AdmissionGate)
|
||||
return _AdmissionGate
|
||||
|
||||
|
||||
async def _configure(probe: bench_patch._RealtimeTTSProbe) -> None:
|
||||
try:
|
||||
await probe.configure("test-model", auto_response=True)
|
||||
finally:
|
||||
await probe.__aexit__(None, None, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_configure_waits_for_a_free_session_slot(gate) -> None:
|
||||
"""Two refusals, then a slot: the request is served, not counted as failed."""
|
||||
gate.refusals = 2
|
||||
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
|
||||
await _configure(probe)
|
||||
assert len(gate.instances) == 3
|
||||
assert gate.instances[-1].entered is True
|
||||
assert probe._client is gate.instances[-1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_configure_raises_other_rejections_at_once(gate) -> None:
|
||||
gate.refusals = 1
|
||||
gate.refusal_code = "unsupported_turn_detection"
|
||||
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
|
||||
with pytest.raises(duplex_client.DuplexProtocolError):
|
||||
await _configure(probe)
|
||||
assert len(gate.instances) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_configure_gives_up_when_the_slot_wait_budget_is_spent(gate, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(bench_patch._RealtimeTTSProbe, "_SESSION_SLOT_WAIT_S", 0.0)
|
||||
gate.refusals = 5
|
||||
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
|
||||
with pytest.raises(duplex_client.DuplexProtocolError) as excinfo:
|
||||
await _configure(probe)
|
||||
assert excinfo.value.code == "resource_exhausted"
|
||||
assert len(gate.instances) == 1
|
||||
|
||||
|
||||
class _SilenceSink:
|
||||
"""Stands in for a configured ``DuplexClient``: records every appended chunk."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = duplex_client.SessionConfig()
|
||||
self.chunks: list[bytes] = []
|
||||
|
||||
async def append_audio(self, pcm: bytes, *, is_speech: bool | None = None, video_frames=None) -> None:
|
||||
del video_frames
|
||||
assert is_speech is False
|
||||
self.chunks.append(pcm)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_silence_stops_at_the_turn_end(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Silence is fed until the turn settles, not for the whole cap."""
|
||||
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
|
||||
sink = _SilenceSink()
|
||||
probe._client = sink
|
||||
|
||||
async def no_wait(_: float) -> None:
|
||||
return
|
||||
|
||||
monkeypatch.setattr(bench_patch.asyncio, "sleep", no_wait)
|
||||
streamed = await probe.stream_silence(seconds=12.0, chunk_ms=200, until=lambda: len(sink.chunks) >= 3)
|
||||
assert len(sink.chunks) == 3
|
||||
assert streamed == pytest.approx(0.6)
|
||||
assert all(len(chunk) == sink.config.input_audio.byte_count(200) for chunk in sink.chunks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_silence_runs_to_the_cap_when_the_turn_never_settles(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
probe = bench_patch._RealtimeTTSProbe("ws://test/v1/realtime?duplex=1")
|
||||
sink = _SilenceSink()
|
||||
probe._client = sink
|
||||
|
||||
async def no_wait(_: float) -> None:
|
||||
return
|
||||
|
||||
monkeypatch.setattr(bench_patch.asyncio, "sleep", no_wait)
|
||||
streamed = await probe.stream_silence(seconds=1.0, chunk_ms=200, until=lambda: False)
|
||||
assert len(sink.chunks) == 5
|
||||
assert streamed == pytest.approx(1.0)
|
||||
|
||||
|
||||
class _TurnEvents:
|
||||
def __init__(self, response_ids: list[str], audio: dict[str, bytes], text: dict[str, str] | None = None) -> None:
|
||||
self.response_ids = response_ids
|
||||
self._audio = audio
|
||||
self._text = text or {}
|
||||
|
||||
def audio_bytes(self, response_id: str) -> bytes:
|
||||
return self._audio.get(response_id, b"")
|
||||
|
||||
def response_text(self, response_id: str) -> str:
|
||||
return self._text.get(response_id, "")
|
||||
|
||||
|
||||
def test_turn_response_is_the_first_one_with_audio_even_if_the_model_spoke_again(caplog) -> None:
|
||||
events = _TurnEvents(["r0", "r1", "r2"], {"r1": b"\x01\x00", "r2": b"\x02\x00"})
|
||||
with caplog.at_level("WARNING", logger=bench_patch.logger.name):
|
||||
assert bench_patch._seed_tts_turn_response_id(events, 0, 0) == "r1"
|
||||
assert "spoke again" in caplog.text
|
||||
|
||||
|
||||
def test_turn_response_ignores_responses_from_before_the_turn() -> None:
|
||||
events = _TurnEvents(["r0", "r1"], {"r0": b"\x01\x00", "r1": b"\x02\x00"})
|
||||
assert bench_patch._seed_tts_turn_response_id(events, 1, 0) == "r1"
|
||||
|
||||
|
||||
def test_turn_without_audio_is_an_error() -> None:
|
||||
events = _TurnEvents(["r0"], {})
|
||||
with pytest.raises(RuntimeError, match="no audio response"):
|
||||
bench_patch._seed_tts_turn_response_id(events, 0, 0)
|
||||
|
||||
|
||||
def test_stall_report_says_the_model_never_answered() -> None:
|
||||
events = _TurnEvents(["r0"], {"r0": b"\x01\x00"})
|
||||
report = bench_patch._seed_tts_turn_stall_report(events, 1, 3, 30.0)
|
||||
assert "turn 3 never started a response" in report
|
||||
assert "30.0s of silence" in report
|
||||
|
||||
|
||||
def test_stall_report_lists_a_response_that_never_finished() -> None:
|
||||
events = _TurnEvents(["r1"], {"r1": b"\x01\x00\x02\x00"}, {"r1": "hello"})
|
||||
report = bench_patch._seed_tts_turn_stall_report(events, 0, 0, 4.2)
|
||||
assert "started 1 response(s) after 4.2s of silence" in report
|
||||
assert "r1 (4 audio bytes, text 'hello')" in report
|
||||
+399
-115
@@ -1,6 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Unit tests for the public duplex client (fake transport, no server)."""
|
||||
"""Unit tests for the public duplex clients (fake transports, no server).
|
||||
|
||||
``DuplexClient`` runs over a fake websocket; ``InlineDuplexClient`` over a
|
||||
fake ``DuplexOmni`` whose handle records typed commands and replays typed
|
||||
events. The demux / handshake tests that only need "feed a server event" are
|
||||
parameterized over both clients.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -8,6 +14,8 @@ import asyncio
|
||||
import base64
|
||||
import json
|
||||
import wave
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
@@ -16,8 +24,11 @@ from vllm_omni.clients.duplex import (
|
||||
AudioFormat,
|
||||
ConnectionResumed,
|
||||
DuplexClient,
|
||||
DuplexClientBase,
|
||||
DuplexConnectionError,
|
||||
DuplexProtocolError,
|
||||
DuplexSessionClosedError,
|
||||
ErrorEvent,
|
||||
EventCollector,
|
||||
ReconnectPolicy,
|
||||
SessionConfig,
|
||||
@@ -31,17 +42,26 @@ from vllm_omni.clients.duplex import (
|
||||
summarize_session_request_metrics,
|
||||
write_pcm16_wav,
|
||||
)
|
||||
from vllm_omni.clients.inline_duplex import InlineDuplexClient
|
||||
from vllm_omni.engine.duplex import commands as duplex_commands
|
||||
from vllm_omni.engine.duplex import events as duplex_events
|
||||
from vllm_omni.engine.duplex.messages import DuplexSessionError
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
SESSION_ID = "duplex-0123456789abcdef"
|
||||
# session.created is not journaled by the server, so it carries no server_event_seq.
|
||||
SESSION_CREATED = {
|
||||
"type": "session.created",
|
||||
"session": {"session_id": "sess-1"},
|
||||
"incarnation": 0,
|
||||
"session": {"id": SESSION_ID, "model": "test-model"},
|
||||
"attachment_generation": 1,
|
||||
"resume_token": "tok-1",
|
||||
"server_event_seq": 1,
|
||||
}
|
||||
SESSION_CLOSED = {"type": "session.closed", "session_id": "sess-1", "server_event_seq": 99}
|
||||
SESSION_CLOSED = {"type": "session.closed", "session_id": SESSION_ID, "reason": "client_close", "server_event_seq": 99}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake websocket transport
|
||||
|
||||
|
||||
class FakeSocket:
|
||||
@@ -88,14 +108,139 @@ def make_client(*sockets: FakeSocket, **kwargs) -> tuple[DuplexClient, list[str]
|
||||
return client, calls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fake DuplexOmni for the inline client
|
||||
|
||||
|
||||
class _WireEvent:
|
||||
"""A typed-event stand-in that renders a fixed wire payload."""
|
||||
|
||||
def __init__(self, raw: dict[str, object]) -> None:
|
||||
self.raw = raw
|
||||
|
||||
@property
|
||||
def is_terminal(self) -> bool:
|
||||
return self.raw.get("type") in {"session.closed", "session.expired"}
|
||||
|
||||
def to_realtime(self) -> dict[str, object]:
|
||||
return dict(self.raw)
|
||||
|
||||
|
||||
class FakeInlineHandle:
|
||||
def __init__(self, session_id: str) -> None:
|
||||
self.session_id = session_id
|
||||
self.closed = False
|
||||
self.close_calls = 0
|
||||
self.commands: list[duplex_commands.DuplexCommand] = []
|
||||
self.consumer_active = False
|
||||
self._outbox: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
def deliver(self, event: Any) -> None:
|
||||
self._outbox.put_nowait(event)
|
||||
|
||||
def feed(self, raw: dict[str, object]) -> None:
|
||||
self.deliver(_WireEvent(raw))
|
||||
|
||||
async def submit(self, command: duplex_commands.DuplexCommand) -> None:
|
||||
if self.closed:
|
||||
raise DuplexSessionError("closed", code="session_closed", session_id=self.session_id)
|
||||
self.commands.append(command)
|
||||
|
||||
async def events(self):
|
||||
self.consumer_active = True
|
||||
try:
|
||||
while True:
|
||||
event = await self._outbox.get()
|
||||
yield event
|
||||
if event.is_terminal:
|
||||
return
|
||||
finally:
|
||||
self.consumer_active = False
|
||||
|
||||
async def close(self, *, reason: str = "client_close", timeout: float | None = None) -> None:
|
||||
self.close_calls += 1
|
||||
if self.closed:
|
||||
return
|
||||
self.closed = True
|
||||
self.deliver(duplex_events.SessionClosed(session_id=self.session_id, reason=reason))
|
||||
|
||||
|
||||
class FakeInlineOmni:
|
||||
def __init__(self) -> None:
|
||||
self.opened: list[dict[str, object]] = []
|
||||
self.handles: list[FakeInlineHandle] = []
|
||||
|
||||
async def open_session(self, config: dict[str, object]) -> FakeInlineHandle:
|
||||
self.opened.append(dict(config))
|
||||
handle = FakeInlineHandle(SESSION_ID)
|
||||
self.handles.append(handle)
|
||||
handle.deliver(
|
||||
duplex_events.SessionCreated(
|
||||
session_id=SESSION_ID,
|
||||
session={"id": SESSION_ID, "model": config.get("model")},
|
||||
)
|
||||
)
|
||||
return handle
|
||||
|
||||
|
||||
def make_inline_client(**kwargs) -> tuple[InlineDuplexClient, FakeInlineOmni]:
|
||||
omni = FakeInlineOmni()
|
||||
kwargs.setdefault("handshake_timeout_s", 5.0)
|
||||
return InlineDuplexClient(omni, model="test-model", **kwargs), omni
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# One rig over both clients: ``feed`` a server event, drive ``client``
|
||||
|
||||
|
||||
class Rig:
|
||||
def __init__(self, client: DuplexClientBase, feed: Callable[[dict[str, object]], None]) -> None:
|
||||
self.client = client
|
||||
self.feed = feed
|
||||
|
||||
|
||||
def _websocket_rig(*, handshake: dict[str, object] | None = SESSION_CREATED) -> Rig:
|
||||
sock = FakeSocket()
|
||||
if handshake is not None:
|
||||
sock.feed(handshake)
|
||||
client, _ = make_client(sock)
|
||||
return Rig(client, sock.feed)
|
||||
|
||||
|
||||
def _inline_rig(*, handshake: dict[str, object] | None = SESSION_CREATED) -> Rig:
|
||||
client, omni = make_inline_client()
|
||||
original_open = omni.open_session
|
||||
|
||||
async def open_session(config: dict[str, object]) -> FakeInlineHandle:
|
||||
handle = await original_open(config)
|
||||
if handshake is not SESSION_CREATED:
|
||||
# Replace the default session.created with the requested first event.
|
||||
handle._outbox = asyncio.Queue()
|
||||
if handshake is not None:
|
||||
handle.feed(handshake)
|
||||
return handle
|
||||
|
||||
omni.open_session = open_session # type: ignore[method-assign]
|
||||
|
||||
def feed(raw: dict[str, object]) -> None:
|
||||
omni.handles[-1].feed(raw)
|
||||
|
||||
return Rig(client, feed)
|
||||
|
||||
|
||||
@pytest.fixture(params=["websocket", "inline"])
|
||||
def make_rig(request: pytest.FixtureRequest) -> Callable[..., Rig]:
|
||||
return _websocket_rig if request.param == "websocket" else _inline_rig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SessionConfig
|
||||
|
||||
|
||||
def test_session_config_payload_defaults():
|
||||
payload = SessionConfig().to_session_payload(model="m", session_id="s")
|
||||
payload = SessionConfig().to_session_payload(model="m")
|
||||
assert payload["model"] == "m"
|
||||
assert payload["session_id"] == "s"
|
||||
assert "session_id" not in payload
|
||||
assert payload["modalities"] == ["audio", "text"]
|
||||
assert payload["input_audio_format"] == "pcm16"
|
||||
assert payload["output_audio_format"] == "pcm16"
|
||||
@@ -137,19 +282,59 @@ def test_audio_format_math():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Handshake and lifecycle
|
||||
# Handshake and lifecycle (both clients)
|
||||
|
||||
|
||||
async def test_handshake_adopts_session_and_acks():
|
||||
@pytest.mark.asyncio
|
||||
async def test_handshake_adopts_server_allocated_session_id(make_rig):
|
||||
rig = make_rig()
|
||||
async with rig.client as client:
|
||||
assert client.session_id == SESSION_ID
|
||||
assert client.session_info == {"id": SESSION_ID, "model": "test-model"}
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handshake_error_raises_protocol_error(make_rig):
|
||||
rig = make_rig(handshake={"type": "error", "error": {"code": "unsupported_audio_format", "message": "bad"}})
|
||||
with pytest.raises(DuplexProtocolError) as excinfo:
|
||||
async with rig.client:
|
||||
pass
|
||||
assert excinfo.value.code == "unsupported_audio_format"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_expired_raises_from_event_stream(make_rig):
|
||||
rig = make_rig()
|
||||
async with rig.client as client:
|
||||
rig.feed(
|
||||
{"type": "session.expired", "session_id": SESSION_ID, "reason": "lease_expired", "server_event_seq": 2}
|
||||
)
|
||||
with pytest.raises(DuplexSessionClosedError) as excinfo:
|
||||
async for _ in client.events():
|
||||
pass
|
||||
assert "lease_expired" in excinfo.value.reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Websocket transport specifics
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_handshake_sends_session_update_and_acks_sequenced_events():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
client, calls = make_client(sock)
|
||||
async with client:
|
||||
assert client.session_id == "sess-1"
|
||||
assert client.session_id == SESSION_ID
|
||||
assert client.resume_token == "tok-1"
|
||||
assert calls == ["ws://test-host:8099/v1/realtime?duplex=1&model=test-model&autostart=0"]
|
||||
assert sock.sent[0]["type"] == "session.update"
|
||||
assert sock.sent[0]["session"]["model"] == "test-model"
|
||||
assert "session_id" not in sock.sent[0]["session"]
|
||||
# session.created is unjournaled (no seq): nothing to ack yet.
|
||||
assert _acks(sock) == []
|
||||
sock.feed({"type": "response.listen", "server_event_seq": 1})
|
||||
await _drain(lambda: {"type": "session.event_ack", "server_event_seq": 1} in _acks(sock))
|
||||
close_task = asyncio.create_task(client.close())
|
||||
await _drain(lambda: "session.close" in sock.sent_types())
|
||||
@@ -158,28 +343,7 @@ async def test_handshake_adopts_session_and_acks():
|
||||
assert "session.close" in sock.sent_types()
|
||||
|
||||
|
||||
async def test_handshake_error_raises_protocol_error():
|
||||
sock = FakeSocket()
|
||||
sock.feed({"type": "error", "error": {"code": "unsupported_audio_format", "message": "bad"}})
|
||||
client, _ = make_client(sock)
|
||||
with pytest.raises(DuplexProtocolError) as excinfo:
|
||||
async with client:
|
||||
pass
|
||||
assert excinfo.value.code == "unsupported_audio_format"
|
||||
|
||||
|
||||
async def test_session_expired_raises_from_event_stream():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed({"type": "session.expired", "reason": "lease_expired", "server_event_seq": 2})
|
||||
with pytest.raises(DuplexSessionClosedError) as excinfo:
|
||||
async for _ in client.events():
|
||||
pass
|
||||
assert "lease_expired" in excinfo.value.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resync_required_drops_resume_credential():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -187,7 +351,7 @@ async def test_resync_required_drops_resume_credential():
|
||||
async with client:
|
||||
assert client.resume_token == "tok-1"
|
||||
stream = client.events()
|
||||
sock.feed({"type": "session.resync_required", "session_id": "sess-1", "server_event_seq": 2})
|
||||
sock.feed({"type": "session.resync_required", "session_id": SESSION_ID, "server_event_seq": 2})
|
||||
async for event in stream:
|
||||
if event.type == "session.resync_required":
|
||||
break
|
||||
@@ -202,9 +366,10 @@ async def test_resync_required_drops_resume_credential():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input events
|
||||
# Input events (websocket wire shape)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_audio_tracks_cumulative_end_ms():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -224,6 +389,7 @@ async def test_append_audio_tracks_cumulative_end_ms():
|
||||
sock.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_audio_strips_video_frame_data_url_prefix():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -238,6 +404,7 @@ async def test_append_audio_strips_video_frame_data_url_prefix():
|
||||
sock.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_pcm_chunking():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -249,6 +416,7 @@ async def test_stream_pcm_chunking():
|
||||
sock.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interruption_primitives_send_documented_events():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -262,22 +430,21 @@ async def test_interruption_primitives_send_documented_events():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response demultiplexing
|
||||
# Response demultiplexing (both clients)
|
||||
|
||||
|
||||
def _b64(data: bytes) -> str:
|
||||
return base64.b64encode(data).decode("ascii")
|
||||
|
||||
|
||||
async def test_response_handle_flow():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_handle_flow(make_rig):
|
||||
rig = make_rig()
|
||||
chunk = b"\x00\x01" * 2400 # 100 ms of pcm16 @ 24 kHz
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
|
||||
sock.feed(
|
||||
async with rig.client as client:
|
||||
rig.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
rig.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
|
||||
rig.feed(
|
||||
{
|
||||
"type": "response.output_audio.delta",
|
||||
"response_id": "resp-1",
|
||||
@@ -286,7 +453,7 @@ async def test_response_handle_flow():
|
||||
"server_event_seq": 4,
|
||||
}
|
||||
)
|
||||
sock.feed(
|
||||
rig.feed(
|
||||
{
|
||||
"type": "response.output_audio_transcript.delta",
|
||||
"response_id": "resp-1",
|
||||
@@ -294,7 +461,7 @@ async def test_response_handle_flow():
|
||||
"server_event_seq": 5,
|
||||
}
|
||||
)
|
||||
sock.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 6})
|
||||
rig.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 6})
|
||||
|
||||
async for response in client.responses():
|
||||
chunks = [piece async for piece in response.audio()]
|
||||
@@ -304,49 +471,46 @@ async def test_response_handle_flow():
|
||||
assert response.played_ms == pytest.approx(100.0)
|
||||
assert response.finished
|
||||
break
|
||||
sock.feed(SESSION_CLOSED)
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
async def test_listen_decision_yields_finished_silent_handle():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed({"type": "response.listen", "server_event_seq": 2})
|
||||
@pytest.mark.asyncio
|
||||
async def test_listen_decision_yields_finished_silent_handle(make_rig):
|
||||
rig = make_rig()
|
||||
async with rig.client as client:
|
||||
rig.feed({"type": "response.listen", "server_event_seq": 2})
|
||||
async for response in client.responses():
|
||||
assert response.decision == "listen"
|
||||
assert response.finished
|
||||
assert [piece async for piece in response.audio()] == []
|
||||
break
|
||||
sock.feed(SESSION_CLOSED)
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
async def test_listen_terminates_active_response():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
sock.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 3})
|
||||
@pytest.mark.asyncio
|
||||
async def test_listen_terminates_active_response(make_rig):
|
||||
rig = make_rig()
|
||||
async with rig.client as client:
|
||||
rig.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
rig.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 3})
|
||||
async for response in client.responses():
|
||||
await response.wait(timeout_s=5.0)
|
||||
assert response.decision == "listen"
|
||||
break
|
||||
sock.feed(SESSION_CLOSED)
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
async def test_id_less_listen_never_closes_an_in_flight_response():
|
||||
@pytest.mark.asyncio
|
||||
async def test_id_less_listen_never_closes_an_in_flight_response(make_rig):
|
||||
# An id-less listen is a standalone decision beat (e.g. a silence-skip
|
||||
# while a response streams); it must surface as its own finished handle
|
||||
# and leave the in-flight response to its real terminal.
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
|
||||
sock.feed({"type": "response.listen", "server_event_seq": 4})
|
||||
sock.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 5})
|
||||
rig = make_rig()
|
||||
async with rig.client as client:
|
||||
rig.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
rig.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
|
||||
rig.feed({"type": "response.listen", "server_event_seq": 4})
|
||||
rig.feed({"type": "response.done", "response_id": "resp-1", "server_event_seq": 5})
|
||||
seen: list[tuple[str | None, str | None, bool]] = []
|
||||
async for response in client.responses():
|
||||
await response.wait(timeout_s=5.0)
|
||||
@@ -355,20 +519,19 @@ async def test_id_less_listen_never_closes_an_in_flight_response():
|
||||
break
|
||||
assert ("resp-1", "speak", True) in seen
|
||||
assert (None, "listen", True) in seen
|
||||
sock.feed(SESSION_CLOSED)
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
async def test_terminal_listen_keeps_spoken_decision():
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_listen_keeps_spoken_decision(make_rig):
|
||||
# A terminal listen after the response spoke is the model yielding the
|
||||
# turn; the handle must stay decision="speak" with its audio intact.
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
rig = make_rig()
|
||||
chunk = b"\x00\x01" * 2400
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
sock.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
|
||||
sock.feed(
|
||||
async with rig.client as client:
|
||||
rig.feed({"type": "response.created", "response": {"id": "resp-1"}, "server_event_seq": 2})
|
||||
rig.feed({"type": "response.speak", "response_id": "resp-1", "server_event_seq": 3})
|
||||
rig.feed(
|
||||
{
|
||||
"type": "response.output_audio.delta",
|
||||
"response_id": "resp-1",
|
||||
@@ -377,24 +540,23 @@ async def test_terminal_listen_keeps_spoken_decision():
|
||||
"server_event_seq": 4,
|
||||
}
|
||||
)
|
||||
sock.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 5})
|
||||
rig.feed({"type": "response.listen", "response_id": "resp-1", "server_event_seq": 5})
|
||||
async for response in client.responses():
|
||||
chunks = [piece async for piece in response.audio()]
|
||||
assert chunks == [chunk]
|
||||
assert response.decision == "speak"
|
||||
assert response.finished
|
||||
break
|
||||
sock.feed(SESSION_CLOSED)
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
async def test_error_event_surfaces_on_response_path():
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_event_surfaces_on_response_path(make_rig):
|
||||
# A rejected send produces no response; a consumer waiting in responses()
|
||||
# must see the rejection instead of waiting forever.
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
client, _ = make_client(sock)
|
||||
async with client:
|
||||
sock.feed(
|
||||
rig = make_rig()
|
||||
async with rig.client as client:
|
||||
rig.feed(
|
||||
{
|
||||
"type": "error",
|
||||
"error": {"code": "input_audio_buffer_empty", "message": "empty commit"},
|
||||
@@ -405,9 +567,10 @@ async def test_error_event_surfaces_on_response_path():
|
||||
async for _ in client.responses():
|
||||
pass
|
||||
assert excinfo.value.code == "input_audio_buffer_empty"
|
||||
sock.feed(SESSION_CLOSED)
|
||||
rig.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_slow_audio_consumer_drops_oldest_instead_of_stalling():
|
||||
from vllm_omni.clients.duplex import AudioDelta, ResponseHandle
|
||||
|
||||
@@ -425,24 +588,24 @@ async def test_slow_audio_consumer_drops_oldest_instead_of_stalling():
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resume
|
||||
# Resume (websocket only)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_after_transport_drop():
|
||||
first = FakeSocket()
|
||||
first.feed(SESSION_CREATED)
|
||||
second = FakeSocket()
|
||||
# Mirror the real resume activation payload
|
||||
# (entrypoints/duplex/serving.py, activation_payload_factory): it carries
|
||||
# no nested "session" object, only the identity fields.
|
||||
# the identity fields plus the (empty) public session object.
|
||||
second.feed(
|
||||
{
|
||||
"type": "session.resumed",
|
||||
"session_id": "sess-1",
|
||||
"incarnation": 0,
|
||||
"attachment_generation": 1,
|
||||
"session_id": SESSION_ID,
|
||||
"session": {},
|
||||
"attachment_generation": 2,
|
||||
"resume_token": "tok-2",
|
||||
"server_event_seq": 5,
|
||||
}
|
||||
)
|
||||
client, calls = make_client(
|
||||
@@ -453,34 +616,39 @@ async def test_resume_after_transport_drop():
|
||||
async with client:
|
||||
stream = client.events()
|
||||
|
||||
async def take_two():
|
||||
async def collect_until_resumed():
|
||||
seen = []
|
||||
async for event in stream:
|
||||
seen.append(event)
|
||||
if len(seen) == 2:
|
||||
if isinstance(event, SessionResumed):
|
||||
return seen
|
||||
|
||||
consumer = asyncio.create_task(take_two())
|
||||
consumer = asyncio.create_task(collect_until_resumed())
|
||||
await asyncio.sleep(0) # let the consumer subscribe before the drop
|
||||
await asyncio.sleep(0)
|
||||
first.feed({"type": "response.listen", "server_event_seq": 7})
|
||||
await _drain(lambda: {"type": "session.event_ack", "server_event_seq": 7} in _acks(first))
|
||||
first.feed(RuntimeError("transport dropped"))
|
||||
seen = await asyncio.wait_for(consumer, timeout=5.0)
|
||||
assert isinstance(seen[0], ConnectionResumed)
|
||||
assert isinstance(seen[1], SessionResumed)
|
||||
assert [type(event) for event in seen[-2:]] == [ConnectionResumed, SessionResumed]
|
||||
assert client.resume_token == "tok-2"
|
||||
# The superseded transport is closed, not leaked.
|
||||
assert first.closed
|
||||
# The activation payload has no session object; the info captured at
|
||||
# the original handshake must survive the resume.
|
||||
assert client.session_info == {"session_id": "sess-1"}
|
||||
assert client.session_info == {"id": SESSION_ID, "model": "test-model"}
|
||||
assert len(calls) == 2
|
||||
resume = second.sent[0]
|
||||
assert resume["type"] == "session.resume"
|
||||
assert resume["session_id"] == "sess-1"
|
||||
assert resume["incarnation"] == 0
|
||||
assert resume["resume_token"] == "tok-1"
|
||||
assert resume["last_received_server_event_seq"] == 1
|
||||
assert resume == {
|
||||
"type": "session.resume",
|
||||
"session_id": SESSION_ID,
|
||||
"resume_token": "tok-1",
|
||||
"last_received_server_event_seq": 7,
|
||||
}
|
||||
second.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_reconnect_policy_surfaces_closed():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -493,6 +661,7 @@ async def test_no_reconnect_policy_surfaces_closed():
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_heartbeat_survives_transient_send_failure():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -514,6 +683,7 @@ async def test_heartbeat_survives_transient_send_failure():
|
||||
sock.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aexit_sends_session_close_on_error_path():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -534,6 +704,7 @@ def test_target_url_forces_autostart_off():
|
||||
assert query["autostart"] == ["0"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fatal_resume_error_gives_up():
|
||||
first = FakeSocket()
|
||||
first.feed(SESSION_CREATED)
|
||||
@@ -553,6 +724,121 @@ async def test_fatal_resume_error_gives_up():
|
||||
assert len(calls) == 2 # no retry after a fatal resume error
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inline client over a fake DuplexOmni
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_handshake_opens_session_with_payload_and_adopts_id():
|
||||
client, omni = make_inline_client(config=SessionConfig(ref_audio="data:audio/wav;base64,AAA="))
|
||||
async with client:
|
||||
assert omni.opened == [client.config.to_session_payload(model="test-model")]
|
||||
assert omni.opened[0]["ref_audio"] == "data:audio/wav;base64,AAA="
|
||||
assert client.session_id == SESSION_ID
|
||||
handle = omni.handles[0]
|
||||
# The session was opened with the full session object already; the
|
||||
# base handshake's session.update must not be re-submitted as a patch
|
||||
# (a ref_audio patch would be rejected by the runner).
|
||||
assert handle.commands == []
|
||||
await client.send({"type": "session.update", "session": {"instructions": "later"}})
|
||||
assert [type(command) for command in handle.commands] == [duplex_commands.UpdateSession]
|
||||
assert handle.commands[0].patch == {"instructions": "later"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_client_submits_typed_commands_in_order():
|
||||
client, omni = make_inline_client()
|
||||
async with client:
|
||||
handle = omni.handles[0]
|
||||
pcm = b"\x01\x00" * 1600
|
||||
await client.append_audio(pcm, is_speech=True)
|
||||
await client.commit(create_response=True)
|
||||
await client.cancel_response("resp-1")
|
||||
await client.ack_playback(120.9, response_id="resp-1")
|
||||
assert [type(command) for command in handle.commands] == [
|
||||
duplex_commands.AppendAudio,
|
||||
duplex_commands.Commit,
|
||||
duplex_commands.CancelResponse,
|
||||
duplex_commands.AckPlayback,
|
||||
]
|
||||
append, commit, cancel, ack = handle.commands
|
||||
assert append.is_speech is True
|
||||
assert append.audio_end_ms == 100
|
||||
assert append.event_id
|
||||
assert commit.final is True and commit.create_response is True
|
||||
assert cancel.response_id == "resp-1"
|
||||
assert ack.played_ms == 120 and ack.response_id == "resp-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_session_close_maps_to_handle_close():
|
||||
client, omni = make_inline_client()
|
||||
async with client:
|
||||
pass
|
||||
handle = omni.handles[0]
|
||||
assert handle.closed
|
||||
assert handle.close_calls >= 1
|
||||
assert not any(isinstance(command, duplex_commands.CloseSession) for command in handle.commands)
|
||||
# The typed session.closed reached the client as the expected end.
|
||||
with pytest.raises(DuplexSessionClosedError):
|
||||
await client.send({"type": "session.heartbeat"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_client_forwards_heartbeats_and_drops_event_acks():
|
||||
client, omni = make_inline_client()
|
||||
async with client:
|
||||
handle = omni.handles[0]
|
||||
await client.send({"type": "session.heartbeat"})
|
||||
await client.send({"type": "session.event_ack", "server_event_seq": 3})
|
||||
# A heartbeat is a lease touch the engine must see; the ack is a journal concern.
|
||||
assert [type(command).__name__ for command in handle.commands] == ["Heartbeat"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_malformed_payload_surfaces_as_error_event():
|
||||
client, omni = make_inline_client()
|
||||
async with client:
|
||||
handle = omni.handles[0]
|
||||
# Subscribe before sending: the translation error is dispatched inline.
|
||||
first_event = asyncio.create_task(client.wait_for("error", timeout_s=5.0))
|
||||
await asyncio.sleep(0)
|
||||
event_id = await client.send({"type": "playback.ack"}) # played_ms missing
|
||||
event = await first_event
|
||||
assert isinstance(event, ErrorEvent)
|
||||
assert event.code == "bad_event"
|
||||
assert event.related_event_id == event_id
|
||||
assert handle.commands == []
|
||||
with pytest.raises(DuplexProtocolError) as excinfo:
|
||||
async for _ in client.responses():
|
||||
pass
|
||||
assert excinfo.value.code == "bad_event"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_teardown_cancels_pump_and_closes_handle():
|
||||
client, omni = make_inline_client()
|
||||
async with client:
|
||||
handle = omni.handles[0]
|
||||
await _drain(lambda: handle.consumer_active)
|
||||
pump = client._pump_task
|
||||
assert pump is not None and not pump.done()
|
||||
assert pump.done()
|
||||
assert not handle.consumer_active
|
||||
assert handle.closed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inline_send_after_handle_closed_raises_connection_error():
|
||||
client, omni = make_inline_client()
|
||||
async with client:
|
||||
handle = omni.handles[0]
|
||||
handle.closed = True # the engine ended the session without an event yet
|
||||
with pytest.raises(DuplexConnectionError, match="not open"):
|
||||
await client.send({"type": "input_audio_buffer.clear"})
|
||||
handle.closed = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collector
|
||||
|
||||
@@ -600,8 +886,7 @@ def test_build_realtime_url_with_model_extra_query():
|
||||
url = build_realtime_url(
|
||||
"ws://localhost:8099/v1/realtime?custom=1",
|
||||
"openbmb/MiniCPM-o-4_5",
|
||||
session_id="session-a",
|
||||
extra_query={"native_duplex": "1"},
|
||||
extra_query={"probe": "1"},
|
||||
)
|
||||
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
@@ -609,8 +894,7 @@ def test_build_realtime_url_with_model_extra_query():
|
||||
"custom": ["1"],
|
||||
"duplex": ["1"],
|
||||
"model": ["openbmb/MiniCPM-o-4_5"],
|
||||
"native_duplex": ["1"],
|
||||
"session_id": ["session-a"],
|
||||
"probe": ["1"],
|
||||
}
|
||||
|
||||
|
||||
@@ -619,12 +903,11 @@ def test_build_realtime_url_resume_only_when_autostart_disabled():
|
||||
"ws://localhost:8099/v1/realtime?duplex=1",
|
||||
"openbmb/MiniCPM-o-4_5",
|
||||
autostart=False,
|
||||
extra_query={"native_duplex": "1"},
|
||||
)
|
||||
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
assert query["autostart"] == ["0"]
|
||||
assert query["native_duplex"] == ["1"]
|
||||
assert "session_id" not in query
|
||||
|
||||
|
||||
def test_event_collector_partitions_audio_by_response():
|
||||
@@ -989,6 +1272,7 @@ def test_duplex_unit_boundary_and_residual_math():
|
||||
assert chunk_period_ms([{"type": "session.created", "session": {}}]) == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_pcm_sends_each_units_composite_beside_its_base_frame():
|
||||
sock = FakeSocket()
|
||||
sock.feed(SESSION_CREATED)
|
||||
@@ -1011,16 +1295,16 @@ async def test_stream_pcm_sends_each_units_composite_beside_its_base_frame():
|
||||
sock.feed(SESSION_CLOSED)
|
||||
|
||||
|
||||
def test_build_realtime_url_native_duplex_flag_and_http_scheme():
|
||||
url = build_realtime_url("http://localhost:8099/v1/realtime", None, native_duplex=None)
|
||||
def test_build_realtime_url_rewrites_http_scheme_and_rejects_others():
|
||||
url = build_realtime_url("http://localhost:8099/v1/realtime", None)
|
||||
parts = urlsplit(url)
|
||||
assert parts.scheme == "ws"
|
||||
assert parse_qs(parts.query) == {"duplex": ["1"]}
|
||||
|
||||
url = build_realtime_url("https://host/v1/realtime", "m", native_duplex=False)
|
||||
url = build_realtime_url("https://host/v1/realtime", "m", autostart=True)
|
||||
parts = urlsplit(url)
|
||||
assert parts.scheme == "wss"
|
||||
assert parse_qs(parts.query)["native_duplex"] == ["0"]
|
||||
assert parse_qs(parts.query) == {"duplex": ["1"], "model": ["m"], "autostart": ["1"]}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
build_realtime_url("ftp://host/v1/realtime", "m")
|
||||
|
||||
@@ -18,13 +18,13 @@ from vllm_omni.clients.personaplex import (
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_minicpmo45_session_config_matches_native_duplex_deployment():
|
||||
def test_minicpmo45_session_config_matches_duplex_deployment():
|
||||
config = create_minicpmo45_session_config(ref_audio="data:audio/wav;base64,AAA=", temperature=0.0)
|
||||
payload = config.to_session_payload(model="openbmb/MiniCPM-o-4_5")
|
||||
assert payload["ref_audio"] == "data:audio/wav;base64,AAA="
|
||||
assert payload["overlap_policy"] == "listen_only"
|
||||
assert payload["playback_commit_policy"] == "ack_only"
|
||||
assert payload["extra_body"]["native_duplex"] is True
|
||||
assert "native_duplex" not in payload["extra_body"]
|
||||
assert payload["extra_body"]["auto_response"] is True
|
||||
assert payload["temperature"] == 0.0
|
||||
|
||||
@@ -41,8 +41,9 @@ def test_personaplex_session_config_matches_deployment():
|
||||
@pytest.mark.parametrize(
|
||||
"duplex_package",
|
||||
[
|
||||
# PersonaPlex and Nemotron VoiceChat rejoin this list with the follow-up
|
||||
# PRs that port them to the duplex plugin framework.
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
|
||||
"vllm_omni.model_executor.models.personaplex.duplex",
|
||||
],
|
||||
)
|
||||
def test_model_duplex_packages_stay_clear_of_client_library(duplex_package: str):
|
||||
|
||||
@@ -26,8 +26,8 @@
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
@@ -42,73 +42,7 @@
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.9731,
|
||||
1.1986,
|
||||
1.686
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
152.5665,
|
||||
270.2559,
|
||||
301.455
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1027.2953,
|
||||
3306.3735,
|
||||
4686.5366
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
458.1553,
|
||||
2030.2335,
|
||||
2068.6233
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2357,
|
||||
0.791,
|
||||
1.0324
|
||||
]
|
||||
},
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.5383,
|
||||
0.6042,
|
||||
0.7547
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
333.2633,
|
||||
533.8656,
|
||||
547.3388
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1857.2154,
|
||||
6600.5101,
|
||||
10450.6811
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
986.4666,
|
||||
3411.0754,
|
||||
3352.7515
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.4423,
|
||||
1.5734,
|
||||
2.3024
|
||||
]
|
||||
}
|
||||
}
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -138,8 +72,8 @@
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
@@ -154,16 +88,6 @@
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -43,43 +43,7 @@
|
||||
"extra_body": {
|
||||
"save_duplex_request_metrics": true
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.3038
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
139.997
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
3269.8807
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
456.073
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2254
|
||||
]
|
||||
},
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.1594
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
274.164
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6250.7696
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
994.422
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.46163
|
||||
]
|
||||
}
|
||||
}
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -26,8 +26,8 @@
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
@@ -40,41 +40,7 @@
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "en",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.9731,
|
||||
1.1986
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
152.5665,
|
||||
270.2559
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1027.2953,
|
||||
3306.3735
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
458.1553,
|
||||
2030.2335
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2357,
|
||||
0.791
|
||||
]
|
||||
}
|
||||
}
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -39,7 +39,8 @@ from vllm_omni.distributed.omni_connectors.utils.serialization import (
|
||||
from vllm_omni.engine.async_omni_engine import StageRuntimeInfo
|
||||
from vllm_omni.engine.messages import ShutdownRequestMessage, StageSubmissionMessage
|
||||
from vllm_omni.engine.stage_init_utils import StageMetadata
|
||||
from vllm_omni.entrypoints.async_omni import AsyncEventResolver, AsyncOmni
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.async_omni_base import AsyncEventResolver
|
||||
from vllm_omni.entrypoints.openai.serving_video_output_stream import OmniStreamingVideoOutputHandler
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
@@ -42,7 +42,8 @@ from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.engine.stage_client import StagePoolClient
|
||||
from vllm_omni.engine.stage_init_utils import StageMetadata
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.entrypoints.async_omni import AsyncEventResolver, AsyncOmni
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.async_omni_base import AsyncEventResolver
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniInteractionPrompt
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
|
||||
@@ -90,6 +90,7 @@ def _get_function_source(source: str, class_name: str | None, func_name: str) ->
|
||||
Args:
|
||||
source: Full file source code.
|
||||
class_name: Enclosing class name, or None for module-level functions.
|
||||
The method may be inherited from a base class defined in the same file.
|
||||
func_name: Function/method name.
|
||||
|
||||
Returns:
|
||||
@@ -97,13 +98,24 @@ def _get_function_source(source: str, class_name: str | None, func_name: str) ->
|
||||
"""
|
||||
tree = ast.parse(source)
|
||||
if class_name is not None:
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ClassDef) and node.name == class_name:
|
||||
# Follow the bases: ``Orchestrator`` inherits its loops and helpers from
|
||||
# ``OrchestratorBase`` in the same module, and the lookup stays anchored
|
||||
# on the public class rather than on where the split happens to fall.
|
||||
classes = {node.name: node for node in ast.walk(tree) if isinstance(node, ast.ClassDef)}
|
||||
pending = [class_name]
|
||||
seen: set[str] = set()
|
||||
while pending:
|
||||
name = pending.pop(0)
|
||||
if name in seen or name not in classes:
|
||||
continue
|
||||
seen.add(name)
|
||||
node = classes[name]
|
||||
for item in node.body:
|
||||
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) and item.name == func_name:
|
||||
result = ast.get_source_segment(source, item)
|
||||
assert result is not None, f"{class_name}.{func_name} source not found"
|
||||
return result
|
||||
pending.extend(base.id for base in node.bases if isinstance(base, ast.Name))
|
||||
else:
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name:
|
||||
|
||||
@@ -14,7 +14,7 @@ import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence
|
||||
from vllm_omni.experimental.fullduplex.core import protocol as ev
|
||||
from vllm_omni.experimental.fullduplex.core.adapter import (
|
||||
DuplexAdapter,
|
||||
@@ -32,12 +32,15 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_fence_is_frozen_value_identity():
|
||||
# The fence carries (session_id, epoch, turn_id). response_seq was
|
||||
# dropped with the incarnation concept: response identity is the session's
|
||||
# active_response_id, not a field of the fence.
|
||||
f = DuplexFence(session_id="s1")
|
||||
assert (f.epoch, f.turn_id, f.response_seq) == (0, 0, 0)
|
||||
assert (f.epoch, f.turn_id) == (0, 0)
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
f.epoch = 1 # type: ignore[misc]
|
||||
assert DuplexFence("s1", 1, 2, 3) == DuplexFence("s1", 1, 2, 3)
|
||||
assert DuplexFence("s1", 1, 2, 3) != DuplexFence("s1", 2, 2, 3)
|
||||
assert DuplexFence("s1", 1, 2) == DuplexFence("s1", 1, 2)
|
||||
assert DuplexFence("s1", 1, 2) != DuplexFence("s1", 2, 2)
|
||||
|
||||
|
||||
def test_session_identity_rules():
|
||||
|
||||
@@ -71,15 +71,8 @@ from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402
|
||||
from vllm_omni.metrics.definitions import compute_audio_rtf # noqa: E402
|
||||
|
||||
|
||||
def _url_with_model(url, model, *, autostart=None, session_id=None):
|
||||
# This driver speaks the MiniCPM-o native duplex wire contract.
|
||||
return build_realtime_url(
|
||||
url,
|
||||
model,
|
||||
autostart=autostart,
|
||||
session_id=session_id,
|
||||
extra_query={"native_duplex": "1"},
|
||||
)
|
||||
def _url_with_model(url, model, *, autostart=None):
|
||||
return build_realtime_url(url, model, autostart=autostart)
|
||||
|
||||
|
||||
_read_wav_pcm16 = read_pcm16_wav
|
||||
@@ -630,7 +623,6 @@ def _session_update_event(args: DemoArgs) -> dict[str, object]:
|
||||
"playback_commit_policy": "ack_only",
|
||||
"extra_body": {
|
||||
"auto_response": True,
|
||||
"native_duplex": True,
|
||||
"force_listen_count": 0,
|
||||
},
|
||||
}
|
||||
@@ -646,9 +638,6 @@ def _session_update_event(args: DemoArgs) -> dict[str, object]:
|
||||
"type": "session.update",
|
||||
"session": session_payload,
|
||||
}
|
||||
session_id = getattr(args, "session_id", None)
|
||||
if session_id:
|
||||
session_payload["session_id"] = session_id
|
||||
return event
|
||||
|
||||
|
||||
@@ -1462,7 +1451,6 @@ async def run_demo(args: DemoArgs) -> dict[str, object]:
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=getattr(args, "session_id", None),
|
||||
)
|
||||
state = DemoState()
|
||||
stop = asyncio.Event()
|
||||
@@ -1674,6 +1662,10 @@ async def run_demo(args: DemoArgs) -> dict[str, object]:
|
||||
)
|
||||
unexpected_error_events = _unexpected_error_events(state)
|
||||
continuous_input_ok = not continuous_input or state.count("input_audio_buffer.committed") == 0
|
||||
# The server allocates every session id; label metrics with the one it announced.
|
||||
created = state.events[0] if state.events else {}
|
||||
created_session = created.get("session") if isinstance(created, dict) else None
|
||||
server_session_id = created_session.get("id") if isinstance(created_session, dict) else None
|
||||
result = {
|
||||
"ok": terminal_activity_ok
|
||||
and state.count("session.closed") > 0
|
||||
@@ -1711,8 +1703,8 @@ async def run_demo(args: DemoArgs) -> dict[str, object]:
|
||||
"input_transcription_ok": input_transcription_ok,
|
||||
"completed_response_ids": completed_response_ids,
|
||||
"response_timings": state.response_timing_summaries(),
|
||||
"request_metrics": state.session_request_metrics(session_id=args.session_id),
|
||||
"session_metrics": state.session_metric_summary(session_id=args.session_id),
|
||||
"request_metrics": state.session_request_metrics(session_id=server_session_id),
|
||||
"session_metrics": state.session_metric_summary(session_id=server_session_id),
|
||||
"lifecycle_counts_ok": lifecycle_counts_ok,
|
||||
"validation_mode": validation_mode,
|
||||
"scenario": scenario,
|
||||
@@ -1773,7 +1765,6 @@ def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--session-id", help="Use an explicit public session ID, including for close/reopen tests.")
|
||||
parser.add_argument("--input-wav", required=True)
|
||||
parser.add_argument("--ref-audio", help="Optional WAV used as the MiniCPM-o voice prompt.")
|
||||
parser.add_argument(
|
||||
|
||||
@@ -17,7 +17,6 @@ import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
@@ -266,8 +265,6 @@ async def _run_demo_process(
|
||||
str(args.chunk_ms),
|
||||
"--timeout-s",
|
||||
str(args.timeout_s),
|
||||
"--session-id",
|
||||
f"duplex-pair-{label}-{uuid.uuid4().hex}",
|
||||
]
|
||||
if args.no_realtime_pacing:
|
||||
command.append("--no-realtime-pacing")
|
||||
|
||||
@@ -21,7 +21,7 @@ from websockets.asyncio.client import ClientConnection
|
||||
from websockets.exceptions import ConnectionClosed
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPO_ROOT = SCRIPT_DIR.parents[3]
|
||||
REPO_ROOT = SCRIPT_DIR.parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
@@ -42,6 +42,15 @@ def _url_with_model(*args, **kwargs) -> str:
|
||||
return _scenario_module()._url_with_model(*args, **kwargs)
|
||||
|
||||
|
||||
def _created_session_id(created: dict[str, object]) -> str:
|
||||
"""The server-allocated session id announced by ``session.created``."""
|
||||
session = created.get("session")
|
||||
session_id = session.get("id") if isinstance(session, dict) else None
|
||||
if not isinstance(session_id, str) or not session_id:
|
||||
raise RuntimeError("session.created omitted the server-allocated session id")
|
||||
return session_id
|
||||
|
||||
|
||||
async def run_demo(args):
|
||||
return await _scenario_module().run_demo(args)
|
||||
|
||||
@@ -217,7 +226,6 @@ async def _open_admission_session(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
ws = await websockets.connect(url, max_size=64 * 1024 * 1024)
|
||||
await ws.send(
|
||||
@@ -225,10 +233,8 @@ async def _open_admission_session(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"native_duplex": True},
|
||||
**({"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) else {}),
|
||||
},
|
||||
}
|
||||
@@ -255,12 +261,10 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
|
||||
for index in range(limit):
|
||||
accepted.append(await _open_admission_session(args, f"{prefix}-accepted-{index}"))
|
||||
|
||||
overflow_id = f"{prefix}-overflow"
|
||||
overflow_url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=overflow_id,
|
||||
)
|
||||
async with websockets.connect(overflow_url, max_size=64 * 1024 * 1024) as overflow:
|
||||
await overflow.send(
|
||||
@@ -268,10 +272,8 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": overflow_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
@@ -326,7 +328,6 @@ async def _resume_probe(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024) as first:
|
||||
await first.send(
|
||||
@@ -334,10 +335,8 @@ async def _resume_probe(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
@@ -348,10 +347,10 @@ async def _resume_probe(
|
||||
)
|
||||
)
|
||||
created, first_events = await _receive_until(first, "session.created", timeout_s=args.timeout_s)
|
||||
session_id = _created_session_id(created)
|
||||
token = created.get("resume_token")
|
||||
incarnation = created.get("incarnation")
|
||||
generation = created.get("attachment_generation")
|
||||
if not isinstance(token, str) or not isinstance(incarnation, int):
|
||||
if not isinstance(token, str):
|
||||
raise RuntimeError("session.created omitted resumable credentials")
|
||||
last_seq = max(_server_event_sequences(first_events), default=0)
|
||||
|
||||
@@ -366,7 +365,6 @@ async def _resume_probe(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": session_id,
|
||||
"incarnation": incarnation,
|
||||
"resume_token": token,
|
||||
"last_received_server_event_seq": last_seq,
|
||||
}
|
||||
@@ -429,7 +427,6 @@ async def _takeover_probe(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
first = await websockets.connect(url, max_size=64 * 1024 * 1024)
|
||||
second = None
|
||||
@@ -439,10 +436,8 @@ async def _takeover_probe(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
@@ -453,10 +448,10 @@ async def _takeover_probe(
|
||||
)
|
||||
)
|
||||
created, first_events = await _receive_until(first, "session.created", timeout_s=args.timeout_s)
|
||||
session_id = _created_session_id(created)
|
||||
token = created.get("resume_token")
|
||||
incarnation = created.get("incarnation")
|
||||
generation = created.get("attachment_generation")
|
||||
if not isinstance(token, str) or not isinstance(incarnation, int) or not isinstance(generation, int):
|
||||
if not isinstance(token, str) or not isinstance(generation, int):
|
||||
raise RuntimeError("session.created omitted takeover credentials")
|
||||
last_seq = max(_server_event_sequences(first_events), default=0)
|
||||
|
||||
@@ -466,7 +461,6 @@ async def _takeover_probe(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": session_id,
|
||||
"incarnation": incarnation,
|
||||
"resume_token": token,
|
||||
"last_received_server_event_seq": last_seq,
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
@@ -412,8 +411,6 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]:
|
||||
str(args.chunk_ms),
|
||||
"--timeout-s",
|
||||
str(args.timeout_s),
|
||||
"--session-id",
|
||||
f"duplex-soft-interrupt-{uuid.uuid4().hex}",
|
||||
]
|
||||
command.extend(["--ref-audio", str(_canonical_path(args.ref_audio))])
|
||||
temperature = getattr(args, "temperature", None)
|
||||
|
||||
@@ -12,7 +12,6 @@ incremental playback acks, a forced transport drop recovered by automatic
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -47,7 +46,6 @@ async def _run_live_session(*, url: str, model: str, ref_audio: Path, input_wav:
|
||||
url,
|
||||
model=model,
|
||||
config=config,
|
||||
session_id=f"duplex-client-live-{uuid.uuid4().hex}",
|
||||
reconnect=ReconnectPolicy(max_attempts=5, backoff_s=(0.25, 1.0)),
|
||||
# Short interval so the periodic heartbeat path runs during the
|
||||
# session — including across the forced transport drop below.
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -90,25 +90,16 @@ async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: f
|
||||
|
||||
|
||||
async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]:
|
||||
session_id = f"duplex-ci-protocol-{uuid.uuid4().hex}"
|
||||
websocket_url = build_realtime_url(
|
||||
url,
|
||||
model,
|
||||
autostart=False,
|
||||
session_id=session_id,
|
||||
extra_query={"native_duplex": "1"},
|
||||
)
|
||||
websocket_url = build_realtime_url(url, model, autostart=False)
|
||||
async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": model,
|
||||
"modalities": ["audio", "text"],
|
||||
"ref_audio": _ref_audio_data_url(str(ref_audio)),
|
||||
"extra_body": {"native_duplex": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -123,6 +114,170 @@ async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[
|
||||
return events
|
||||
|
||||
|
||||
async def _run_text_only_response_create(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
ref_audio: Path,
|
||||
text: str,
|
||||
timeout_s: float = 120.0,
|
||||
) -> dict[str, object]:
|
||||
"""Drive a text prompt with no audio and return the server's answer.
|
||||
|
||||
A Realtime client asks for speech from text with a user message item
|
||||
followed by ``response.create``. A model-native duplex session generates
|
||||
from audio units, so this returns whatever the server answers rather than
|
||||
assuming a response.
|
||||
"""
|
||||
websocket_url = build_realtime_url(url, model, autostart=False)
|
||||
async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"model": model,
|
||||
"modalities": ["audio", "text"],
|
||||
"output_audio_format": "pcm16",
|
||||
"turn_detection": None,
|
||||
"temperature": 0.0,
|
||||
"ref_audio": _ref_audio_data_url(str(ref_audio)),
|
||||
"extra_body": {"auto_response": False},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
await _receive_protocol_events(ws, {"session.created", "session.updated"}, timeout_s=60)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": text}],
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
await ws.send(json.dumps({"type": "response.create"}))
|
||||
|
||||
async def receive_outcome() -> dict[str, object]:
|
||||
while True:
|
||||
raw = await ws.recv()
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
event = json.loads(raw)
|
||||
if isinstance(event, dict) and event.get("type") in {"error", "response.done"}:
|
||||
return event
|
||||
|
||||
return await asyncio.wait_for(receive_outcome(), timeout=timeout_s)
|
||||
|
||||
|
||||
async def _run_seeded_text_to_audio(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
ref_audio: Path | None,
|
||||
text: str,
|
||||
modalities: tuple[str, ...] = ("audio", "text"),
|
||||
silence_seconds: float = 12.0,
|
||||
timeout_s: float = 180.0,
|
||||
) -> dict[str, object]:
|
||||
"""Speak a seeded text: the duplex route's text-to-speech shape.
|
||||
|
||||
A model-native session takes its text once, in the session context
|
||||
(``duplex_initial_user_text``), and then generates per audio unit. Silence
|
||||
carries no content of its own, so it only advances the clock and lets the
|
||||
model answer the seeded turn.
|
||||
"""
|
||||
websocket_url = build_realtime_url(url, model, autostart=False)
|
||||
audio_bytes = 0
|
||||
transcript: list[str] = []
|
||||
output_text: list[str] = []
|
||||
seen: list[str] = []
|
||||
session_payload: dict[str, object] = {
|
||||
"model": model,
|
||||
"modalities": list(modalities),
|
||||
"input_audio_format": "pcm16",
|
||||
"output_audio_format": "pcm16",
|
||||
"turn_detection": None,
|
||||
"temperature": 0.0,
|
||||
"extra_body": {
|
||||
"auto_response": True,
|
||||
"force_listen_count": 0,
|
||||
"duplex_initial_user_text": text,
|
||||
},
|
||||
}
|
||||
if ref_audio is not None:
|
||||
session_payload["ref_audio"] = _ref_audio_data_url(str(ref_audio))
|
||||
async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws:
|
||||
await ws.send(json.dumps({"type": "session.update", "session": session_payload}))
|
||||
|
||||
async def reader() -> None:
|
||||
nonlocal audio_bytes
|
||||
while True:
|
||||
raw = await ws.recv()
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
event = json.loads(raw)
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
event_type = event.get("type")
|
||||
if isinstance(event_type, str):
|
||||
seen.append(event_type)
|
||||
delta = event.get("delta")
|
||||
if event_type == "response.output_audio.delta" and isinstance(delta, str):
|
||||
audio_bytes += len(base64.b64decode(delta))
|
||||
elif event_type == "response.output_audio_transcript.delta" and isinstance(delta, str):
|
||||
transcript.append(delta)
|
||||
elif event_type == "response.output_text.delta" and isinstance(delta, str):
|
||||
output_text.append(delta)
|
||||
elif event_type == "error":
|
||||
raise AssertionError(f"seeded text turn received an error: {event}")
|
||||
|
||||
reader_task = asyncio.create_task(reader())
|
||||
silence = bytes(2 * 16_000 * 200 // 1000)
|
||||
sent_ms = 0
|
||||
try:
|
||||
while sent_ms < silence_seconds * 1000 and "response.done" not in seen:
|
||||
sent_ms += 200
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(silence).decode("ascii"),
|
||||
"input_audio_format": "pcm16",
|
||||
"sample_rate_hz": 16_000,
|
||||
"duration_ms": 200,
|
||||
"audio_end_ms": sent_ms,
|
||||
}
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(0.2)
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout_s
|
||||
while loop.time() < deadline and "response.done" not in seen:
|
||||
await asyncio.sleep(0.5)
|
||||
if reader_task.done():
|
||||
await reader_task
|
||||
# Close explicitly. Dropping the socket parks the session in its
|
||||
# disconnect grace, where it keeps holding an admission slot and
|
||||
# starves the tests that follow.
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
deadline = loop.time() + 30
|
||||
while loop.time() < deadline and "session.closed" not in seen:
|
||||
await asyncio.sleep(0.25)
|
||||
finally:
|
||||
reader_task.cancel()
|
||||
return {
|
||||
"audio_bytes": audio_bytes,
|
||||
"transcript": "".join(transcript),
|
||||
"output_text": "".join(output_text),
|
||||
"event_types": seen,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@@ -205,6 +360,215 @@ def test_duplex_single_session_video_input(omni_server, tmp_path: Path) -> None:
|
||||
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
@pytest.mark.parametrize(
|
||||
("locale", "text"),
|
||||
[
|
||||
("en", "Please say exactly: the quick brown fox jumps over the lazy dog."),
|
||||
("zh", "请朗读：今天天气很好，我们一起去公园散步。"),
|
||||
],
|
||||
)
|
||||
def test_duplex_seeded_text_to_audio(omni_server, locale: str, text: str) -> None:
|
||||
"""text -> audio over the duplex route, in both locales.
|
||||
|
||||
This is the case the deleted turn-based ``text -> audio`` tests covered.
|
||||
It is also the shape the Seed-TTS Realtime backend uses, so it guards the
|
||||
benchmark path as well as the modality.
|
||||
"""
|
||||
result = asyncio.run(
|
||||
_run_seeded_text_to_audio(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
|
||||
assert "response.done" in result["event_types"], result["event_types"]
|
||||
assert int(result["audio_bytes"]) > 0, f"{locale} seeded text produced no audio"
|
||||
assert str(result["transcript"]).strip(), f"{locale} seeded text produced audio with no transcript"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_seeded_text_to_text_needs_no_reference_voice(omni_server) -> None:
|
||||
"""text -> text, the one duplex session that opens without a reference voice.
|
||||
|
||||
``ref_audio`` is required only when the session asks for audio output, so a
|
||||
``modalities: ["text"]`` session is the duplex equivalent of the deleted
|
||||
turn-based ``text -> text`` case. The model is model-native and still
|
||||
speaks its answer, so this asserts the text side and the absence of the
|
||||
``ref_audio_required`` rejection, not the absence of audio.
|
||||
"""
|
||||
result = asyncio.run(
|
||||
_run_seeded_text_to_audio(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=None,
|
||||
text="What is the capital of France? Answer in one short sentence.",
|
||||
modalities=("text",),
|
||||
)
|
||||
)
|
||||
|
||||
assert "response.done" in result["event_types"], result["event_types"]
|
||||
produced_text = str(result["output_text"]) or str(result["transcript"])
|
||||
assert produced_text.strip(), f"text-only session produced nothing: {result['event_types']}"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_seeded_text_to_long_audio_output(omni_server) -> None:
|
||||
"""A longer answer, so Code2Wav runs across many frames rather than one.
|
||||
|
||||
Replaces the deleted ``text_to_audio_long_output`` case. The assertion is
|
||||
on the amount of audio rather than its content: the point is that a long
|
||||
answer streams to completion, not that the model says any given word.
|
||||
"""
|
||||
result = asyncio.run(
|
||||
_run_seeded_text_to_audio(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
text="What is the capital of China? Answer in about 40 words.",
|
||||
silence_seconds=30.0,
|
||||
timeout_s=240.0,
|
||||
)
|
||||
)
|
||||
|
||||
assert "response.done" in result["event_types"], result["event_types"]
|
||||
# 24 kHz mono pcm16: 2 s of speech is 96000 bytes, comfortably more than a
|
||||
# single Code2Wav frame and well under a 40-word answer.
|
||||
assert int(result["audio_bytes"]) > 96_000, f"expected a long answer, got {result['audio_bytes']} bytes"
|
||||
assert str(result["transcript"]).strip()
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_seeded_long_form_generation(omni_server) -> None:
|
||||
"""Long-form Chinese generation, replacing the deleted long-form case."""
|
||||
result = asyncio.run(
|
||||
_run_seeded_text_to_audio(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
text="帮我讲一个100字的故事。",
|
||||
silence_seconds=30.0,
|
||||
timeout_s=240.0,
|
||||
)
|
||||
)
|
||||
|
||||
assert "response.done" in result["event_types"], result["event_types"]
|
||||
assert int(result["audio_bytes"]) > 96_000, f"expected long-form audio, got {result['audio_bytes']} bytes"
|
||||
assert str(result["transcript"]).strip()
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_sequential_sessions_are_independent(omni_server) -> None:
|
||||
"""A second session must answer its own prompt, not replay the first one.
|
||||
|
||||
Replaces the deleted ``sequential_requests_independent`` case. Sessions are
|
||||
engine-resident now, so this is the check that one session's context does
|
||||
not leak into the next one on the same stages.
|
||||
"""
|
||||
first = asyncio.run(
|
||||
_run_seeded_text_to_audio(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
text="What is the capital of France? Answer in one short sentence.",
|
||||
)
|
||||
)
|
||||
second = asyncio.run(
|
||||
_run_seeded_text_to_audio(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
text="What is the capital of China? Answer in one short sentence.",
|
||||
)
|
||||
)
|
||||
|
||||
for result in (first, second):
|
||||
assert "response.done" in result["event_types"], result["event_types"]
|
||||
assert int(result["audio_bytes"]) > 0
|
||||
assert str(result["transcript"]).strip()
|
||||
|
||||
# Different prompts must not produce the same answer: that would mean the
|
||||
# second session inherited the first one's context.
|
||||
assert str(first["transcript"]).strip() != str(second["transcript"]).strip()
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_text_only_response_create_is_rejected(omni_server) -> None:
|
||||
"""A duplex session generates from audio: a text-only prompt is refused, not ignored.
|
||||
|
||||
Every session on this route is model-native, and such a model decides per
|
||||
audio unit whether to speak: conversation items are context, not a turn.
|
||||
Refusing is the point -- opening a response the model never fills costs the
|
||||
caller the session's whole idle timeout to discover. A text prompt reaches
|
||||
the model through the session's seeded opening turn
|
||||
(``initial_user_text``) instead, which is what ``/v1/chat/completions``
|
||||
uses and what the Seed-TTS Realtime backend has to be ported to.
|
||||
"""
|
||||
outcome = asyncio.run(
|
||||
_run_text_only_response_create(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=resolve_ref_audio(),
|
||||
text="Please say: the quick brown fox jumps over the lazy dog.",
|
||||
)
|
||||
)
|
||||
|
||||
assert outcome.get("type") == "error", outcome
|
||||
error = outcome.get("error")
|
||||
assert isinstance(error, dict), outcome
|
||||
assert error.get("code") == "text_only_turn_unsupported", outcome
|
||||
assert error.get("type") == "invalid_request_error", outcome
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_single_session_still_image_input(omni_server, tmp_path: Path) -> None:
|
||||
"""A spoken turn with one still image attached.
|
||||
|
||||
The still-image case is a one-element frame list, which the client repeats
|
||||
on every unit of the turn: unlike the camera track it never advances, so
|
||||
this covers the image path the turn-based ``image -> text + audio`` request
|
||||
used to cover.
|
||||
"""
|
||||
args = demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(),
|
||||
output_dir=tmp_path / "still_image",
|
||||
)
|
||||
args.turns = 2
|
||||
args.turn_duration_ms = [args.first_turn_ms] * args.turns
|
||||
args.video_frames_b64 = duplex_camera_frames(seconds=1, cache_dir=tmp_path / "still")[:1]
|
||||
|
||||
result = asyncio.run(run_demo(args))
|
||||
|
||||
assert result["ok"] is True
|
||||
assert result["video_frame_count"] == 1
|
||||
assert result["video_stacked_frame_count"] == 0
|
||||
_assert_positive_int(result["audio_delta_count"])
|
||||
assert result["done_count"] == 2
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
assert result["transcript_delta_done_ok"] is True
|
||||
_assert_request_metrics(result["request_metrics"], expected_count=2)
|
||||
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
|
||||
@@ -30,7 +30,7 @@ from tests.e2e.online_serving.run_minicpmo_realtime_duplex_soft_interrupt import
|
||||
run_soft_interrupt,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
from vllm_omni.engine.duplex.vad import (
|
||||
SILERO_VAD_FILENAME,
|
||||
SILERO_VAD_REPO_ID,
|
||||
SILERO_VAD_REVISION,
|
||||
|
||||
@@ -152,7 +152,6 @@ def test_realtime_duplex_demo_pair_launches_demo_processes_concurrently(tmp_path
|
||||
"parser.add_argument('--output-dir')",
|
||||
"parser.add_argument('--chunk-ms')",
|
||||
"parser.add_argument('--timeout-s')",
|
||||
"parser.add_argument('--session-id')",
|
||||
"parser.add_argument('--ref-audio')",
|
||||
"parser.add_argument('--require-audio', action='store_true')",
|
||||
"parser.add_argument('--no-realtime-pacing', action='store_true')",
|
||||
@@ -1027,7 +1026,7 @@ def test_realtime_duplex_demo_resolves_distinct_turn_inputs():
|
||||
demo._turn_input_paths(primary, ["second.wav"], turns=3)
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_explicitly_enables_native_runtime_before_connect():
|
||||
def test_realtime_duplex_demo_url_carries_no_model_opt_in_or_session_id():
|
||||
demo = _load_demo_module()
|
||||
|
||||
url = demo._url_with_model(
|
||||
@@ -1036,23 +1035,12 @@ def test_realtime_duplex_demo_explicitly_enables_native_runtime_before_connect()
|
||||
)
|
||||
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
assert query["native_duplex"] == ["1"]
|
||||
assert query["duplex"] == ["1"]
|
||||
assert "native_duplex" not in query
|
||||
assert "session_id" not in query
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_explicit_session_id_reaches_autostart_query():
|
||||
demo = _load_demo_module()
|
||||
|
||||
url = demo._url_with_model(
|
||||
"ws://localhost:8099/v1/realtime?duplex=1",
|
||||
"openbmb/MiniCPM-o-4_5",
|
||||
session_id="reopen-e2e",
|
||||
)
|
||||
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
assert query["session_id"] == ["reopen-e2e"]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_session_update_uses_explicit_session_id():
|
||||
def test_realtime_duplex_demo_session_update_leaves_session_id_to_the_server():
|
||||
demo = _load_demo_module()
|
||||
|
||||
event = demo._session_update_event(
|
||||
@@ -1066,8 +1054,8 @@ def test_realtime_duplex_demo_session_update_uses_explicit_session_id():
|
||||
|
||||
assert event["type"] == "session.update"
|
||||
assert "session_id" not in event
|
||||
assert event["session"]["session_id"] == "reopen-e2e"
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "session_id" not in event["session"]
|
||||
assert "native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_duplex_demo_response_required_uses_deterministic_sampling():
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ import base64
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.audio import convert_input_audio_with_rate
|
||||
from vllm_omni.engine.duplex.audio import convert_input_audio_with_rate
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -0,0 +1,453 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""OpenAI Realtime client events -> typed ``DuplexCommand`` objects and their mailbox payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.commands import (
|
||||
REALTIME_COMMAND_TYPES,
|
||||
AckPlayback,
|
||||
AppendAudio,
|
||||
AppendText,
|
||||
BargeIn,
|
||||
CancelInput,
|
||||
CancelResponse,
|
||||
ClearInput,
|
||||
ClearOutputAudio,
|
||||
CloseSession,
|
||||
Commit,
|
||||
CreateItem,
|
||||
CreateResponse,
|
||||
DeleteItem,
|
||||
DuplexCommand,
|
||||
DuplexCommandError,
|
||||
Heartbeat,
|
||||
SignalTurn,
|
||||
TruncateItem,
|
||||
UpdateSession,
|
||||
command_from_realtime,
|
||||
)
|
||||
from vllm_omni.engine.duplex.realtime_commands import (
|
||||
RealtimeInputDefaults,
|
||||
translate_realtime_command,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_LOUD_PCM16 = base64.b64encode(b"\x00\x10" * 8).decode("ascii")
|
||||
_SILENT_PCM16 = base64.b64encode(b"\x00\x00" * 8).decode("ascii")
|
||||
_JPEG_FRAME = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 12).decode("ascii")
|
||||
|
||||
_MINIMAL_PAYLOADS: dict[str, tuple[dict[str, object], type[DuplexCommand]]] = {
|
||||
"input_audio_buffer.append": ({"audio": _LOUD_PCM16}, AppendAudio),
|
||||
"input_audio_buffer.commit": ({}, Commit),
|
||||
"input_audio_buffer.clear": ({}, ClearInput),
|
||||
"output_audio_buffer.clear": ({}, ClearOutputAudio),
|
||||
"response.create": ({}, CreateResponse),
|
||||
"response.cancel": ({}, CancelResponse),
|
||||
"conversation.item.create": ({"item": {"type": "message", "role": "user", "content": []}}, CreateItem),
|
||||
"conversation.item.delete": ({"item_id": "item_1"}, DeleteItem),
|
||||
"conversation.item.truncate": ({"item_id": "item_1", "audio_end_ms": 100}, TruncateItem),
|
||||
"session.update": ({"session": {"instructions": "hi"}}, UpdateSession),
|
||||
"playback.ack": ({"played_ms": 10}, AckPlayback),
|
||||
"session.heartbeat": ({}, Heartbeat),
|
||||
"session.close": ({}, CloseSession),
|
||||
"turn.signal": ({"event": "user_started"}, SignalTurn),
|
||||
"input.text.append": ({"text": "hello"}, AppendText),
|
||||
"input.cancel": ({}, CancelInput),
|
||||
"barge_in": ({}, BargeIn),
|
||||
}
|
||||
|
||||
|
||||
def test_every_realtime_command_type_has_a_mapping_case():
|
||||
assert set(_MINIMAL_PAYLOADS) == set(REALTIME_COMMAND_TYPES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event_type", sorted(REALTIME_COMMAND_TYPES))
|
||||
def test_command_from_realtime_maps_each_type_to_its_dataclass_and_propagates_event_id(event_type: str):
|
||||
body, expected_cls = _MINIMAL_PAYLOADS[event_type]
|
||||
|
||||
command = command_from_realtime({"type": event_type, "event_id": f"evt-{event_type}", **body})
|
||||
|
||||
assert type(command) is expected_cls
|
||||
assert command.event_id == f"evt-{event_type}"
|
||||
payload = command.payload()
|
||||
assert payload["type"] == expected_cls.type
|
||||
assert payload["realtime_event_id"] == f"evt-{event_type}"
|
||||
assert "event_id" not in payload
|
||||
|
||||
anonymous = command_from_realtime({"type": event_type, **body})
|
||||
assert anonymous.event_id is None
|
||||
assert "realtime_event_id" not in anonymous.payload()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("alias", "expected_cls", "body"),
|
||||
[
|
||||
("push_text", AppendText, {"text": "hello"}),
|
||||
("input_text.append", AppendText, {"text": "hello"}),
|
||||
("signal_turn", SignalTurn, {"event": "user_started"}),
|
||||
("close_session", CloseSession, {}),
|
||||
("close", CloseSession, {}),
|
||||
("audio.playback_ack", AckPlayback, {"played_ms": 10}),
|
||||
("input.commit", Commit, {}),
|
||||
],
|
||||
)
|
||||
def test_pre_realtime_client_event_aliases_map_to_their_canonical_command(
|
||||
alias: str, expected_cls: type[DuplexCommand], body: dict[str, object]
|
||||
):
|
||||
assert type(command_from_realtime({"type": alias, **body})) is expected_cls
|
||||
|
||||
|
||||
def test_append_audio_decodes_and_converts_pcm16_to_16k_float32():
|
||||
command = command_from_realtime(
|
||||
{"type": "input_audio_buffer.append", "audio": _LOUD_PCM16, "format": "pcm16", "sample_rate_hz": 16000}
|
||||
)
|
||||
|
||||
assert isinstance(command, AppendAudio)
|
||||
assert command.format == "pcm_f32le"
|
||||
assert command.sample_rate_hz == 16000
|
||||
samples = np.frombuffer(command.audio, dtype="<f4")
|
||||
assert samples.shape == (8,)
|
||||
assert np.allclose(samples, 4096 / 32768.0)
|
||||
assert command.is_speech is True
|
||||
assert command.video_frames == ()
|
||||
assert command.duration_ms is None
|
||||
assert command.audio_end_ms is None
|
||||
|
||||
|
||||
def test_append_audio_defaults_come_from_the_session_payload():
|
||||
defaults = RealtimeInputDefaults().with_session_payload({"input_audio_format": "pcm16", "sample_rate_hz": 8000})
|
||||
assert defaults.input_audio_format == "pcm16"
|
||||
assert defaults.input_sample_rate_hz == 8000
|
||||
|
||||
command = translate_realtime_command({"type": "input_audio_buffer.append", "audio": _LOUD_PCM16}, defaults=defaults)
|
||||
|
||||
assert isinstance(command, AppendAudio)
|
||||
assert command.format == "pcm_f32le"
|
||||
# 8 kHz input is resampled to the model's 16 kHz float stream.
|
||||
assert command.sample_rate_hz == 16000
|
||||
assert len(command.audio) > 8 * 4
|
||||
|
||||
passthrough = translate_realtime_command(
|
||||
{"type": "input_audio_buffer.append", "audio": base64.b64encode(np.zeros(4, dtype="<f4").tobytes()).decode()},
|
||||
defaults=RealtimeInputDefaults(input_audio_format="pcm_f32le", input_sample_rate_hz=16000),
|
||||
)
|
||||
assert passthrough.format == "pcm_f32le"
|
||||
assert passthrough.audio == np.zeros(4, dtype="<f4").tobytes()
|
||||
|
||||
|
||||
def test_append_audio_speech_classification_uses_hints_then_rms():
|
||||
silent = command_from_realtime({"type": "input_audio_buffer.append", "audio": _SILENT_PCM16})
|
||||
assert silent.is_speech is False
|
||||
|
||||
forced = command_from_realtime({"type": "input_audio_buffer.append", "audio": _SILENT_PCM16, "is_speech": True})
|
||||
assert forced.is_speech is True
|
||||
assert forced.hints == {"is_speech": True}
|
||||
|
||||
vad = command_from_realtime(
|
||||
{"type": "input_audio_buffer.append", "audio": _LOUD_PCM16, "vad": {"speech_probability": 0.1}}
|
||||
)
|
||||
assert vad.is_speech is False
|
||||
assert vad.hints["vad"] == {"speech_probability": 0.1}
|
||||
|
||||
|
||||
def test_append_audio_carries_wire_hints_and_video_frames():
|
||||
command = command_from_realtime(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"event_id": "evt-append",
|
||||
"audio": _LOUD_PCM16,
|
||||
"transcript": "hi there",
|
||||
"duration_ms": 1000,
|
||||
"audio_end_ms": 3000,
|
||||
"video_frames": [_JPEG_FRAME],
|
||||
"unknown_key": "dropped",
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(command, AppendAudio)
|
||||
assert command.duration_ms == 1000
|
||||
assert command.audio_end_ms == 3000
|
||||
assert command.video_frames == (_JPEG_FRAME,)
|
||||
assert dict(command.hints) == {"transcript": "hi there", "duration_ms": 1000, "audio_end_ms": 3000}
|
||||
|
||||
payload = command.payload()
|
||||
assert payload["type"] == "input_audio_buffer.append"
|
||||
assert payload["realtime_event_id"] == "evt-append"
|
||||
assert payload["audio"] == base64.b64encode(command.audio).decode("ascii")
|
||||
assert payload["format"] == "pcm_f32le"
|
||||
assert payload["sample_rate_hz"] == 16000
|
||||
assert payload["is_speech"] is True
|
||||
assert payload["transcript"] == "hi there"
|
||||
assert payload["duration_ms"] == 1000
|
||||
assert payload["audio_end_ms"] == 3000
|
||||
assert payload["video_frames"] == [_JPEG_FRAME]
|
||||
assert "hints" not in payload
|
||||
assert "unknown_key" not in payload
|
||||
|
||||
|
||||
def test_append_audio_payload_re_encodes_bytes_and_flattens_hints():
|
||||
command = AppendAudio(audio=b"\x01\x02", hints={"vad": {"is_speech": True}}, is_speech=True)
|
||||
|
||||
assert command.payload() == {
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": "AQI=",
|
||||
"format": "pcm16",
|
||||
"is_speech": True,
|
||||
"vad": {"is_speech": True},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "code"),
|
||||
[
|
||||
({"audio": _LOUD_PCM16, "format": "mp3"}, "unsupported_audio_format"),
|
||||
({"audio": "@@@@", "format": "pcm_f32le"}, "bad_audio"),
|
||||
({"audio": _LOUD_PCM16, "video_frames": ["aGVsbG8="]}, "invalid_video_frames"),
|
||||
({"audio": _LOUD_PCM16, "video_frames": [_JPEG_FRAME], "max_slice_nums": 4}, "invalid_video_frames"),
|
||||
({"audio": _LOUD_PCM16, "format": "pcm16", "sample_rate_hz": 3}, "bad_event"),
|
||||
],
|
||||
)
|
||||
def test_malformed_append_raises_command_error_with_code(body: dict[str, object], code: str):
|
||||
with pytest.raises(DuplexCommandError) as excinfo:
|
||||
command_from_realtime({"type": "input_audio_buffer.append", "event_id": "evt-bad", **body})
|
||||
|
||||
assert excinfo.value.code == code
|
||||
assert excinfo.value.event_id == "evt-bad"
|
||||
|
||||
|
||||
# ---- other malformed payloads ----
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "code"),
|
||||
[
|
||||
({}, "bad_event"),
|
||||
({"type": 7}, "bad_event"),
|
||||
({"type": "session.resume", "session_id": "s", "resume_token": "t"}, "unknown_event"),
|
||||
({"type": "session.event_ack", "server_event_seq": 1}, "unknown_event"),
|
||||
({"type": "conversation.item.frobnicate", "item_id": "item_1"}, "unknown_event"),
|
||||
({"type": "conversation.item.create"}, "bad_event"),
|
||||
(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_audio", "format": "mp3"}]},
|
||||
},
|
||||
"unsupported_audio_format",
|
||||
),
|
||||
({"type": "conversation.item.delete"}, "missing_item_id"),
|
||||
({"type": "conversation.item.truncate", "audio_end_ms": 1}, "missing_item_id"),
|
||||
({"type": "conversation.item.truncate", "item_id": "item_1"}, "bad_event"),
|
||||
({"type": "session.update", "session": {"output_audio_format": "opus"}}, "unsupported_audio_format"),
|
||||
(
|
||||
{"type": "session.update", "session": {"turn_detection": {"type": "semantic_vad"}}},
|
||||
"unsupported_turn_detection",
|
||||
),
|
||||
({"type": "response.create", "response": {"output_audio_format": "opus"}}, "unsupported_audio_format"),
|
||||
({"type": "playback.ack"}, "bad_event"),
|
||||
({"type": "input.text.append"}, "bad_event"),
|
||||
({"type": "turn.signal"}, "bad_event"),
|
||||
({"type": "turn.signal", "event": "conversation.item.delete", "payload": {}}, "missing_item_id"),
|
||||
({"type": "turn.signal", "event": "conversation.item.truncate", "payload": {"item_id": "i"}}, "bad_event"),
|
||||
],
|
||||
)
|
||||
def test_malformed_payloads_raise_command_error_with_code(payload: dict[str, object], code: str):
|
||||
with pytest.raises(DuplexCommandError) as excinfo:
|
||||
command_from_realtime({"event_id": "evt-bad", **payload})
|
||||
|
||||
assert excinfo.value.code == code
|
||||
assert excinfo.value.event_id == "evt-bad"
|
||||
|
||||
|
||||
# ---- payload() rendering ----
|
||||
|
||||
|
||||
def test_commit_maps_create_response_to_response_create():
|
||||
command = command_from_realtime(
|
||||
{"type": "input_audio_buffer.commit", "event_id": "evt-commit", "response_create": True, "final": False}
|
||||
)
|
||||
|
||||
assert command == Commit(event_id="evt-commit", final=False, create_response=True)
|
||||
assert command.payload() == {
|
||||
"type": "input_audio_buffer.commit",
|
||||
"realtime_event_id": "evt-commit",
|
||||
"final": False,
|
||||
"response_create": True,
|
||||
}
|
||||
assert Commit().payload() == {"type": "input_audio_buffer.commit", "final": True}
|
||||
assert Commit(is_speech=False, realtime_item_id="item_1").payload() == {
|
||||
"type": "input_audio_buffer.commit",
|
||||
"final": True,
|
||||
"is_speech": False,
|
||||
"realtime_item_id": "item_1",
|
||||
}
|
||||
assert (
|
||||
command_from_realtime({"type": "input_audio_buffer.commit", "create_response": False}).create_response is False
|
||||
)
|
||||
|
||||
|
||||
def test_create_response_renders_options_as_response_object():
|
||||
command = command_from_realtime({"type": "response.create", "response": {"modalities": ["text"]}})
|
||||
|
||||
assert isinstance(command, CreateResponse)
|
||||
assert dict(command.options) == {"modalities": ["text"]}
|
||||
assert command.payload() == {"type": "response.create", "response": {"modalities": ["text"]}}
|
||||
assert command_from_realtime({"type": "response.create"}).payload() == {"type": "response.create", "response": {}}
|
||||
|
||||
|
||||
def test_cancel_and_clear_commands_keep_optional_response_id():
|
||||
assert command_from_realtime({"type": "response.cancel", "response_id": ""}).response_id is None
|
||||
cancel = command_from_realtime({"type": "response.cancel", "response_id": "resp_1"})
|
||||
assert cancel == CancelResponse(response_id="resp_1")
|
||||
assert cancel.payload() == {"type": "response.cancel", "response_id": "resp_1"}
|
||||
clear = command_from_realtime({"type": "output_audio_buffer.clear", "response_id": "resp_2"})
|
||||
assert clear == ClearOutputAudio(response_id="resp_2")
|
||||
assert ClearInput().payload() == {"type": "input_audio_buffer.clear"}
|
||||
|
||||
|
||||
def test_update_session_renders_as_turn_signal_payload():
|
||||
command = command_from_realtime(
|
||||
{"type": "session.update", "session": {"instructions": "be brief", "voice": "alloy"}}
|
||||
)
|
||||
|
||||
assert command == UpdateSession(patch={"instructions": "be brief", "voice": "alloy"})
|
||||
assert command.payload() == {
|
||||
"type": "turn.signal",
|
||||
"event": "session.update",
|
||||
"payload": {"instructions": "be brief", "voice": "alloy"},
|
||||
}
|
||||
|
||||
|
||||
def test_create_item_normalizes_item_and_renders_as_turn_signal_payload():
|
||||
command = command_from_realtime(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"previous_item_id": "item_0",
|
||||
"item": {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}]},
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(command, CreateItem)
|
||||
assert command.previous_item_id == "item_0"
|
||||
assert command.item["id"].startswith("item_")
|
||||
assert command.item["object"] == "realtime.item"
|
||||
assert command.item["status"] == "completed"
|
||||
payload = command.payload()
|
||||
assert payload["type"] == "turn.signal"
|
||||
assert payload["event"] == "conversation.item.create"
|
||||
assert payload["payload"] == {"item": dict(command.item), "previous_item_id": "item_0"}
|
||||
assert "item" not in payload
|
||||
assert "previous_item_id" not in payload
|
||||
|
||||
without_previous = command_from_realtime({"type": "conversation.item.create", "item": {"content": "x"}})
|
||||
assert without_previous.item["role"] == "user"
|
||||
assert without_previous.item["content"] == []
|
||||
assert without_previous.payload()["payload"] == {"item": dict(without_previous.item)}
|
||||
|
||||
|
||||
def test_delete_and_truncate_render_as_turn_signal_payloads():
|
||||
delete = command_from_realtime({"type": "conversation.item.delete", "item_id": "item_1"})
|
||||
assert delete == DeleteItem(item_id="item_1")
|
||||
assert delete.payload() == {
|
||||
"type": "turn.signal",
|
||||
"event": "conversation.item.delete",
|
||||
"payload": {"item_id": "item_1"},
|
||||
}
|
||||
|
||||
truncate = command_from_realtime(
|
||||
{"type": "conversation.item.truncate", "item_id": "item_1", "audio_end_ms": 1500.0, "content_index": 1}
|
||||
)
|
||||
assert truncate == TruncateItem(item_id="item_1", audio_end_ms=1500, content_index=1)
|
||||
assert truncate.payload() == {
|
||||
"type": "turn.signal",
|
||||
"event": "conversation.item.truncate",
|
||||
"payload": {"item_id": "item_1", "audio_end_ms": 1500, "content_index": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_playback_ack_close_and_text_append_payloads():
|
||||
ack = command_from_realtime(
|
||||
{"type": "playback.ack", "played_ms": 1200.0, "committed_ms": 1000, "response_id": "resp_1", "item_id": ""}
|
||||
)
|
||||
assert ack == AckPlayback(played_ms=1200, committed_ms=1000, response_id="resp_1")
|
||||
assert ack.payload() == {
|
||||
"type": "playback.ack",
|
||||
"played_ms": 1200,
|
||||
"committed_ms": 1000,
|
||||
"response_id": "resp_1",
|
||||
}
|
||||
|
||||
assert command_from_realtime({"type": "session.close"}) == CloseSession(reason="client_close")
|
||||
assert command_from_realtime({"type": "session.close", "reason": "done"}).payload() == {
|
||||
"type": "session.close",
|
||||
"reason": "done",
|
||||
}
|
||||
assert command_from_realtime({"type": "input.text.append", "text": "hello"}).payload() == {
|
||||
"type": "input.text.append",
|
||||
"text": "hello",
|
||||
}
|
||||
assert Heartbeat().payload() == {"type": "session.heartbeat"}
|
||||
|
||||
|
||||
def test_turn_signal_renders_payload_only_when_present_and_dispatches_known_events():
|
||||
signal = command_from_realtime({"type": "turn.signal", "event": "user_started", "payload": {"source": "client"}})
|
||||
assert signal == SignalTurn(event="user_started", signal_payload={"source": "client"})
|
||||
assert signal.payload() == {"type": "turn.signal", "event": "user_started", "payload": {"source": "client"}}
|
||||
assert command_from_realtime({"type": "turn.signal", "event": "user_started"}).payload() == {
|
||||
"type": "turn.signal",
|
||||
"event": "user_started",
|
||||
}
|
||||
|
||||
assert command_from_realtime({"type": "turn.signal", "event": "barge_in"}) == BargeIn()
|
||||
assert command_from_realtime({"type": "turn.signal", "event": "input.cancel"}) == CancelInput()
|
||||
assert command_from_realtime(
|
||||
{"type": "turn.signal", "event": "response.cancel", "payload": {"response_id": "resp_1"}}
|
||||
) == CancelResponse(response_id="resp_1")
|
||||
assert command_from_realtime(
|
||||
{"type": "turn.signal", "event": "session.update", "payload": {"voice": "alloy"}}
|
||||
) == UpdateSession(patch={"voice": "alloy"})
|
||||
assert command_from_realtime(
|
||||
{"type": "turn.signal", "event": "conversation.item.delete", "payload": {"item_id": "item_1"}}
|
||||
) == DeleteItem(item_id="item_1")
|
||||
assert command_from_realtime(
|
||||
{"type": "turn.signal", "event": "conversation.item.truncate", "payload": {"item_id": "i", "audio_end_ms": 5}}
|
||||
) == TruncateItem(item_id="i", audio_end_ms=5)
|
||||
created = command_from_realtime(
|
||||
{"type": "turn.signal", "event": "conversation.item.create", "payload": {"item": {"role": "system"}}}
|
||||
)
|
||||
assert isinstance(created, CreateItem)
|
||||
assert created.item["role"] == "system"
|
||||
|
||||
|
||||
def test_raw_wire_hints_cannot_override_the_normalized_typed_fields():
|
||||
"""``build_append_audio`` normalizes; a client's raw value must not undo it.
|
||||
|
||||
A wire ``"is_speech": 0`` used to land in ``hints`` and overwrite the
|
||||
computed ``bool | None`` in the rendered payload, so the runner's
|
||||
silent-commit fast path (``event.get("is_speech") is False``) missed and
|
||||
overlap classification followed the unvalidated value.
|
||||
"""
|
||||
command = AppendAudio(
|
||||
audio=b"\x00\x00" * 8,
|
||||
is_speech=False,
|
||||
hints={"is_speech": 0, "rms": 0.001},
|
||||
)
|
||||
|
||||
payload = command.payload()
|
||||
|
||||
assert payload["is_speech"] is False, "the normalized field wins over the raw hint"
|
||||
assert payload["rms"] == 0.001, "a hint with no typed counterpart still comes through"
|
||||
|
||||
|
||||
def test_a_hint_survives_when_its_typed_field_is_unset():
|
||||
"""Unset typed fields are absent from the payload, so the hint is the only value."""
|
||||
command = AppendAudio(audio=b"\x00\x00" * 8, hints={"is_speech": True})
|
||||
|
||||
assert command.payload()["is_speech"] is True
|
||||
@@ -1,110 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.control_client import (
|
||||
DuplexControlClient,
|
||||
DuplexControlRequestError,
|
||||
)
|
||||
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
DuplexControlError,
|
||||
DuplexControlResultMessage,
|
||||
DuplexFence,
|
||||
ResumeDuplexSessionMessage,
|
||||
TouchDuplexSessionMessage,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _Transport:
|
||||
def __init__(self) -> None:
|
||||
self.calls = []
|
||||
|
||||
def execute(self, key, message, **kwargs):
|
||||
self.calls.append((key, message, kwargs))
|
||||
return DuplexControlResultMessage(
|
||||
control_id=message.control_id,
|
||||
fence=message.fence,
|
||||
operation="touch" if isinstance(message, TouchDuplexSessionMessage) else "resume",
|
||||
session_id=message.session_id,
|
||||
ok=True,
|
||||
stage_results=[],
|
||||
)
|
||||
|
||||
|
||||
class _UnsupportedTransport:
|
||||
def execute(self, key, message, **kwargs):
|
||||
del key, kwargs
|
||||
return DuplexControlResultMessage(
|
||||
control_id=message.control_id,
|
||||
fence=message.fence,
|
||||
operation="touch",
|
||||
session_id=message.session_id,
|
||||
ok=False,
|
||||
stage_results=[{"result": {"supported": False}}],
|
||||
unsupported_count=1,
|
||||
error=DuplexControlError(code="invalid_capability", message="unsupported", retryable=False),
|
||||
accepted_fence=DuplexFence(message.session_id, epoch=2),
|
||||
lease_generation=7,
|
||||
)
|
||||
|
||||
|
||||
def test_control_client_routes_touch_and_resume_by_control_id() -> None:
|
||||
transport = _Transport()
|
||||
control_ids = iter(("touch-id", "resume-id"))
|
||||
client = DuplexControlClient(transport, control_id_factory=lambda: next(control_ids))
|
||||
fence = DuplexFence("sid-client")
|
||||
|
||||
assert (
|
||||
client.touch(
|
||||
fence.session_id,
|
||||
fence=fence,
|
||||
activity=DuplexLeaseActivity.PLAYBACK_ACK,
|
||||
timeout=2.0,
|
||||
)["ok"]
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
client.resume(
|
||||
fence.session_id,
|
||||
fence=fence,
|
||||
expected_lease_generation=7,
|
||||
timeout=3.0,
|
||||
)["ok"]
|
||||
is True
|
||||
)
|
||||
|
||||
touch_key, touch_message, touch_kwargs = transport.calls[0]
|
||||
assert touch_key == ("duplex", "touch-id")
|
||||
assert isinstance(touch_message, TouchDuplexSessionMessage)
|
||||
assert touch_message.activity == DuplexLeaseActivity.PLAYBACK_ACK.value
|
||||
assert touch_kwargs["timeout"] == 2.0
|
||||
|
||||
resume_key, resume_message, resume_kwargs = transport.calls[1]
|
||||
assert resume_key == ("duplex", "resume-id")
|
||||
assert isinstance(resume_message, ResumeDuplexSessionMessage)
|
||||
assert resume_message.expected_lease_generation == 7
|
||||
assert resume_kwargs["timeout"] == 3.0
|
||||
|
||||
|
||||
def test_control_client_treats_ok_false_as_authoritative_failure() -> None:
|
||||
client = DuplexControlClient(_UnsupportedTransport(), control_id_factory=lambda: "unsupported")
|
||||
fence = DuplexFence("sid-unsupported")
|
||||
|
||||
with pytest.raises(DuplexControlRequestError, match="duplex touch failed") as exc_info:
|
||||
client.touch(
|
||||
fence.session_id,
|
||||
fence=fence,
|
||||
activity=DuplexLeaseActivity.HEARTBEAT,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "invalid_capability"
|
||||
assert exc_info.value.retryable is False
|
||||
assert exc_info.value.accepted_fence == DuplexFence(fence.session_id, epoch=2)
|
||||
assert exc_info.value.lease_generation == 7
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,386 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Typed duplex events: pure ``to_realtime()`` rendering and the stateful Realtime projection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex import events as events_module
|
||||
from vllm_omni.engine.duplex.events import (
|
||||
REALTIME_ERROR_TYPES_BY_CODE,
|
||||
AudioDelta,
|
||||
DuplexEvent,
|
||||
DuplexRawEvent,
|
||||
ErrorEvent,
|
||||
InputCommitted,
|
||||
ItemAdded,
|
||||
ItemCreated,
|
||||
ItemDone,
|
||||
Listen,
|
||||
OutputItemAdded,
|
||||
OverlapDecision,
|
||||
ResponseDone,
|
||||
SessionClosed,
|
||||
SessionCreated,
|
||||
SessionExpired,
|
||||
SessionHeartbeatAck,
|
||||
SessionReplaced,
|
||||
SessionResumed,
|
||||
SessionResyncRequired,
|
||||
TextDelta,
|
||||
TranscriptDone,
|
||||
TurnEvent,
|
||||
error_event,
|
||||
)
|
||||
from vllm_omni.engine.duplex.realtime_events import (
|
||||
RealtimeProjectionState,
|
||||
project_internal_event,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
#: Classes whose wire object carries ``session_id`` by contract.
|
||||
_SESSION_ID_ON_WIRE = {
|
||||
SessionClosed,
|
||||
SessionExpired,
|
||||
SessionHeartbeatAck,
|
||||
Listen,
|
||||
TurnEvent,
|
||||
OverlapDecision,
|
||||
SessionResumed,
|
||||
SessionReplaced,
|
||||
SessionResyncRequired,
|
||||
}
|
||||
#: Classes whose wire object carries ``epoch`` by contract.
|
||||
_EPOCH_ON_WIRE = {Listen, TurnEvent, OverlapDecision}
|
||||
|
||||
|
||||
def _public_event_classes() -> list[type[DuplexEvent]]:
|
||||
classes = [
|
||||
getattr(events_module, name)
|
||||
for name in events_module.__all__
|
||||
if inspect.isclass(getattr(events_module, name)) and issubclass(getattr(events_module, name), DuplexEvent)
|
||||
]
|
||||
assert len(classes) > 40
|
||||
return [cls for cls in classes if cls is not DuplexEvent]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("event_cls", _public_event_classes(), ids=lambda cls: cls.__name__)
|
||||
def test_to_realtime_renders_wire_type_event_id_and_documented_fields(event_cls: type[DuplexEvent]):
|
||||
event = event_cls(session_id="duplex-hidden", epoch=7)
|
||||
|
||||
wire = event.to_realtime()
|
||||
|
||||
assert wire["type"] == event.type
|
||||
assert wire["event_id"] == event.event_id
|
||||
assert wire["event_id"].startswith("event_")
|
||||
if event_cls is DuplexRawEvent:
|
||||
assert wire["type"] == "duplex."
|
||||
else:
|
||||
assert wire["type"] == event_cls.wire_type
|
||||
if event_cls in _SESSION_ID_ON_WIRE:
|
||||
assert wire["session_id"] == "duplex-hidden"
|
||||
else:
|
||||
assert "session_id" not in wire
|
||||
if event_cls in _EPOCH_ON_WIRE:
|
||||
assert wire["epoch"] == 7
|
||||
else:
|
||||
assert "epoch" not in wire
|
||||
# Every value is plain JSON (mappings / sequences are copied into dict / list).
|
||||
for value in wire.values():
|
||||
assert isinstance(value, str | int | float | bool | dict | list) or value is None
|
||||
|
||||
|
||||
def test_optional_wire_fields_are_omitted_when_none():
|
||||
created = SessionCreated(session={"id": "sid"})
|
||||
assert created.optional_wire_fields == frozenset({"attachment_generation", "resume_token"})
|
||||
wire = created.to_realtime()
|
||||
assert "attachment_generation" not in wire
|
||||
assert "resume_token" not in wire
|
||||
assert wire["session"] == {"id": "sid"}
|
||||
|
||||
with_resume = SessionCreated(session={"id": "sid"}, attachment_generation=2, resume_token="tok").to_realtime()
|
||||
assert with_resume["attachment_generation"] == 2
|
||||
assert with_resume["resume_token"] == "tok"
|
||||
|
||||
delta = AudioDelta(response_id="resp_1", item_id="item_resp_1", delta="AAAA").to_realtime()
|
||||
assert "sample_rate_hz" not in delta
|
||||
assert "metadata" not in delta
|
||||
assert delta["format"] == "pcm16"
|
||||
assert delta["output_index"] == 0
|
||||
assert delta["content_index"] == 0
|
||||
|
||||
anonymous_listen = Listen(session_id="sid", epoch=0, details={"reason": "silence"}).to_realtime()
|
||||
assert "response_id" not in anonymous_listen
|
||||
assert "id" not in anonymous_listen["response"]
|
||||
assert anonymous_listen["response"]["status"] == "listening"
|
||||
assert anonymous_listen["response"]["metadata"] == {"reason": "silence"}
|
||||
|
||||
bound_listen = Listen(session_id="sid", epoch=0, response_id="resp_1").to_realtime()
|
||||
assert bound_listen["response_id"] == "resp_1"
|
||||
assert bound_listen["response"]["id"] == "resp_1"
|
||||
|
||||
|
||||
def test_error_event_maps_codes_to_openai_error_types_and_echoes_client_event_id():
|
||||
assert REALTIME_ERROR_TYPES_BY_CODE["resource_exhausted"] == "rate_limit_error"
|
||||
assert REALTIME_ERROR_TYPES_BY_CODE["internal_error"] == "server_error"
|
||||
assert REALTIME_ERROR_TYPES_BY_CODE["bad_event"] == "invalid_request_error"
|
||||
|
||||
error = ErrorEvent(
|
||||
code="resource_exhausted",
|
||||
message="too many sessions",
|
||||
related_event_id="evt_client",
|
||||
param="session",
|
||||
extra={"retryable": True},
|
||||
)
|
||||
|
||||
assert error.error_type == "rate_limit_error"
|
||||
assert error.text == "too many sessions"
|
||||
assert error.error == {
|
||||
"type": "rate_limit_error",
|
||||
"code": "resource_exhausted",
|
||||
"message": "too many sessions",
|
||||
"event_id": "evt_client",
|
||||
"param": "session",
|
||||
"retryable": True,
|
||||
}
|
||||
wire = error.to_realtime()
|
||||
assert wire["type"] == "error"
|
||||
assert wire["error"] == error.error
|
||||
# The server event id and the echoed client event id are distinct.
|
||||
assert wire["event_id"] != "evt_client"
|
||||
assert ErrorEvent(code="never-seen-code").error_type == "invalid_request_error"
|
||||
|
||||
|
||||
def test_error_event_helper_normalizes_optional_client_event_id_and_param():
|
||||
event = error_event("bad_event", "nope", event_id="evt_1", param="item", extra={"retryable": False})
|
||||
assert isinstance(event, ErrorEvent)
|
||||
assert event.related_event_id == "evt_1"
|
||||
assert event.param == "item"
|
||||
assert event.error["retryable"] is False
|
||||
|
||||
anonymous = error_event("bad_event", "nope", event_id=42, param="")
|
||||
assert anonymous.related_event_id is None
|
||||
assert anonymous.param is None
|
||||
assert "event_id" not in anonymous.error
|
||||
assert "param" not in anonymous.error
|
||||
|
||||
|
||||
def test_audio_delta_decodes_base64_and_invalid_payloads_yield_none():
|
||||
raw = bytes(range(16))
|
||||
delta = AudioDelta(delta=base64.b64encode(raw).decode("ascii"), format="pcm16", sample_rate_hz=24000)
|
||||
|
||||
assert delta.audio == raw
|
||||
assert delta.to_realtime()["sample_rate_hz"] == 24000
|
||||
assert AudioDelta(delta="@@not base64@@").audio is None
|
||||
|
||||
|
||||
def test_session_closed_and_expired_are_the_only_terminal_events():
|
||||
closed = SessionClosed(session_id="sid", reason="client_close", details={"type": "session.closed"})
|
||||
expired = SessionExpired(session_id="sid", reason="idle_timeout")
|
||||
|
||||
assert closed.is_terminal is True
|
||||
assert expired.is_terminal is True
|
||||
assert closed.to_realtime() == {
|
||||
"type": "session.closed",
|
||||
"event_id": closed.event_id,
|
||||
"session_id": "sid",
|
||||
"reason": "client_close",
|
||||
"event": {"type": "session.closed"},
|
||||
}
|
||||
assert expired.to_realtime() == {
|
||||
"type": "session.expired",
|
||||
"event_id": expired.event_id,
|
||||
"session_id": "sid",
|
||||
"reason": "idle_timeout",
|
||||
}
|
||||
non_terminal = [cls for cls in _public_event_classes() if cls().is_terminal]
|
||||
assert non_terminal == [SessionClosed, SessionExpired]
|
||||
|
||||
|
||||
def test_generic_accessors_expose_response_item_text_and_status():
|
||||
assert TextDelta(response_id="resp_1", item_id="item_resp_1", delta="hi").text == "hi"
|
||||
assert TranscriptDone(transcript="done").text == "done"
|
||||
assert OutputItemAdded(item={"id": "item_1"}).item_id == "item_1"
|
||||
assert ItemAdded(item={"id": 3}).item_id is None
|
||||
assert ResponseDone(response={"status": "cancelled"}).status == "cancelled"
|
||||
assert ResponseDone().status is None
|
||||
assert TurnEvent(event="user_started", turn_state="user_speaking").response_id is None
|
||||
assert TurnEvent().audio is None
|
||||
|
||||
|
||||
def test_raw_event_prefixes_internal_type_and_wraps_details():
|
||||
raw = DuplexRawEvent(internal_type="model.custom", details={"type": "model.custom", "value": (1, 2)})
|
||||
|
||||
assert raw.type == "duplex.model.custom"
|
||||
assert raw.to_realtime() == {
|
||||
"type": "duplex.model.custom",
|
||||
"event_id": raw.event_id,
|
||||
"event": {"type": "model.custom", "value": [1, 2]},
|
||||
}
|
||||
|
||||
|
||||
def test_turn_event_and_overlap_decision_carry_session_identity_on_the_wire():
|
||||
turn = TurnEvent(session_id="sid", epoch=2, event="barge_in", turn_state="barge_in").to_realtime()
|
||||
assert turn == {
|
||||
"type": "turn.event",
|
||||
"event_id": turn["event_id"],
|
||||
"session_id": "sid",
|
||||
"event": "barge_in",
|
||||
"turn_state": "barge_in",
|
||||
"epoch": 2,
|
||||
}
|
||||
decision = OverlapDecision(
|
||||
session_id="sid", epoch=2, policy="listen_only", action="ignore", reason="short", details={"ms": 300}
|
||||
).to_realtime()
|
||||
assert decision["policy"] == "listen_only"
|
||||
assert decision["action"] == "ignore"
|
||||
assert decision["metadata"] == {"ms": 300}
|
||||
assert decision["session_id"] == "sid"
|
||||
|
||||
|
||||
# ---- RealtimeProjectionState / project_internal_event ----
|
||||
|
||||
|
||||
def _types(events: list[DuplexEvent]) -> list[str]:
|
||||
return [event.type for event in events]
|
||||
|
||||
|
||||
def test_projection_of_the_main_internal_event_sequence():
|
||||
state = RealtimeProjectionState(session_id="duplex-proj", model="test-model", initial_session_update=True)
|
||||
pcm = base64.b64encode(b"\x00\x10" * 8).decode("ascii")
|
||||
|
||||
created = project_internal_event(state, {"type": "session.created", "session": {"voice": "alloy"}})
|
||||
assert _types(created) == ["session.created", "session.updated"]
|
||||
assert isinstance(created[0], SessionCreated)
|
||||
session_payload = created[0].session
|
||||
assert session_payload["id"] == "duplex-proj"
|
||||
assert session_payload["model"] == "test-model"
|
||||
assert session_payload["voice"] == "alloy"
|
||||
assert session_payload["input_audio_format"] == "pcm16"
|
||||
assert session_payload["output_audio_format"] == "pcm16"
|
||||
assert state.initial_session_update is False
|
||||
# The extra session.updated is a one-shot for the opening session.update.
|
||||
assert _types(project_internal_event(state, {"type": "session.created", "session": {}})) == ["session.created"]
|
||||
|
||||
response_created = project_internal_event(
|
||||
state, {"type": "response.created", "response_id": "resp_1", "modalities": ["audio", "text"]}
|
||||
)
|
||||
assert _types(response_created) == [
|
||||
"response.created",
|
||||
"conversation.item.added",
|
||||
"conversation.item.created",
|
||||
"response.output_item.added",
|
||||
"response.content_part.added",
|
||||
]
|
||||
assert state.active_response_id == "resp_1"
|
||||
assert response_created[0].response_id == "resp_1"
|
||||
assert response_created[0].to_realtime()["response"]["status"] == "in_progress"
|
||||
assert isinstance(response_created[1], ItemAdded)
|
||||
assert isinstance(response_created[2], ItemCreated)
|
||||
assert response_created[1].item_id == "item_resp_1"
|
||||
assert response_created[3].item_id == "item_resp_1"
|
||||
assert response_created[4].to_realtime()["part"] == {"type": "audio", "transcript": ""}
|
||||
|
||||
audio_delta = project_internal_event(
|
||||
state,
|
||||
{"type": "response.output_audio.delta", "response_id": "resp_1", "audio": pcm, "text": "hi", "format": "pcm16"},
|
||||
)
|
||||
assert _types(audio_delta) == ["response.output_audio.delta", "response.output_audio_transcript.delta"]
|
||||
assert isinstance(audio_delta[0], AudioDelta)
|
||||
assert audio_delta[0].delta == pcm
|
||||
assert audio_delta[0].format == "pcm16"
|
||||
assert audio_delta[0].item_id == "item_resp_1"
|
||||
assert audio_delta[1].text == "hi"
|
||||
assert state.conversation_items["item_resp_1"]["content"][0]["transcript"] == "hi"
|
||||
|
||||
done = project_internal_event(state, {"type": "response.done", "response_id": "resp_1"})
|
||||
assert _types(done) == [
|
||||
"response.output_audio.done",
|
||||
"response.output_audio_transcript.done",
|
||||
"response.content_part.done",
|
||||
"response.output_item.done",
|
||||
"conversation.item.done",
|
||||
"response.done",
|
||||
"rate_limits.updated",
|
||||
]
|
||||
assert done[1].text == "hi"
|
||||
assert isinstance(done[4], ItemDone)
|
||||
assert done[4].item["status"] == "completed"
|
||||
assert isinstance(done[5], ResponseDone)
|
||||
assert done[5].status == "completed"
|
||||
assert done[5].response["output"][0]["content"][0] == {"type": "output_audio", "transcript": "hi"}
|
||||
assert state.active_response_id is None
|
||||
# A duplicate terminal for the same response is idempotent.
|
||||
assert project_internal_event(state, {"type": "response.done", "response_id": "resp_1"}) == []
|
||||
|
||||
committed = project_internal_event(
|
||||
state, {"type": "input.committed", "message": {"role": "user", "content": "hello"}, "input_commit_seq": 1}
|
||||
)
|
||||
assert _types(committed) == [
|
||||
"conversation.item.added",
|
||||
"conversation.item.created",
|
||||
"input_audio_buffer.committed",
|
||||
"conversation.item.done",
|
||||
]
|
||||
assert isinstance(committed[2], InputCommitted)
|
||||
assert committed[2].previous_item_id == "item_resp_1"
|
||||
assert committed[2].item_id == committed[0].item_id
|
||||
assert committed[2].to_realtime()["event"]["input_commit_seq"] == 1
|
||||
assert committed[0].item["content"] == [{"type": "input_text", "text": "hello"}]
|
||||
|
||||
project_internal_event(state, {"type": "response.created", "response_id": "resp_2"})
|
||||
project_internal_event(state, {"type": "response.output_audio.delta", "response_id": "resp_2", "audio": pcm})
|
||||
cancelled = project_internal_event(
|
||||
state, {"type": "audio.cancelled", "response_id": "resp_2", "reason": "barge_in", "committed_ms": 0}
|
||||
)
|
||||
assert _types(cancelled) == [
|
||||
"response.output_audio.done",
|
||||
"response.content_part.done",
|
||||
"response.output_item.done",
|
||||
"conversation.item.done",
|
||||
"response.done",
|
||||
"rate_limits.updated",
|
||||
]
|
||||
assert cancelled[4].status == "cancelled"
|
||||
assert cancelled[4].response["status_details"] == {"type": "cancelled", "reason": "barge_in"}
|
||||
assert state.active_response_id is None
|
||||
|
||||
closed = project_internal_event(state, {"type": "session.closed", "reason": "client_close"})
|
||||
assert _types(closed) == ["session.closed"]
|
||||
assert closed[0].is_terminal is True
|
||||
assert closed[0].to_realtime()["reason"] == "client_close"
|
||||
|
||||
|
||||
def test_projection_of_output_audio_buffer_clear_emits_cleared_before_terminals():
|
||||
state = RealtimeProjectionState(session_id="duplex-clear")
|
||||
pcm = base64.b64encode(b"\x00\x10" * 8).decode("ascii")
|
||||
project_internal_event(state, {"type": "response.created", "response_id": "resp_1"})
|
||||
project_internal_event(state, {"type": "response.output_audio.delta", "response_id": "resp_1", "audio": pcm})
|
||||
|
||||
cleared = project_internal_event(
|
||||
state, {"type": "audio.cancelled", "reason": "output_audio_buffer_clear", "committed_ms": 250}
|
||||
)
|
||||
|
||||
assert _types(cleared)[:2] == ["output_audio_buffer.cleared", "response.output_audio.done"]
|
||||
assert cleared[0].response_id == "resp_1"
|
||||
assert _types(cleared)[-2:] == ["response.done", "rate_limits.updated"]
|
||||
assert state.item_truncation_cursors["item_resp_1"] == (0, 250)
|
||||
|
||||
|
||||
def test_projection_leaves_error_to_typed_emit_sites():
|
||||
# Errors are constructed typed (``error_event``) at the emit site; the projector
|
||||
# has no ``error`` branch and falls back to the raw wrapper for such a dict.
|
||||
state = RealtimeProjectionState(session_id="duplex-err")
|
||||
|
||||
projected = project_internal_event(state, {"type": "error", "code": "bad_event", "error": "nope"})
|
||||
|
||||
assert len(projected) == 1
|
||||
assert isinstance(projected[0], DuplexRawEvent)
|
||||
assert projected[0].type == "duplex.error"
|
||||
assert projected[0].to_realtime()["event"]["code"] == "bad_event"
|
||||
@@ -1,19 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Lease semantics of the engine-resident duplex session (idle TTL, disconnect grace, resume CAS)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.lease import (
|
||||
from vllm_omni.engine.duplex.config import DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession, DuplexFenceMismatchError
|
||||
from vllm_omni.engine.duplex.session.lease import (
|
||||
DuplexLeaseActivity,
|
||||
DuplexLeaseConfig,
|
||||
DuplexLeaseState,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import DuplexInputMode
|
||||
from vllm_omni.engine.duplex.session import DuplexSessionRuntimeManager
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -30,195 +33,148 @@ class FakeMonotonicClock:
|
||||
|
||||
|
||||
def _lease_config(*, idle_ttl_s: float | None = 300.0) -> DuplexLeaseConfig:
|
||||
return DuplexLeaseConfig(
|
||||
idle_ttl_s=idle_ttl_s,
|
||||
disconnect_grace_s=30.0,
|
||||
return DuplexLeaseConfig(idle_ttl_s=idle_ttl_s, disconnect_grace_s=30.0)
|
||||
|
||||
|
||||
def _session(session_id: str, clock: FakeMonotonicClock, *, idle_ttl_s: float | None = 300.0) -> DuplexEngineSession:
|
||||
return DuplexEngineSession(
|
||||
session_id=session_id,
|
||||
config=DuplexSessionConfig(),
|
||||
lease=DuplexLeaseState(config=_lease_config(idle_ttl_s=idle_ttl_s), generation=0, last_activity=clock()),
|
||||
_clock=clock,
|
||||
)
|
||||
|
||||
|
||||
def test_open_touch_detach_and_idle_expiry_use_monotonic_time() -> None:
|
||||
def test_open_touch_detach_and_expiry_use_monotonic_time() -> None:
|
||||
clock = FakeMonotonicClock(100.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-expiry")
|
||||
session = manager.open_session(fence, lease_config=_lease_config())
|
||||
session = _session("sid-expiry", clock)
|
||||
|
||||
assert session.lease.last_activity == 100.0
|
||||
clock.advance(10.0)
|
||||
session.touch(fence, DuplexLeaseActivity.HEARTBEAT)
|
||||
session.detach(fence)
|
||||
session.touch_lease(DuplexLeaseActivity.HEARTBEAT)
|
||||
assert session.lease.last_activity == 110.0
|
||||
session.detach_lease()
|
||||
clock.advance(29.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
assert manager.collect_expired() == []
|
||||
assert session.lease.idle_expired(clock()) is False
|
||||
|
||||
clock.advance(272.0)
|
||||
expired = manager.collect_expired()
|
||||
clock.advance(1.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is True
|
||||
|
||||
assert [item.session_id for item in expired] == ["sid-expiry"]
|
||||
assert expired[0].reason == "disconnect_grace_expired"
|
||||
assert manager.get("sid-expiry") is session
|
||||
manager.finalize_close_session(session)
|
||||
assert manager.get("sid-expiry") is None
|
||||
clock.advance(271.0)
|
||||
assert session.lease.idle_expired(clock()) is True
|
||||
|
||||
|
||||
def test_detach_grace_and_resume_advance_token_independent_lease_generation() -> None:
|
||||
def test_detach_grace_and_resume_advance_the_lease_generation() -> None:
|
||||
clock = FakeMonotonicClock(10.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-resume")
|
||||
session = manager.open_session(fence, lease_config=_lease_config())
|
||||
session = _session("sid-resume", clock)
|
||||
|
||||
session.detach(fence)
|
||||
session.detach_lease()
|
||||
clock.advance(29.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
clock.advance(2.0)
|
||||
assert session.lease.disconnect_grace_expired(clock()) is True
|
||||
|
||||
generation = session.resume(fence, expected_lease_generation=0)
|
||||
generation = session.resume_lease(expected_lease_generation=0)
|
||||
|
||||
assert generation == 1
|
||||
assert session.lease_generation == 1
|
||||
assert session.lease.detached_at is None
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
with pytest.raises(ValueError, match="lease generation mismatch"):
|
||||
session.resume(fence, expected_lease_generation=0)
|
||||
session.resume_lease(expected_lease_generation=0)
|
||||
|
||||
|
||||
def test_active_operation_prevents_mid_transaction_expiry() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-operation")
|
||||
session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=5.0))
|
||||
session = _session("sid-operation", clock, idle_ttl_s=5.0)
|
||||
|
||||
session.begin_operation(fence, "append-1")
|
||||
session.begin_lease_operation(session.fence, "append-1")
|
||||
clock.advance(10.0)
|
||||
assert manager.collect_expired() == []
|
||||
assert session.lease.idle_expired(clock()) is False
|
||||
|
||||
session.end_operation(fence, "append-1")
|
||||
assert manager.collect_expired() == []
|
||||
session.end_lease_operation("append-1")
|
||||
assert session.lease.idle_expired(clock()) is False
|
||||
clock.advance(6.0)
|
||||
assert [item.session_id for item in manager.collect_expired()] == ["sid-operation"]
|
||||
assert session.lease.idle_expired(clock()) is True
|
||||
|
||||
|
||||
def test_sessions_have_independent_activity_deadlines_and_resources() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence_a = DuplexFence("sid-a")
|
||||
fence_b = DuplexFence("sid-b")
|
||||
session_a = manager.open_session(fence_a, lease_config=_lease_config(idle_ttl_s=10.0))
|
||||
session_b = manager.open_session(fence_b, lease_config=_lease_config(idle_ttl_s=10.0))
|
||||
session_a.reserve_stage_request(0, "req-a-reserved", fence=fence_a)
|
||||
session_a.bind_stage_request(1, "req-a-submitted", fence=fence_a)
|
||||
session_b.bind_stage_request(0, "req-b", fence=fence_b)
|
||||
session_a = _session("sid-a", clock, idle_ttl_s=10.0)
|
||||
session_b = _session("sid-b", clock, idle_ttl_s=10.0)
|
||||
session_a.reserve_stage_request(0, "req-a-reserved", fence=session_a.fence)
|
||||
session_a.bind_stage_request(1, "req-a-submitted", fence=session_a.fence)
|
||||
session_b.bind_stage_request(0, "req-b", fence=session_b.fence)
|
||||
|
||||
clock.advance(6.0)
|
||||
session_b.touch(fence_b, DuplexLeaseActivity.MODEL_OUTPUT)
|
||||
session_b.touch_lease(DuplexLeaseActivity.MODEL_OUTPUT)
|
||||
clock.advance(5.0)
|
||||
expired = manager.collect_expired()
|
||||
|
||||
assert len(expired) == 1
|
||||
assert expired[0].session_id == "sid-a"
|
||||
assert expired[0].reserved_request_ids == ("req-a-reserved",)
|
||||
assert expired[0].submitted_request_ids == ("req-a-submitted",)
|
||||
assert manager.get("sid-b") is session_b
|
||||
assert session_a.lease.idle_expired(clock()) is True
|
||||
assert session_b.lease.idle_expired(clock()) is False
|
||||
assert session_a.resource_request_ids(submitted=False) == ["req-a-reserved"]
|
||||
assert session_a.resource_request_ids(submitted=True) == ["req-a-submitted"]
|
||||
assert session_b.resource_request_ids() == ["req-b"]
|
||||
|
||||
|
||||
def test_close_and_reaper_select_exactly_one_terminal_transition() -> None:
|
||||
def test_close_is_a_single_terminal_transition() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-race")
|
||||
manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=1.0))
|
||||
session = _session("sid-race", clock, idle_ttl_s=1.0)
|
||||
clock.advance(2.0)
|
||||
assert session.lease.idle_expired(clock()) is True
|
||||
|
||||
assert len(manager.collect_expired()) == 1
|
||||
assert manager.close_session(fence, reason="explicit_close") is None
|
||||
assert manager.collect_expired() == []
|
||||
|
||||
fence_2 = DuplexFence("sid-race-2")
|
||||
manager.open_session(fence_2, lease_config=_lease_config(idle_ttl_s=1.0))
|
||||
assert manager.close_session(fence_2, reason="explicit_close") is not None
|
||||
clock.advance(2.0)
|
||||
assert manager.collect_expired() == []
|
||||
assert session.begin_close(reason="explicit_close") is True
|
||||
assert session.lease.terminal_reason == "explicit_close"
|
||||
# A terminal lease is never reaped again and a second close is a no-op.
|
||||
assert session.lease.idle_expired(clock()) is False
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
assert session.begin_close(reason="idle_ttl_expired") is True
|
||||
assert session.lease.terminal_reason == "explicit_close"
|
||||
with pytest.raises(RuntimeError, match="terminal"):
|
||||
session.touch_lease(DuplexLeaseActivity.HEARTBEAT)
|
||||
|
||||
|
||||
def test_stale_fence_cannot_touch_detach_or_resume_lease() -> None:
|
||||
def test_stale_fence_cannot_start_a_lease_operation() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
current = DuplexFence("sid-fence", epoch=1)
|
||||
session = _session("sid-fence", clock)
|
||||
session.epoch = 1
|
||||
session.sync_fence()
|
||||
stale = DuplexFence("sid-fence", epoch=0)
|
||||
session = manager.open_session(current, lease_config=_lease_config())
|
||||
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.touch(stale, DuplexLeaseActivity.APPEND)
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.detach(stale)
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.resume(stale, expected_lease_generation=0)
|
||||
with pytest.raises(DuplexFenceMismatchError, match="fence mismatch"):
|
||||
session.begin_lease_operation(stale, "append-1")
|
||||
with pytest.raises(DuplexFenceMismatchError, match="fence mismatch"):
|
||||
session.begin_lease_operation(DuplexFence("other-session", epoch=1), "append-2")
|
||||
|
||||
|
||||
def test_disabled_idle_expiry_never_collects_session() -> None:
|
||||
def test_disabled_idle_expiry_never_expires_session() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
manager.open_session(DuplexFence("sid-no-expiry"), lease_config=_lease_config(idle_ttl_s=None))
|
||||
session = _session("sid-no-expiry", clock, idle_ttl_s=None)
|
||||
|
||||
clock.advance(1_000_000.0)
|
||||
|
||||
assert manager.collect_expired() == []
|
||||
assert session.lease.expires_at is None
|
||||
assert session.lease.idle_expired(clock()) is False
|
||||
|
||||
|
||||
def test_detached_session_expires_at_disconnect_grace_when_idle_ttl_is_disabled() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
fence = DuplexFence("sid-disconnect-grace")
|
||||
session = manager.open_session(fence, lease_config=_lease_config(idle_ttl_s=None))
|
||||
session = _session("sid-disconnect-grace", clock, idle_ttl_s=None)
|
||||
|
||||
session.detach(fence)
|
||||
session.detach_lease()
|
||||
clock.advance(29.0)
|
||||
assert manager.collect_expired() == []
|
||||
assert session.lease.disconnect_grace_expired(clock()) is False
|
||||
|
||||
clock.advance(1.0)
|
||||
expired = manager.collect_expired()
|
||||
|
||||
assert [item.session_id for item in expired] == [fence.session_id]
|
||||
assert expired[0].reason == "disconnect_grace_expired"
|
||||
assert session.lease.disconnect_grace_expired(clock()) is True
|
||||
|
||||
|
||||
def test_runtime_manager_enforces_server_owned_session_admission() -> None:
|
||||
manager = DuplexSessionRuntimeManager(max_sessions=2)
|
||||
manager.open_session(DuplexFence("sid-admission-a"))
|
||||
manager.open_session(DuplexFence("sid-admission-b"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="duplex_session_capacity_exhausted"):
|
||||
manager.open_session(DuplexFence("sid-admission-c"))
|
||||
|
||||
|
||||
def test_completed_append_cache_is_bounded() -> None:
|
||||
manager = DuplexSessionRuntimeManager(completed_append_limit=2)
|
||||
fence = DuplexFence("sid-completed-cache")
|
||||
session = manager.open_session(fence)
|
||||
|
||||
for index in range(3):
|
||||
session.record_completed_append(
|
||||
f"operation-{index}",
|
||||
fence=fence,
|
||||
mode=DuplexInputMode.TURN_COMMIT_ONLY,
|
||||
final=True,
|
||||
stage_results=[{"index": index}],
|
||||
)
|
||||
|
||||
assert list(session.completed_appends) == ["operation-1", "operation-2"]
|
||||
|
||||
session.accept_fence(DuplexFence(fence.session_id, epoch=1))
|
||||
assert session.completed_appends == {}
|
||||
|
||||
|
||||
def test_lease_config_and_expiry_record_are_immutable() -> None:
|
||||
def test_lease_config_is_immutable_and_validated() -> None:
|
||||
config = _lease_config()
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
config.idle_ttl_s = 1.0 # type: ignore[misc]
|
||||
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
manager = DuplexSessionRuntimeManager(clock=clock)
|
||||
manager.open_session(DuplexFence("sid-record"), lease_config=_lease_config(idle_ttl_s=1.0))
|
||||
clock.advance(2.0)
|
||||
record = manager.collect_expired()[0]
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
record.reason = "changed" # type: ignore[misc]
|
||||
with pytest.raises(ValueError, match="idle_ttl_s"):
|
||||
DuplexLeaseConfig(idle_ttl_s=0.0)
|
||||
with pytest.raises(ValueError, match="disconnect_grace_s"):
|
||||
DuplexLeaseConfig(disconnect_grace_s=0.0)
|
||||
@@ -0,0 +1,324 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``DuplexModelPlugin`` loading/validation helpers and the ``PipelineConfig.duplex_plugin`` binding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import fields
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig, PipelineConfig
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.contracts import DuplexAppendPlan
|
||||
from vllm_omni.engine.duplex.plugin import (
|
||||
DuplexDataPlane,
|
||||
DuplexModelPlugin,
|
||||
DuplexModelSessionState,
|
||||
DuplexRuntimeConfigError,
|
||||
coerce_int,
|
||||
load_duplex_plugin,
|
||||
payload_turn_id,
|
||||
reject_changed_runtime_value,
|
||||
validate_duplex_plugin_sampling,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _encode_audio(audio: object, sample_rate_hz: int, fmt: str, speed: float | None) -> str | None:
|
||||
del audio, sample_rate_hz, fmt, speed
|
||||
return None
|
||||
|
||||
|
||||
class FakeDataPlane(DuplexDataPlane):
|
||||
def begin_request(self, request_id: str) -> None:
|
||||
pass
|
||||
|
||||
def is_terminal(self, request_id: str | None) -> bool:
|
||||
return False
|
||||
|
||||
def mark_terminal(self, request_id: str) -> None:
|
||||
pass
|
||||
|
||||
def close_stream(self, request_id: str) -> None:
|
||||
pass
|
||||
|
||||
def close_session(self, session_id: str, *, active_request_id: str | None = None) -> None:
|
||||
pass
|
||||
|
||||
def project(self, result, *, context=None):
|
||||
del result, context
|
||||
return []
|
||||
|
||||
|
||||
class FakePlugin(DuplexModelPlugin):
|
||||
"""Minimal plugin; referenced by dotted path from ``load_duplex_plugin`` tests."""
|
||||
|
||||
plugin_id = "fake-duplex"
|
||||
|
||||
def __init__(self, encode_audio) -> None:
|
||||
super().__init__(encode_audio)
|
||||
self.data_plane = FakeDataPlane()
|
||||
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return tuple(defaults)
|
||||
|
||||
def plan_append(self, **kwargs) -> DuplexAppendPlan:
|
||||
del kwargs
|
||||
return DuplexAppendPlan(prompt={"prompt_token_ids": [1]})
|
||||
|
||||
def decide_output(self, **kwargs):
|
||||
del kwargs
|
||||
return None
|
||||
|
||||
def create_session_state(self) -> DuplexModelSessionState:
|
||||
raise NotImplementedError
|
||||
|
||||
def capabilities(self, *, max_sessions: int) -> DuplexCapabilities:
|
||||
del max_sessions
|
||||
return DuplexCapabilities()
|
||||
|
||||
def validate_client_extra_body(self, extra_body: object) -> None:
|
||||
pass
|
||||
|
||||
async def prepare_runtime_config(self, config: DuplexSessionConfig, *, model_config: Any) -> dict[str, object]:
|
||||
del config, model_config
|
||||
return {}
|
||||
|
||||
def runtime_config_for_update(self, config, current):
|
||||
del config
|
||||
return dict(current)
|
||||
|
||||
def data_plane_context(self, **kwargs) -> object:
|
||||
return dict(kwargs)
|
||||
|
||||
|
||||
class PluginWithoutId(FakePlugin):
|
||||
plugin_id = ""
|
||||
|
||||
|
||||
class PluginWithoutDataPlane(FakePlugin):
|
||||
def __init__(self, encode_audio) -> None:
|
||||
DuplexModelPlugin.__init__(self, encode_audio)
|
||||
|
||||
|
||||
class PluginWithWrongDataPlane(FakePlugin):
|
||||
def __init__(self, encode_audio) -> None:
|
||||
DuplexModelPlugin.__init__(self, encode_audio)
|
||||
self.data_plane = object() # type: ignore[assignment]
|
||||
|
||||
|
||||
class NotAPlugin:
|
||||
plugin_id = "lookalike"
|
||||
data_plane = FakeDataPlane()
|
||||
|
||||
def __init__(self, encode_audio) -> None:
|
||||
self.encode_audio = encode_audio
|
||||
|
||||
|
||||
def _path(cls: type) -> str:
|
||||
return f"{__name__}.{cls.__name__}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# load_duplex_plugin #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_load_duplex_plugin_instantiates_the_class_at_the_dotted_path() -> None:
|
||||
plugin = load_duplex_plugin(_path(FakePlugin), _encode_audio)
|
||||
|
||||
assert type(plugin) is FakePlugin
|
||||
assert plugin.plugin_id == "fake-duplex"
|
||||
assert isinstance(plugin.data_plane, FakeDataPlane)
|
||||
# Defaults of the optional class-level policy.
|
||||
assert plugin.private_runtime_config_keys == frozenset()
|
||||
assert plugin.silence_continuation_samples == 16000
|
||||
assert plugin.runtime_config_for_function_output(DuplexSessionConfig(), {}, {"name": "tool"}) is None
|
||||
|
||||
|
||||
def test_load_duplex_plugin_rejects_a_non_plugin_class() -> None:
|
||||
with pytest.raises(TypeError, match="not a DuplexModelPlugin"):
|
||||
load_duplex_plugin(_path(NotAPlugin), _encode_audio)
|
||||
|
||||
|
||||
def test_load_duplex_plugin_requires_a_plugin_id() -> None:
|
||||
with pytest.raises(TypeError, match="plugin_id"):
|
||||
load_duplex_plugin(_path(PluginWithoutId), _encode_audio)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("plugin_cls", [PluginWithoutDataPlane, PluginWithWrongDataPlane])
|
||||
def test_load_duplex_plugin_requires_a_data_plane(plugin_cls: type) -> None:
|
||||
with pytest.raises(TypeError, match="DuplexDataPlane as data_plane"):
|
||||
load_duplex_plugin(_path(plugin_cls), _encode_audio)
|
||||
|
||||
|
||||
def test_load_duplex_plugin_rejects_paths_without_a_module() -> None:
|
||||
with pytest.raises(ValueError, match="Invalid duplex plugin path"):
|
||||
load_duplex_plugin("FakePlugin", _encode_audio)
|
||||
|
||||
|
||||
def test_load_duplex_plugin_propagates_missing_attributes() -> None:
|
||||
with pytest.raises(AttributeError):
|
||||
load_duplex_plugin(f"{__name__}.NoSuchPlugin", _encode_audio)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# validate_duplex_plugin_sampling #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_validate_duplex_plugin_sampling_accepts_one_parameter_per_stage() -> None:
|
||||
plugin = FakePlugin(_encode_audio)
|
||||
|
||||
validate_duplex_plugin_sampling(plugin, sampling_defaults=(SamplingParams(), SamplingParams()))
|
||||
validate_duplex_plugin_sampling(plugin, sampling_defaults=())
|
||||
|
||||
|
||||
def test_validate_duplex_plugin_sampling_rejects_stage_count_mismatch() -> None:
|
||||
class WrongStageCountPlugin(FakePlugin):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return defaults[:1]
|
||||
|
||||
with pytest.raises(ValueError, match="one sampling parameter per stage"):
|
||||
validate_duplex_plugin_sampling(
|
||||
WrongStageCountPlugin(_encode_audio),
|
||||
sampling_defaults=(SamplingParams(), SamplingParams()),
|
||||
)
|
||||
|
||||
|
||||
def test_validate_duplex_plugin_sampling_rejects_type_mismatch() -> None:
|
||||
class WrongSamplingTypePlugin(FakePlugin):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return tuple(object() for _ in defaults)
|
||||
|
||||
with pytest.raises(TypeError, match="sampling parameter type mismatch for stage 0"):
|
||||
validate_duplex_plugin_sampling(
|
||||
WrongSamplingTypePlugin(_encode_audio),
|
||||
sampling_defaults=(SamplingParams(), SamplingParams()),
|
||||
)
|
||||
|
||||
|
||||
def test_validate_duplex_plugin_sampling_skips_type_check_for_stages_without_defaults() -> None:
|
||||
class ObjectSamplingPlugin(FakePlugin):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return tuple(object() for _ in defaults)
|
||||
|
||||
validate_duplex_plugin_sampling(ObjectSamplingPlugin(_encode_audio), sampling_defaults=(None, None))
|
||||
|
||||
|
||||
def test_validate_duplex_plugin_sampling_rejects_non_tuple_results() -> None:
|
||||
class ListSamplingPlugin(FakePlugin):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return list(defaults)
|
||||
|
||||
with pytest.raises(TypeError, match="as a tuple"):
|
||||
validate_duplex_plugin_sampling(ListSamplingPlugin(_encode_audio), sampling_defaults=(SamplingParams(),))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Runtime-config helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_reject_changed_runtime_value_only_raises_on_change() -> None:
|
||||
reject_changed_runtime_value("voice-a", "voice-a", message="voice is fixed", code="voice_changed")
|
||||
|
||||
with pytest.raises(DuplexRuntimeConfigError, match="voice is fixed") as excinfo:
|
||||
reject_changed_runtime_value("voice-b", "voice-a", message="voice is fixed", code="voice_changed")
|
||||
|
||||
assert excinfo.value.code == "voice_changed"
|
||||
assert isinstance(excinfo.value, ValueError)
|
||||
|
||||
|
||||
def test_reject_changed_runtime_value_uses_the_requested_error_class() -> None:
|
||||
class CustomRuntimeConfigError(DuplexRuntimeConfigError):
|
||||
pass
|
||||
|
||||
with pytest.raises(CustomRuntimeConfigError) as excinfo:
|
||||
reject_changed_runtime_value(1, 2, message="changed", code="custom", error_cls=CustomRuntimeConfigError)
|
||||
|
||||
assert excinfo.value.code == "custom"
|
||||
|
||||
|
||||
def test_duplex_runtime_config_error_defaults_its_code() -> None:
|
||||
assert DuplexRuntimeConfigError("bad").code == "invalid_duplex_runtime_config"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "expected"),
|
||||
[
|
||||
({"duplex_turn_id": 3}, 3),
|
||||
({"duplex_turn_id": "7"}, 7),
|
||||
({"model_turn_id": 2}, 2),
|
||||
({"duplex_turn_id": 5, "model_turn_id": 9}, 5),
|
||||
({"duplex_turn_id": "not-a-number"}, None),
|
||||
({}, None),
|
||||
("turn", None),
|
||||
(None, None),
|
||||
],
|
||||
)
|
||||
def test_payload_turn_id_prefers_duplex_turn_id_and_coerces(payload: object, expected: int | None) -> None:
|
||||
assert payload_turn_id(payload) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[(4, 4), ("11", 11), (2.9, 2), (True, 1), (None, None), ("x", None), (object(), None), ([1], None)],
|
||||
)
|
||||
def test_coerce_int(value: object, expected: int | None) -> None:
|
||||
assert coerce_int(value) == expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# PipelineConfig binding #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_pipeline_config_binds_one_duplex_plugin_path() -> None:
|
||||
field_defaults = {f.name: f.default for f in fields(PipelineConfig)}
|
||||
|
||||
assert "duplex_plugin" in field_defaults
|
||||
assert field_defaults["duplex_plugin"] is None
|
||||
|
||||
# The pre-framework fields survive only so that the pipelines of the models
|
||||
# that are not ported yet still construct (RFC vllm-omni#7181 ports them).
|
||||
# Nothing in the duplex framework may read them: a pipeline is a duplex
|
||||
# model iff it declares ``duplex_plugin``.
|
||||
legacy_fields = ("duplex_runtime_extension", "duplex_serving_adapter", "duplex_control_enabled")
|
||||
for legacy_field in legacy_fields:
|
||||
assert legacy_field in field_defaults
|
||||
|
||||
framework_sources = [
|
||||
*(Path("vllm_omni/engine/duplex").glob("*.py")),
|
||||
*(Path("vllm_omni/entrypoints/duplex").glob("*.py")),
|
||||
Path("vllm_omni/engine/duplex_omni_engine.py"),
|
||||
Path("vllm_omni/engine/duplex_orchestrator.py"),
|
||||
Path("vllm_omni/entrypoints/duplex_omni.py"),
|
||||
]
|
||||
readers = [
|
||||
f"{source}:{legacy_field}"
|
||||
for source in framework_sources
|
||||
for legacy_field in legacy_fields
|
||||
if legacy_field in source.read_text(encoding="utf-8")
|
||||
]
|
||||
assert readers == []
|
||||
|
||||
# Same rule for the runtime-config knob the framework stopped reading: the
|
||||
# append-retry table it bounded went away with the correlated append RPC.
|
||||
assert "completed_append_cache_size" in {f.name for f in fields(DuplexSessionRuntimeConfig)}
|
||||
assert [
|
||||
str(source)
|
||||
for source in framework_sources
|
||||
if "completed_append_cache_size" in source.read_text(encoding="utf-8")
|
||||
] == []
|
||||
@@ -1,563 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.engine.duplex import runtime as duplex_runtime
|
||||
from vllm_omni.engine.duplex.contracts import (
|
||||
duplex_data_plane_request_info,
|
||||
duplex_resource_request_belongs_to_session,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexInputMode,
|
||||
DuplexOutputAction,
|
||||
DuplexOutputDecision,
|
||||
DuplexRuntimeCapabilities,
|
||||
DuplexSessionRuntimeManager,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import (
|
||||
MiniCPMO45DuplexRuntimeExtension,
|
||||
_duplex_hd_slice_count,
|
||||
build_duplex_data_plane_prompt,
|
||||
duplex_scheduler_token_budget,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_duplex_fence_is_immutable():
|
||||
fence = DuplexFence("session")
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
fence.epoch = 1 # type: ignore[misc]
|
||||
|
||||
assert not hasattr(fence, "__dict__")
|
||||
|
||||
|
||||
def test_duplex_runtime_tracks_stage_bindings_and_barge_in_epoch():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
DuplexFence("sid-1"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_TOKENS},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(stage_id=0, request_id="req-stage0", fence=session.fence)
|
||||
session.bind_stage_request(stage_id=1, request_id="req-stage1", fence=session.fence)
|
||||
|
||||
update = session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence)
|
||||
next_fence = DuplexFence("sid-1", epoch=1)
|
||||
stale_request_ids = session.release_fence(session.fence)
|
||||
session.accept_fence(next_fence)
|
||||
|
||||
assert update.seq == 1
|
||||
assert session.fence == next_fence
|
||||
assert stale_request_ids == ["req-stage0", "req-stage1"]
|
||||
assert session.stage_bindings == {}
|
||||
|
||||
|
||||
def test_duplex_runtime_tracks_same_request_id_for_each_pipeline_stage():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
fence = DuplexFence("sid-shared-pipeline-request")
|
||||
session = manager.open_session(fence)
|
||||
|
||||
session.reserve_stage_request(0, "req-shared", fence=fence)
|
||||
session.bind_stage_request(0, "req-shared", fence=fence)
|
||||
session.bind_stage_request(1, "req-shared", fence=fence)
|
||||
|
||||
assert session.stage_bindings[0].request_id == "req-shared"
|
||||
assert session.stage_bindings[1].request_id == "req-shared"
|
||||
assert session.resource_request_ids() == ["req-shared"]
|
||||
assert session.input_seq == 0
|
||||
|
||||
|
||||
def test_duplex_runtime_extension_validation_rejects_missing_methods():
|
||||
class IncompleteExtension:
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return defaults
|
||||
|
||||
with pytest.raises(TypeError, match="plan_append"):
|
||||
duplex_runtime.validate_duplex_runtime_extension(IncompleteExtension())
|
||||
|
||||
|
||||
def test_duplex_runtime_extension_validation_rejects_stage_count_mismatch():
|
||||
class WrongStageCountExtension(MiniCPMO45DuplexRuntimeExtension):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return defaults[:1]
|
||||
|
||||
with pytest.raises(ValueError, match="one sampling parameter per stage"):
|
||||
duplex_runtime.validate_duplex_runtime_extension(
|
||||
WrongStageCountExtension(),
|
||||
sampling_defaults=(SamplingParams(), SamplingParams()),
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_runtime_extension_validation_rejects_sampling_type_mismatch():
|
||||
class WrongSamplingTypeExtension(MiniCPMO45DuplexRuntimeExtension):
|
||||
def configure_sampling_params(self, *, runtime_config, defaults):
|
||||
del runtime_config
|
||||
return tuple(object() for _ in defaults)
|
||||
|
||||
with pytest.raises(TypeError, match="sampling parameter type"):
|
||||
duplex_runtime.validate_duplex_runtime_extension(
|
||||
WrongSamplingTypeExtension(),
|
||||
sampling_defaults=(SamplingParams(), SamplingParams()),
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_output_decision_metadata_is_immutable():
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={"model_listen": True},
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
decision.metadata["model_listen"] = False
|
||||
|
||||
|
||||
def _decide_minicpmo_output(
|
||||
output: object,
|
||||
*,
|
||||
segment_token_ids: tuple[int, ...] = (),
|
||||
segment_output_metadata: dict | None = None,
|
||||
):
|
||||
return MiniCPMO45DuplexRuntimeExtension().decide_output(
|
||||
stage_id=0,
|
||||
final_stage_id=1,
|
||||
segment_finished=True,
|
||||
segment_token_ids=segment_token_ids,
|
||||
segment_output_metadata=segment_output_metadata or {},
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
def test_minicpmo_extension_owns_stage_sampling_overrides():
|
||||
defaults = (
|
||||
SamplingParams(max_tokens=4),
|
||||
SamplingParams(max_tokens=8),
|
||||
)
|
||||
|
||||
configured = MiniCPMO45DuplexRuntimeExtension().configure_sampling_params(
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 20},
|
||||
"duplex_stage_sampling_params": {"1": {"stop_token_ids": [151645]}},
|
||||
},
|
||||
defaults=defaults,
|
||||
)
|
||||
|
||||
assert configured[0].max_tokens == 20
|
||||
assert configured[1].stop_token_ids == [151645]
|
||||
assert defaults[0].max_tokens == 4
|
||||
assert 151645 not in (defaults[1].stop_token_ids or [])
|
||||
|
||||
|
||||
def test_minicpmo_output_decision_uses_raw_streaming_token_snapshot():
|
||||
decision = _decide_minicpmo_output(
|
||||
SimpleNamespace(outputs=[SimpleNamespace()]),
|
||||
segment_token_ids=(151705,),
|
||||
segment_output_metadata={"special_token_ids": {"listen_token_id": 151705}},
|
||||
)
|
||||
|
||||
assert decision is not None
|
||||
assert decision.action is DuplexOutputAction.DIRECT_RESPONSE
|
||||
assert decision.metadata["duplex_native_decision"] == "listen"
|
||||
assert decision.metadata["model_listen"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"])
|
||||
def test_minicpmo_output_decision_ignores_output_level_token_history(attr):
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
|
||||
outputs=[SimpleNamespace()],
|
||||
**{attr: [42, 151705]},
|
||||
)
|
||||
|
||||
assert _decide_minicpmo_output(output) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attr", ["token_ids", "cumulative_token_ids"])
|
||||
def test_minicpmo_output_decision_uses_completion_token_ids(attr):
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
|
||||
outputs=[SimpleNamespace(**{attr: [42, 151705]})],
|
||||
)
|
||||
|
||||
assert _decide_minicpmo_output(output) is not None
|
||||
|
||||
|
||||
def test_minicpmo_output_decision_uses_completion_stop_reason():
|
||||
output = SimpleNamespace(
|
||||
multimodal_output={"special_token_ids": {"listen_token_id": 151705}},
|
||||
outputs=[SimpleNamespace(stop_reason=151705)],
|
||||
)
|
||||
|
||||
assert _decide_minicpmo_output(output) is not None
|
||||
|
||||
|
||||
def test_duplex_runtime_cancel_fence_rejects_late_append_and_accepts_next_epoch():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
cancelled_fence = DuplexFence("sid-cancel-race")
|
||||
next_fence = DuplexFence("sid-cancel-race", epoch=1)
|
||||
session = manager.open_session(
|
||||
cancelled_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(
|
||||
stage_id=0,
|
||||
request_id="req-cancelled",
|
||||
fence=cancelled_fence,
|
||||
)
|
||||
|
||||
stale_request_ids = session.cancel_fence(cancelled_fence, next_fence)
|
||||
|
||||
assert stale_request_ids == ["req-cancelled"]
|
||||
assert session.fence == next_fence
|
||||
assert session.stage_bindings == {}
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=cancelled_fence,
|
||||
)
|
||||
update = session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=next_fence,
|
||||
)
|
||||
assert update.seq == 1
|
||||
|
||||
|
||||
def test_duplex_runtime_stale_close_preserves_live_session_and_bindings():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
current_fence = DuplexFence("sid-stale-close", epoch=1)
|
||||
session = manager.open_session(
|
||||
current_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(0, "req-live", fence=current_fence)
|
||||
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
manager.close_session(DuplexFence("sid-stale-close"))
|
||||
|
||||
assert manager.get("sid-stale-close") is session
|
||||
assert session.stage_request_ids() == ["req-live"]
|
||||
|
||||
|
||||
def test_duplex_runtime_reopen_rejects_late_append_from_old_incarnation():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
old_fence = DuplexFence("sid-reopen", incarnation=0)
|
||||
old_session = manager.open_session(
|
||||
old_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
manager.close_session(old_fence)
|
||||
|
||||
new_fence = DuplexFence("sid-reopen", incarnation=1)
|
||||
new_session = manager.open_session(
|
||||
new_fence,
|
||||
capabilities=old_session.capabilities,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
new_session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=old_fence,
|
||||
)
|
||||
assert (
|
||||
new_session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=new_fence,
|
||||
).seq
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_prompt_expands_incarnation_metadata():
|
||||
fence = DuplexFence("sid-incarnation", incarnation=3)
|
||||
|
||||
prompt = build_duplex_data_plane_prompt(
|
||||
request_id=duplex_resource_request_id(fence, "stage0"),
|
||||
fence=fence,
|
||||
session_config={},
|
||||
runtime_config={},
|
||||
seq=1,
|
||||
turn_seq=1,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload={"is_speech": True},
|
||||
final=False,
|
||||
)
|
||||
|
||||
assert prompt["model_intermediate_buffer"]["duplex"]["incarnation"] == 3
|
||||
|
||||
|
||||
def test_duplex_runtime_tracks_turn_local_append_sequence():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
DuplexFence("sid-turn-seq"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
first = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence)
|
||||
second = session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK, fence=session.fence)
|
||||
next_turn = DuplexFence("sid-turn-seq", turn_id=1, response_seq=1)
|
||||
third = session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=next_turn,
|
||||
)
|
||||
fourth = session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=next_turn,
|
||||
)
|
||||
|
||||
assert [first.seq, second.seq, third.seq, fourth.seq] == [1, 2, 3, 4]
|
||||
assert [first.turn_seq, second.turn_seq, third.turn_seq, fourth.turn_seq] == [1, 2, 1, 2]
|
||||
assert [first.turn_id, second.turn_id, third.turn_id, fourth.turn_id] == [0, 0, 1, 1]
|
||||
|
||||
|
||||
def test_duplex_runtime_rejects_unsupported_append_mode():
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
DuplexFence("sid-2"),
|
||||
capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.TURN_COMMIT_ONLY}),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="not supported"):
|
||||
session.append_input(mode=DuplexInputMode.APPEND_TOKENS, fence=session.fence)
|
||||
|
||||
|
||||
def test_duplex_data_plane_request_info_extracts_structured_stage_result():
|
||||
request_id, response_stage_id = duplex_data_plane_request_info(
|
||||
{
|
||||
"stage_results": [
|
||||
{"result": {"supported": True}},
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": "duplex-sid-e0-stage0-s1",
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert request_id == "duplex-sid-e0-stage0-s1"
|
||||
assert response_stage_id == 1
|
||||
|
||||
|
||||
def test_duplex_data_plane_request_info_rejects_missing_request_id():
|
||||
assert duplex_data_plane_request_info(
|
||||
{
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": "",
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
) == (None, None)
|
||||
|
||||
|
||||
def test_duplex_scheduler_token_budget_estimates_pcm_slots():
|
||||
assert (
|
||||
duplex_scheduler_token_budget(
|
||||
{
|
||||
"audio": "AAAAAA==",
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
== 16
|
||||
)
|
||||
|
||||
|
||||
def test_duplex_scheduler_token_budget_ignores_client_budget_fields():
|
||||
assert (
|
||||
duplex_scheduler_token_budget(
|
||||
{
|
||||
"audio": "AAAAAA==",
|
||||
"format": "pcm_f32le",
|
||||
"duplex_num_input_tokens": 999,
|
||||
"num_input_tokens": 999,
|
||||
}
|
||||
)
|
||||
== 16
|
||||
)
|
||||
|
||||
|
||||
def test_resource_state_rejects_fence_regression_and_requires_explicit_fence():
|
||||
current = DuplexFence("sid", epoch=2, turn_id=3, response_seq=4)
|
||||
manager = DuplexSessionRuntimeManager()
|
||||
session = manager.open_session(
|
||||
current,
|
||||
capabilities=DuplexRuntimeCapabilities(input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK}),
|
||||
)
|
||||
|
||||
for stale in (
|
||||
DuplexFence("sid", epoch=1, turn_id=99, response_seq=99),
|
||||
DuplexFence("sid", epoch=2, turn_id=2, response_seq=4),
|
||||
DuplexFence("sid", epoch=2, turn_id=3, response_seq=3),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="fence mismatch"):
|
||||
session.accept_fence(stale)
|
||||
assert session.fence == current
|
||||
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
session.bind_stage_request(0, "request")
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
session.append_input(mode=DuplexInputMode.APPEND_AUDIO_CHUNK)
|
||||
with pytest.raises(TypeError, match="DuplexFence"):
|
||||
manager.open_session("legacy-session")
|
||||
with pytest.raises(TypeError, match="DuplexFence"):
|
||||
manager.close_session("legacy-session")
|
||||
|
||||
|
||||
def test_resource_request_id_is_derived_from_fence_and_role():
|
||||
fence = DuplexFence("sid-with-dashes", epoch=7, turn_id=11, response_seq=13)
|
||||
|
||||
assert duplex_resource_request_id(fence, "stage0") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage0"
|
||||
assert duplex_resource_request_id(fence, "stage1") == "duplex-s.c2lkLXdpdGgtZGFzaGVz.i.0.e.7.r.stage1"
|
||||
|
||||
|
||||
def test_resource_request_id_codec_separates_session_id_from_incarnation():
|
||||
embedded_incarnation = duplex_resource_request_id(
|
||||
DuplexFence("foo-i1", incarnation=0),
|
||||
"stage0",
|
||||
)
|
||||
actual_incarnation = duplex_resource_request_id(
|
||||
DuplexFence("foo", incarnation=1),
|
||||
"stage0",
|
||||
)
|
||||
|
||||
assert embedded_incarnation != actual_incarnation
|
||||
|
||||
|
||||
def test_resource_request_id_session_membership_uses_encoded_identity():
|
||||
request_id = duplex_resource_request_id(
|
||||
DuplexFence("sid-with-dashes", incarnation=2, epoch=7),
|
||||
"stage0",
|
||||
)
|
||||
|
||||
assert duplex_resource_request_belongs_to_session(request_id, "sid-with-dashes") is True
|
||||
assert duplex_resource_request_belongs_to_session(request_id, "sid") is False
|
||||
assert duplex_resource_request_belongs_to_session("duplex-s.invalid.i.x.e.7.r.stage0", "sid") is False
|
||||
|
||||
|
||||
def test_placeholder_budget_is_planned_inside_omni_engine_boundary():
|
||||
fence = DuplexFence("sid", turn_id=1, response_seq=1)
|
||||
prompt = build_duplex_data_plane_prompt(
|
||||
request_id=duplex_resource_request_id(fence, "stage0"),
|
||||
fence=fence,
|
||||
session_config={},
|
||||
runtime_config={},
|
||||
seq=2,
|
||||
turn_seq=1,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
payload={
|
||||
"audio": "AAAAAA==",
|
||||
"format": "pcm_f32le",
|
||||
"duplex_num_input_tokens": 999,
|
||||
"num_input_tokens": 999,
|
||||
},
|
||||
final=False,
|
||||
)
|
||||
|
||||
assert len(prompt["prompt_token_ids"]) == 16
|
||||
assert prompt["model_intermediate_buffer"]["duplex"]["fence"] == fence
|
||||
assert prompt["model_intermediate_buffer"]["duplex"]["scheduler_token_budget"] == 16
|
||||
|
||||
|
||||
def _jpeg_b64(width: int, height: int) -> str:
|
||||
import base64
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (width, height), (32, 64, 96)).save(buffer, format="JPEG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("ascii")
|
||||
|
||||
|
||||
def _pcm_unit_payload(frames: list[str]) -> dict[str, object]:
|
||||
import base64
|
||||
|
||||
return {
|
||||
"audio": base64.b64encode(b"\x00" * (16000 * 4)).decode("ascii"),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
"video_frames": frames,
|
||||
}
|
||||
|
||||
|
||||
def test_duplex_scheduler_token_budget_one_frame_is_one_block():
|
||||
audio_only = duplex_scheduler_token_budget(_pcm_unit_payload([]))
|
||||
one_frame = duplex_scheduler_token_budget(_pcm_unit_payload([_jpeg_b64(960, 540)]))
|
||||
|
||||
assert one_frame - audio_only == 66
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_size", "expected_blocks"),
|
||||
[
|
||||
((960, 540), 4), # official camera frame: source + 2 HD patches + composite
|
||||
((540, 960), 4), # portrait phone capture
|
||||
((640, 360), 4),
|
||||
((448, 448), 2), # fits one tile: the processor does not slice it
|
||||
((320, 240), 2),
|
||||
],
|
||||
)
|
||||
def test_duplex_scheduler_token_budget_matches_stage0_hd_slicing(base_size, expected_blocks):
|
||||
"""A stacked pair is processed with max_slice_nums=[2, 1]; the budget must
|
||||
equal the blocks Stage0 builds for that base-frame size."""
|
||||
audio_only = duplex_scheduler_token_budget(_pcm_unit_payload([]))
|
||||
pair = duplex_scheduler_token_budget(_pcm_unit_payload([_jpeg_b64(*base_size), _jpeg_b64(448, 448)]))
|
||||
|
||||
assert pair - audio_only == expected_blocks * 66
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("size", "expected_patches"),
|
||||
[
|
||||
# multiple = min(ceil(w * h / 448**2), max_slice_nums); <= 1 is never sliced.
|
||||
((448, 448), 0), # exactly one tile
|
||||
((320, 240), 0), # smaller than one tile
|
||||
((447, 449), 0), # just under one tile
|
||||
((449, 449), 2), # just over: the processor cuts a 2-tile grid
|
||||
((640, 360), 2),
|
||||
((960, 540), 2), # official camera frame
|
||||
((540, 960), 2), # same frame rotated
|
||||
((1920, 1080), 2), # capped by max_slice_nums=2, not by area
|
||||
],
|
||||
)
|
||||
def test_duplex_hd_slice_count_matches_official_grid(size, expected_patches):
|
||||
"""Lock the port of ``MiniCPMVImageProcessor.get_sliced_grid``.
|
||||
|
||||
Stage0 slices a stacked unit with ``max_slice_nums=[2, 1]``, so the patch
|
||||
count for the base frame decides the scheduler budget. A frame that fits one
|
||||
``scale_resolution=448`` tile is not sliced at all.
|
||||
"""
|
||||
assert _duplex_hd_slice_count(size, 2) == expected_patches
|
||||
|
||||
|
||||
def test_duplex_scheduler_token_budget_unreadable_base_frame_keeps_hd_fallback():
|
||||
audio_only = duplex_scheduler_token_budget(_pcm_unit_payload([]))
|
||||
pair = duplex_scheduler_token_budget(_pcm_unit_payload(["not-an-image", _jpeg_b64(448, 448)]))
|
||||
|
||||
assert pair - audio_only == 4 * 66
|
||||
@@ -0,0 +1,612 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Session-state semantics of the engine-resident duplex session (``DuplexEngineSession``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import FrozenInstanceError
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.config import (
|
||||
DuplexCapabilities,
|
||||
DuplexConfigError,
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSessionConfig,
|
||||
DuplexTurnEventType,
|
||||
DuplexTurnState,
|
||||
ResponseCreateOptions,
|
||||
)
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence
|
||||
from vllm_omni.engine.duplex.events import TurnEvent
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession, DuplexFenceMismatchError
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
|
||||
minicpmo45_native_capabilities,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _session(session_id: str = "duplex-test", config: DuplexSessionConfig | None = None) -> DuplexEngineSession:
|
||||
return DuplexEngineSession(session_id=session_id, config=config or DuplexSessionConfig(model="test-model"))
|
||||
|
||||
|
||||
def test_commit_audio_input_does_not_advance_model_turn_identity():
|
||||
session = _session()
|
||||
|
||||
first = session.commit_audio_input(transcript="first chunk")
|
||||
second = session.commit_audio_input(transcript="second chunk")
|
||||
|
||||
assert session.input_commit_seq == 2
|
||||
assert first.input_commit_seq == 1
|
||||
assert second.input_commit_seq == 2
|
||||
assert first.turn_id == second.turn_id == session.turn_id == 0
|
||||
assert first.message["transcript"] == "first chunk"
|
||||
assert first.message["content"] == [
|
||||
{"type": "audio_url", "audio_url": {"url": "native-duplex:input-audio"}, "transcript": "first chunk"}
|
||||
]
|
||||
assert len(session.history) == 2
|
||||
assert session.turn_state == DuplexTurnState.USER_COMMITTED
|
||||
|
||||
|
||||
def test_commit_audio_input_accepts_explicit_turn_id():
|
||||
session = _session()
|
||||
|
||||
committed = session.commit_audio_input(turn_id=4)
|
||||
|
||||
assert committed.turn_id == 4
|
||||
assert session.turn_id == 0
|
||||
assert "transcript" not in committed.message
|
||||
|
||||
|
||||
# ---- response options ----
|
||||
|
||||
|
||||
def test_response_options_apply_to_one_response_without_mutating_session_defaults():
|
||||
session = _session(config=DuplexSessionConfig(instructions="base", voice="base-voice", max_tokens=64))
|
||||
session.reserve_response_options(
|
||||
ResponseCreateOptions(instructions="one response", voice="override-voice", max_tokens=8)
|
||||
)
|
||||
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.response_config is session.config
|
||||
session.begin_response()
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.response_config.instructions == "one response"
|
||||
assert session.response_config.voice == "override-voice"
|
||||
assert session.response_config.max_tokens == 8
|
||||
|
||||
session.end_response()
|
||||
assert session.response_config is session.config
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.config.max_tokens == 64
|
||||
|
||||
|
||||
def test_response_options_cannot_overwrite_an_unconsumed_reservation():
|
||||
session = _session(config=DuplexSessionConfig(instructions="base"))
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="first"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="already reserved"):
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="second"))
|
||||
|
||||
session.begin_response()
|
||||
assert session.response_config.instructions == "first"
|
||||
|
||||
|
||||
def test_response_options_cannot_be_reserved_while_response_is_active():
|
||||
session = _session(config=DuplexSessionConfig(instructions="base"))
|
||||
session.begin_response()
|
||||
|
||||
with pytest.raises(RuntimeError, match="active"):
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="too late"))
|
||||
|
||||
|
||||
def test_discarded_response_options_do_not_apply():
|
||||
session = _session(config=DuplexSessionConfig(instructions="base"))
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="dropped"))
|
||||
session.discard_response_options()
|
||||
|
||||
session.begin_response()
|
||||
|
||||
assert session.response_config.instructions == "base"
|
||||
|
||||
|
||||
# ---- response / overlap identity ----
|
||||
|
||||
|
||||
def test_session_owns_response_and_overlap_identity():
|
||||
session = _session()
|
||||
|
||||
response_id = session.begin_response()
|
||||
assert session.turn_state == DuplexTurnState.ASSISTANT_GENERATING
|
||||
session.accumulate_overlap_speech(320)
|
||||
session.accumulate_overlap_speech(180)
|
||||
session.end_response()
|
||||
|
||||
assert session.active_response_id is None
|
||||
assert session.last_response_id == response_id
|
||||
assert session.overlap_speech_ms == 500
|
||||
assert session.reset_overlap_speech() == 500
|
||||
assert session.overlap_speech_ms == 0
|
||||
assert session.turn_state == DuplexTurnState.IDLE
|
||||
|
||||
|
||||
def test_session_composes_single_owner_ledgers_with_immutable_views():
|
||||
session = _session()
|
||||
|
||||
session.bind_request("req-1")
|
||||
response_id = session.begin_response(turn_id=3)
|
||||
session.mark_audio_sent(duration_ms=240)
|
||||
|
||||
assert session.active_request_id == "req-1"
|
||||
assert session.active_response_id == response_id
|
||||
assert session.active_response_turn_id == 3
|
||||
assert session.playback.sent_ms == 240
|
||||
assert session.turn_state == DuplexTurnState.ASSISTANT_PLAYING
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
session.playback.sent_ms = 480 # type: ignore[misc]
|
||||
assert session.playback.sent_ms == 240
|
||||
|
||||
|
||||
def test_barge_in_advances_epoch_and_drops_uncommitted_assistant_text():
|
||||
session = _session()
|
||||
response_id = session.begin_response()
|
||||
session.bind_request("chatcmpl-duplex-test")
|
||||
session.append_assistant_text("unplayed answer")
|
||||
|
||||
new_epoch = session.barge_in()
|
||||
|
||||
assert response_id is not None
|
||||
assert new_epoch == 1
|
||||
assert session.epoch == 1
|
||||
assert session.accepted_fence == DuplexFence(session.session_id, epoch=1, turn_id=0)
|
||||
assert session.active_request_id is None
|
||||
assert session.active_response_id is None
|
||||
assert session.assistant_text_buffer == ()
|
||||
assert session.history == ()
|
||||
assert session.turn_state == DuplexTurnState.BARGE_IN
|
||||
|
||||
|
||||
# ---- playback ledger ----
|
||||
|
||||
|
||||
def test_playback_ack_tracks_committed_cursor_separately():
|
||||
session = _session()
|
||||
session.mark_audio_sent(duration_ms=10_000)
|
||||
|
||||
session.acknowledge_playback(played_ms=2_000)
|
||||
|
||||
assert session.playback.generated_ms == 10_000
|
||||
assert session.playback.sent_ms == 10_000
|
||||
assert session.playback.played_ms == 2_000
|
||||
assert session.playback.committed_ms == 2_000
|
||||
assert session.playback.as_dict() == {
|
||||
"generated_ms": 10_000,
|
||||
"sent_ms": 10_000,
|
||||
"played_ms": 2_000,
|
||||
"committed_ms": 2_000,
|
||||
}
|
||||
|
||||
|
||||
def test_playback_ack_is_scoped_per_response():
|
||||
session = _session()
|
||||
first = session.begin_response()
|
||||
session.mark_audio_sent(duration_ms=3_000)
|
||||
session.end_response()
|
||||
second = session.begin_response()
|
||||
session.mark_audio_sent(duration_ms=1_000)
|
||||
|
||||
session.acknowledge_playback(played_ms=2_500, response_id=first)
|
||||
|
||||
assert session.playback_for_response(first).played_ms == 2_500
|
||||
assert session.playback_for_response(second).played_ms == 0
|
||||
assert session.playback.played_ms == 0
|
||||
|
||||
|
||||
def test_history_commit_uses_audio_text_alignment_marks():
|
||||
session = _session()
|
||||
session.begin_response()
|
||||
|
||||
session.append_assistant_text("hello ")
|
||||
session.mark_audio_sent(duration_ms=1_000, text_chars=6)
|
||||
session.append_assistant_text("world")
|
||||
session.mark_audio_sent(duration_ms=2_000, text_chars=11)
|
||||
assert [(mark.text_chars, mark.audio_end_ms) for mark in session.assistant_audio_text_marks] == [
|
||||
(6, 1_000),
|
||||
(11, 2_000),
|
||||
]
|
||||
session.acknowledge_playback(played_ms=1_200, committed_ms=1_200)
|
||||
|
||||
committed = session.end_response(commit_text=True)
|
||||
|
||||
assert committed == {"role": "assistant", "content": "hello w"}
|
||||
assert session.history[-1] == committed
|
||||
assert session.last_assistant_full_message == {"role": "assistant", "content": "hello world"}
|
||||
|
||||
|
||||
def test_history_commit_with_ack_only_policy_defers_unacknowledged_text():
|
||||
session = _session(config=DuplexSessionConfig(playback_commit_policy="ack_only"))
|
||||
response_id = session.begin_response()
|
||||
session.append_assistant_text("never played")
|
||||
session.mark_audio_sent(duration_ms=1_000, text_chars=12)
|
||||
|
||||
committed = session.end_response(commit_text=True)
|
||||
|
||||
assert committed is None
|
||||
assert session.history == ()
|
||||
assert f"item_{response_id}" in session.pending_history_item_ids
|
||||
|
||||
|
||||
def test_truncate_history_item_uses_response_alignment_marks():
|
||||
session = _session()
|
||||
response_id = session.begin_response()
|
||||
session.append_assistant_text("hello ")
|
||||
session.mark_audio_sent(duration_ms=1_000, text_chars=6)
|
||||
session.append_assistant_text("world")
|
||||
session.mark_audio_sent(duration_ms=2_000, text_chars=11)
|
||||
session.acknowledge_playback(played_ms=2_000)
|
||||
committed = session.end_response(commit_text=True)
|
||||
assert committed == {"role": "assistant", "content": "hello world"}
|
||||
item_id = f"item_{response_id}"
|
||||
session.register_history_item(item_id, committed)
|
||||
|
||||
assert session.truncate_history_item(item_id, audio_end_ms=1_000) is True
|
||||
|
||||
assert session.history[-1] == {"role": "assistant", "content": "hello"}
|
||||
assert session.history_item_ids[item_id] == {"role": "assistant", "content": "hello"}
|
||||
assert session.delete_history_item(item_id) is True
|
||||
assert session.history == ()
|
||||
|
||||
|
||||
# ---- turn signals / model turns ----
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event_type", "turn_state"),
|
||||
[
|
||||
(DuplexTurnEventType.USER_STARTED, DuplexTurnState.USER_SPEAKING),
|
||||
(DuplexTurnEventType.USER_COMMITTED, DuplexTurnState.USER_COMMITTED),
|
||||
(DuplexTurnEventType.ASSISTANT_STARTED, DuplexTurnState.ASSISTANT_GENERATING),
|
||||
(DuplexTurnEventType.ASSISTANT_DONE, DuplexTurnState.IDLE),
|
||||
(DuplexTurnEventType.BARGE_IN, DuplexTurnState.BARGE_IN),
|
||||
],
|
||||
)
|
||||
def test_signal_turn_transitions_and_returns_typed_turn_event(event_type, turn_state):
|
||||
session = _session()
|
||||
|
||||
event = session.signal_turn(event_type.value)
|
||||
|
||||
assert isinstance(event, TurnEvent)
|
||||
assert session.turn_state == turn_state
|
||||
wire = event.to_realtime()
|
||||
assert wire["type"] == "turn.event"
|
||||
assert wire["event"] == event_type.value
|
||||
assert wire["turn_state"] == turn_state.value
|
||||
|
||||
|
||||
def test_signal_turn_playback_ack_updates_cursor_and_close_marks_session_closing():
|
||||
session = _session()
|
||||
session.mark_audio_sent(duration_ms=5_000)
|
||||
|
||||
ack = session.signal_turn("playback_ack", {"played_ms": 1_500, "committed_ms": 1_000})
|
||||
assert ack.event == "playback_ack"
|
||||
assert session.playback.played_ms == 1_500
|
||||
assert session.playback.committed_ms == 1_000
|
||||
|
||||
close = session.signal_turn(DuplexTurnEventType.CLOSE.value)
|
||||
assert close.event == "close"
|
||||
assert session.state.value == "closing"
|
||||
|
||||
|
||||
def test_complete_model_turn_advances_turn_id_and_fence():
|
||||
session = _session()
|
||||
assert session.fence == DuplexFence(session.session_id, epoch=0, turn_id=0)
|
||||
|
||||
session.complete_model_turn(0)
|
||||
|
||||
assert session.turn_id == 1
|
||||
assert session.fence == DuplexFence(session.session_id, epoch=0, turn_id=1)
|
||||
assert session.accepted_fence == session.fence
|
||||
|
||||
# A stale terminal for an already-completed turn does not move identity backwards.
|
||||
session.complete_model_turn(0)
|
||||
assert session.turn_id == 1
|
||||
session.complete_model_turn(3)
|
||||
assert session.turn_id == 4
|
||||
assert session.accepted_fence.turn_id == 4
|
||||
|
||||
|
||||
# ---- fence validation ----
|
||||
|
||||
|
||||
def test_accept_fence_rejects_stale_or_foreign_fences():
|
||||
session = _session("sid-fence")
|
||||
session.complete_model_turn(0)
|
||||
|
||||
with pytest.raises(DuplexFenceMismatchError):
|
||||
session.accept_fence(DuplexFence("sid-fence", epoch=0, turn_id=0))
|
||||
with pytest.raises(DuplexFenceMismatchError):
|
||||
session.accept_fence(DuplexFence("sid-other", epoch=0, turn_id=1))
|
||||
|
||||
session.accept_fence(DuplexFence("sid-fence", epoch=1, turn_id=0))
|
||||
assert session.accepted_fence == DuplexFence("sid-fence", epoch=1, turn_id=0)
|
||||
|
||||
|
||||
def test_prepare_and_commit_append_sequence_chunks_per_turn_and_epoch():
|
||||
session = _session("sid-append")
|
||||
fence = session.fence
|
||||
|
||||
first = session.commit_append(session.prepare_append(fence))
|
||||
second = session.commit_append(session.prepare_append(fence))
|
||||
assert (first.seq, first.turn_seq, first.turn_id) == (1, 1, 0)
|
||||
assert (second.seq, second.turn_seq, second.turn_id) == (2, 2, 0)
|
||||
|
||||
session.complete_model_turn(0)
|
||||
third = session.commit_append(session.prepare_append(session.fence))
|
||||
assert (third.seq, third.turn_seq, third.turn_id) == (3, 1, 1)
|
||||
|
||||
# Barge-in advances the epoch (turn_id is kept) and restarts the append sequence.
|
||||
session.barge_in()
|
||||
assert session.fence == DuplexFence("sid-append", epoch=1, turn_id=1)
|
||||
fourth = session.commit_append(session.prepare_append(session.fence))
|
||||
assert (fourth.seq, fourth.turn_seq, fourth.turn_id) == (1, 1, 1)
|
||||
assert session.input_seq == 1
|
||||
|
||||
|
||||
def test_commit_append_rejects_stale_reservation():
|
||||
session = _session("sid-append-stale")
|
||||
fence = session.fence
|
||||
reservation = session.prepare_append(fence)
|
||||
session.commit_append(session.prepare_append(fence))
|
||||
|
||||
with pytest.raises(RuntimeError, match="stale"):
|
||||
session.commit_append(reservation)
|
||||
|
||||
with pytest.raises(DuplexFenceMismatchError):
|
||||
session.prepare_append(DuplexFence("sid-other", epoch=0, turn_id=0))
|
||||
|
||||
|
||||
def test_cancel_fence_releases_stage_requests_and_advances_identity():
|
||||
session = _session("sid-cancel")
|
||||
cancelled = session.fence
|
||||
session.reserve_stage_request(0, "req-a", fence=cancelled)
|
||||
session.bind_stage_request(1, "req-b", fence=cancelled)
|
||||
assert session.stage_request_submitted(1, "req-b") is True
|
||||
assert session.stage_request_submitted(0, "req-a") is False
|
||||
next_fence = DuplexFence("sid-cancel", epoch=1, turn_id=0)
|
||||
|
||||
stale = session.cancel_fence(cancelled, next_fence)
|
||||
|
||||
assert stale == ["req-a", "req-b"]
|
||||
assert session.resource_request_ids() == []
|
||||
assert session.accepted_fence == next_fence
|
||||
|
||||
with pytest.raises(DuplexFenceMismatchError):
|
||||
session.cancel_fence(next_fence, DuplexFence("sid-cancel", epoch=1, turn_id=1))
|
||||
with pytest.raises(DuplexFenceMismatchError):
|
||||
session.cancel_fence(DuplexFence("sid-other", epoch=1, turn_id=0), DuplexFence("sid-cancel", epoch=2))
|
||||
|
||||
|
||||
# ---- public view ----
|
||||
|
||||
|
||||
def test_as_public_dict_exposes_identity_capabilities_and_playback():
|
||||
session = _session("duplex-public", config=DuplexSessionConfig(model="m", voice="alloy"))
|
||||
session.mark_audio_sent(duration_ms=100)
|
||||
|
||||
payload = session.as_public_dict()
|
||||
|
||||
assert payload["id"] == "duplex-public"
|
||||
assert payload["model"] == "m"
|
||||
assert payload["voice"] == "alloy"
|
||||
assert payload["state"] == "open"
|
||||
assert payload["turn_state"] == "assistant_playing"
|
||||
assert payload["epoch"] == 0
|
||||
assert payload["turn_id"] == 0
|
||||
assert payload["capabilities"] == DuplexCapabilities().as_dict()
|
||||
assert payload["playback"] == {"generated_ms": 100, "sent_ms": 100, "played_ms": 0, "committed_ms": 0}
|
||||
|
||||
|
||||
# ---- DuplexSessionConfig.from_realtime ----
|
||||
|
||||
|
||||
def test_from_realtime_rejects_unsupported_audio_formats():
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
DuplexSessionConfig.from_realtime({"input_audio_format": "mp3"})
|
||||
assert excinfo.value.code == "unsupported_audio_format"
|
||||
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
DuplexSessionConfig.from_realtime({"audio": {"output": {"format": {"type": "audio/opus"}}}})
|
||||
assert excinfo.value.code == "unsupported_audio_format"
|
||||
|
||||
|
||||
def test_from_realtime_validates_turn_detection():
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
DuplexSessionConfig.from_realtime({"turn_detection": {"type": "semantic_vad"}})
|
||||
assert excinfo.value.code == "unsupported_turn_detection"
|
||||
assert excinfo.value.param == "turn_detection"
|
||||
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
DuplexSessionConfig.from_realtime({"overlap_policy": "barge_in_on_speech"})
|
||||
assert excinfo.value.code == "unsupported_turn_detection"
|
||||
|
||||
server_vad = DuplexSessionConfig.from_realtime({"turn_detection": {"type": "server_vad"}})
|
||||
assert server_vad.overlap_policy == DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value
|
||||
assert server_vad.extra_body["realtime_turn_detection"]["threshold"] == 0.5
|
||||
|
||||
model_owned = DuplexSessionConfig.from_realtime({"turn_detection": None})
|
||||
assert model_owned.overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert model_owned.extra_body["realtime_turn_detection"] is None
|
||||
|
||||
|
||||
def test_from_realtime_maps_wire_fields_and_stores_realtime_keys_in_extra_body():
|
||||
config = DuplexSessionConfig.from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"instructions": "be brief",
|
||||
"output_audio_format": "pcm16",
|
||||
"max_response_output_tokens": "inf",
|
||||
"audio": {"output": {"voice": "alloy", "speed": 1.25}},
|
||||
"tools": [{"type": "function", "name": "lookup"}],
|
||||
"tool_choice": "auto",
|
||||
"metadata": {"tenant": "t1"},
|
||||
"include": ["item.input_audio_transcription.logprobs"],
|
||||
"input_audio_transcription": {"model": "whisper-1"},
|
||||
"extra_body": {"custom": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert config.model == "openbmb/MiniCPM-o-4_5"
|
||||
assert config.instructions == "be brief"
|
||||
assert config.voice == "alloy"
|
||||
assert config.speed == 1.25
|
||||
assert config.max_tokens is None
|
||||
assert config.response_format == "pcm"
|
||||
assert config.modalities == ["text", "audio"]
|
||||
assert config.extra_body["custom"] is True
|
||||
assert config.extra_body["realtime_tools"] == [{"type": "function", "name": "lookup"}]
|
||||
assert config.extra_body["realtime_tool_choice"] == "auto"
|
||||
assert config.extra_body["realtime_metadata"] == {"tenant": "t1"}
|
||||
assert config.extra_body["realtime_include"] == ["item.input_audio_transcription.logprobs"]
|
||||
assert config.extra_body["realtime_input_audio_transcription"] == {"model": "whisper-1"}
|
||||
assert config.extra_body["realtime_audio"] == {"output": {"voice": "alloy", "speed": 1.25}}
|
||||
assert config.extra_body["realtime_output_audio_format"] == "pcm16"
|
||||
assert "extra_body" not in config.extra_body["realtime_session_payload"]
|
||||
assert config.extra_body["realtime_session_payload"]["model"] == "openbmb/MiniCPM-o-4_5"
|
||||
|
||||
|
||||
def test_from_realtime_uses_served_model_when_payload_has_none():
|
||||
config = DuplexSessionConfig.from_realtime({}, model="served-model")
|
||||
|
||||
assert config.model == "served-model"
|
||||
assert config.response_format == "pcm"
|
||||
assert config.idle_timeout_s == 300.0
|
||||
|
||||
|
||||
def test_from_realtime_ignores_client_chosen_session_ids():
|
||||
config = DuplexSessionConfig.from_realtime({"id": "client-chosen", "session_id": "also-client-chosen"})
|
||||
|
||||
assert not hasattr(config, "session_id")
|
||||
assert "id" not in config.as_dict()
|
||||
session = _session("duplex-engine-allocated", config=config)
|
||||
assert session.as_public_dict()["id"] == "duplex-engine-allocated"
|
||||
|
||||
|
||||
# ---- DuplexSessionConfig.apply_realtime_update ----
|
||||
|
||||
|
||||
def test_apply_realtime_update_rejects_model_change():
|
||||
config = DuplexSessionConfig(model="served-model")
|
||||
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
config.apply_realtime_update({"model": "other-model"}, session_id="sid")
|
||||
|
||||
assert excinfo.value.code == "model_update_unsupported"
|
||||
assert config.model == "served-model"
|
||||
|
||||
|
||||
def test_apply_realtime_update_rejects_voice_change_after_audio_started():
|
||||
config = DuplexSessionConfig(model="m", voice="alloy")
|
||||
|
||||
config.apply_realtime_update({"voice": "verse"}, audio_started=False)
|
||||
assert config.voice == "verse"
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
config.apply_realtime_update({"audio": {"output": {"voice": "alloy"}}}, audio_started=True)
|
||||
assert excinfo.value.code == "voice_update_after_audio_unsupported"
|
||||
assert config.voice == "verse"
|
||||
|
||||
|
||||
def test_apply_realtime_update_rejects_ref_audio_change():
|
||||
config = DuplexSessionConfig(model="m")
|
||||
|
||||
with pytest.raises(DuplexConfigError) as excinfo:
|
||||
config.apply_realtime_update({"ref_audio": "/tmp/voice.wav"})
|
||||
|
||||
assert excinfo.value.code == "ref_audio_update_unsupported"
|
||||
assert config.ref_audio is None
|
||||
|
||||
|
||||
def test_apply_realtime_update_patches_fields_and_realtime_keys():
|
||||
config = DuplexSessionConfig(model="m", instructions="old", extra_body={"realtime_tools": [{"name": "x"}]})
|
||||
|
||||
config.apply_realtime_update(
|
||||
{
|
||||
"instructions": "new",
|
||||
"output_audio_format": "g711_ulaw",
|
||||
"max_output_tokens": 32,
|
||||
"tools": None,
|
||||
"metadata": {"k": "v"},
|
||||
"overlap_policy": "not-a-policy",
|
||||
}
|
||||
)
|
||||
|
||||
assert config.instructions == "new"
|
||||
assert config.response_format == "pcm"
|
||||
assert config.max_tokens == 32
|
||||
assert "realtime_tools" not in config.extra_body
|
||||
assert config.extra_body["realtime_metadata"] == {"k": "v"}
|
||||
assert config.overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert config.extra_body["realtime_session_payload"]["instructions"] == "new"
|
||||
|
||||
|
||||
# ---- overlap policy / capabilities ----
|
||||
|
||||
|
||||
def test_overlap_policy_defaults_and_invalid_values_to_listen_only():
|
||||
assert DuplexSessionConfig().overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert DuplexSessionConfig._normalize_overlap_policy("auto") == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert DuplexSessionConfig._normalize_overlap_policy("not-a-policy") == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert (
|
||||
DuplexSessionConfig._normalize_overlap_policy(" Barge_In_On_Speech ")
|
||||
== DuplexOverlapPolicy.BARGE_IN_ON_SPEECH.value
|
||||
)
|
||||
assert DuplexSessionConfig.from_event({"session": {"overlap_policy": "bogus"}}).overlap_policy == "listen_only"
|
||||
|
||||
|
||||
def test_capabilities_as_dict_reports_model_native_duplex_constants():
|
||||
caps = DuplexCapabilities().as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "model_native_duplex"
|
||||
assert caps["input_modes"] == ["append_audio_chunk"]
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_core_kv_lease"] is False
|
||||
assert "client_event" in caps["signal_sources"]
|
||||
assert set(caps) == set(minicpmo45_native_capabilities().as_dict())
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
|
||||
caps = minicpmo45_native_capabilities(max_sessions=2).as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "model_native_duplex"
|
||||
assert caps["supports_input_append"] is True
|
||||
assert caps["input_modes"] == ["append_audio_chunk"]
|
||||
assert caps["adapter_patterns"] == ["scheduler_data_plane"]
|
||||
assert caps["supports_model_internal_state"] is True
|
||||
assert caps["requires_model_runner_kv"] is True
|
||||
assert caps["requires_native_stage_role"] is True
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_core_kv_lease"] is False
|
||||
assert caps["supports_stage_resumption"] is True
|
||||
assert caps["supports_scheduler_native_append"] is False
|
||||
assert caps["supports_core_resumable_request"] is True
|
||||
assert caps["supports_stage_connector_handoff"] is True
|
||||
assert caps["supports_audio_truncate"] is True
|
||||
assert caps["supports_barge_in"] is True
|
||||
assert caps["target_barge_in_latency_ms"] is None
|
||||
assert caps["supports_multi_session"] is True
|
||||
assert caps["supports_multi_session_same_replica"] is True
|
||||
assert caps["supports_session_lease"] is True
|
||||
assert caps["supports_session_resume"] is True
|
||||
assert caps["session_admission_mode"] == "engine_managed"
|
||||
assert caps["stage_handoff_transport"] == "scheduler_data_plane"
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment():
|
||||
caps = minicpmo45_native_capabilities(max_sessions=1).as_dict()
|
||||
|
||||
assert caps["supports_multi_session"] is False
|
||||
assert caps["supports_multi_session_same_replica"] is False
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The overlap decision, reached directly.
|
||||
|
||||
These rules decide what happens when input arrives while the model is speaking.
|
||||
They were unreachable except by driving a whole append through the runner, so
|
||||
none of them had a test that named the rule it was checking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.config import DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.session import overlap_policy
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _session(**config: object) -> DuplexEngineSession:
|
||||
values: dict[str, object] = {"model": "fake-model"}
|
||||
values.update(config)
|
||||
return DuplexEngineSession(session_id="sid", config=DuplexSessionConfig(**values))
|
||||
|
||||
|
||||
def _pcm16(rms: float, samples: int = 1600) -> str:
|
||||
data = np.full(samples, int(rms * 32768), dtype=np.int16)
|
||||
return base64.b64encode(data.tobytes()).decode("ascii")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Speech detection #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", [True, False])
|
||||
def test_an_explicit_is_speech_flag_wins_over_the_audio(flag: bool) -> None:
|
||||
session = _session()
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.9 if not flag else 0.0)}
|
||||
assert overlap_policy.input_looks_like_speech(session, {"is_speech": flag}, payload) is flag
|
||||
|
||||
|
||||
def test_vad_probability_decides_when_no_flag_is_given() -> None:
|
||||
session = _session()
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.0)}
|
||||
assert overlap_policy.input_looks_like_speech(session, {"vad": {"speech_probability": 0.9}}, payload) is True
|
||||
assert overlap_policy.input_looks_like_speech(session, {"vad": {"speech_probability": 0.1}}, payload) is False
|
||||
|
||||
|
||||
def test_quiet_audio_below_the_configured_rms_is_not_speech() -> None:
|
||||
session = _session(overlap_silence_rms=0.05)
|
||||
loud = {"format": "pcm16", "audio": _pcm16(0.5)}
|
||||
quiet = {"format": "pcm16", "audio": _pcm16(0.001)}
|
||||
assert overlap_policy.input_looks_like_speech(session, {}, loud) is True
|
||||
assert overlap_policy.input_looks_like_speech(session, {}, quiet) is False
|
||||
|
||||
|
||||
def test_undecodable_audio_is_treated_as_speech() -> None:
|
||||
"""Fail toward listening: dropping real speech is worse than a spurious turn."""
|
||||
session = _session()
|
||||
assert overlap_policy.input_looks_like_speech(session, {}, {"format": "pcm16", "audio": "!!not base64!!"}) is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Barge-in #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_an_explicit_client_overlap_action_requests_barge_in() -> None:
|
||||
session = _session()
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.5)}
|
||||
decision = overlap_policy.decide(session, {"overlap_action": "barge_in"}, payload, auto_responds=True)
|
||||
assert decision["action"] == "barge_in"
|
||||
assert decision["reason"] == "client_overlap_action"
|
||||
|
||||
|
||||
def test_barge_in_is_deferred_when_the_session_cannot_support_it() -> None:
|
||||
session = _session()
|
||||
session.capabilities = type(session.capabilities)(supports_barge_in=False)
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.5)}
|
||||
decision = overlap_policy.decide(session, {"force_barge_in": True}, payload, auto_responds=True)
|
||||
assert decision["action"] != "barge_in"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Force-listen #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_a_short_response_create_is_forced_to_listen() -> None:
|
||||
"""A sub-threshold ack should not take the turn from the model."""
|
||||
session = _session(overlap_short_ack_ms=700)
|
||||
short = {"response_create": True, "duration_ms": 200}
|
||||
long = {"response_create": True, "duration_ms": 1500}
|
||||
assert overlap_policy.should_force_listen_for_short_commit(session, short, {}) is True
|
||||
assert overlap_policy.should_force_listen_for_short_commit(session, long, {}) is False
|
||||
|
||||
|
||||
def test_force_barge_in_overrides_the_short_ack_rule() -> None:
|
||||
session = _session(overlap_short_ack_ms=700)
|
||||
event = {"response_create": True, "duration_ms": 200, "force_barge_in": True}
|
||||
assert overlap_policy.should_force_listen_for_short_commit(session, event, {}) is False
|
||||
|
||||
|
||||
def test_auto_response_overlap_forces_listen_only_when_auto_responding() -> None:
|
||||
event = {"force_listen": True}
|
||||
assert overlap_policy.should_force_listen_for_auto_response_overlap(event, {}, auto_responds=True) is True
|
||||
assert overlap_policy.should_force_listen_for_auto_response_overlap(event, {}, auto_responds=False) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Payload merging #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize("speech_first", [True, False], ids=["speech-then-silence", "silence-then-speech"])
|
||||
def test_merging_two_appends_keeps_speech_if_either_half_was_speech(speech_first: bool) -> None:
|
||||
"""Matching rates and decodable audio, so the halves really are concatenated.
|
||||
|
||||
Without a ``sample_rate_hz`` on both, ``merge_audio_payloads`` falls back
|
||||
to the newer chunk before merging, and the speech flag it returned was the
|
||||
second chunk's own -- the test passed without exercising the merge.
|
||||
"""
|
||||
speech = {
|
||||
"format": "pcm_f32le",
|
||||
"audio": base64.b64encode(b"\x01" * 8).decode("ascii"),
|
||||
"sample_rate_hz": 16000,
|
||||
"is_speech": True,
|
||||
}
|
||||
silence = {
|
||||
"format": "pcm_f32le",
|
||||
"audio": base64.b64encode(b"\x00" * 8).decode("ascii"),
|
||||
"sample_rate_hz": 16000,
|
||||
"is_speech": False,
|
||||
}
|
||||
first, second = (speech, silence) if speech_first else (silence, speech)
|
||||
merged = overlap_policy.merge_audio_payloads(first, second)
|
||||
assert base64.b64decode(merged["audio"]) == base64.b64decode(first["audio"]) + base64.b64decode(second["audio"])
|
||||
assert merged["sample_rate_hz"] == 16000
|
||||
assert merged["is_speech"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("first", "second"),
|
||||
[
|
||||
({"format": "pcm_f32le", "audio": "", "sample_rate_hz": 16000}, {"format": "pcm16", "audio": ""}),
|
||||
(
|
||||
{"format": "pcm_f32le", "audio": "", "sample_rate_hz": 16000},
|
||||
{"format": "pcm_f32le", "audio": "", "sample_rate_hz": 24000},
|
||||
),
|
||||
(
|
||||
{"format": "pcm_f32le", "audio": "!!not base64!!", "sample_rate_hz": 16000},
|
||||
{"format": "pcm_f32le", "audio": "", "sample_rate_hz": 16000},
|
||||
),
|
||||
],
|
||||
ids=["format-mismatch", "rate-mismatch", "undecodable"],
|
||||
)
|
||||
def test_unmergeable_payloads_fall_back_to_the_newer_one(first: dict, second: dict) -> None:
|
||||
"""Whatever cannot be concatenated keeps the newer chunk and drops the older.
|
||||
|
||||
Worth pinning: the fallback discards ``first`` silently, so a bug that makes
|
||||
two mergeable payloads look mismatched loses buffered audio rather than
|
||||
raising.
|
||||
"""
|
||||
assert overlap_policy.merge_audio_payloads(first, second) == second
|
||||
|
||||
|
||||
def test_mergeable_payloads_are_concatenated_in_order() -> None:
|
||||
head = bytes([1, 2, 3, 4])
|
||||
tail = bytes([5, 6, 7, 8])
|
||||
first = {"format": "pcm_f32le", "audio": base64.b64encode(head).decode(), "sample_rate_hz": 16000}
|
||||
second = {"format": "pcm_f32le", "audio": base64.b64encode(tail).decode(), "sample_rate_hz": 16000}
|
||||
merged = overlap_policy.merge_audio_payloads(first, second)
|
||||
assert base64.b64decode(merged["audio"]) == head + tail
|
||||
assert merged["sample_rate_hz"] == 16000
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Server VAD + barge_in_on_speech #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_server_vad_speech_started_reaches_a_barge_in_decision() -> None:
|
||||
"""The configuration this PR introduces must decide, not raise.
|
||||
|
||||
Both call sites read the VAD verdict through the module-level
|
||||
``vad_speech_started`` helper while binding its result to a local of the
|
||||
same name, which made the name local for the whole scope and raised
|
||||
``UnboundLocalError`` on the call itself. Every ``server_vad`` +
|
||||
``barge_in_on_speech`` append hit it, and the client saw ``internal_error``
|
||||
instead of a barge-in. The MiniCPM-o preset is ``listen_only``, so no demo
|
||||
exercised this branch.
|
||||
"""
|
||||
session = _session(overlap_policy="barge_in_on_speech")
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.5)}
|
||||
event = {"vad": {"speech_started": True}}
|
||||
|
||||
decision = overlap_policy.decide(session, event, payload, auto_responds=True)
|
||||
|
||||
assert decision["action"] == "barge_in"
|
||||
assert decision["reason"] == "server_vad_speech_started"
|
||||
assert decision["cancel_reason"] == "turn_detected"
|
||||
|
||||
|
||||
def test_server_vad_mid_utterance_keeps_listening_instead_of_barging_in() -> None:
|
||||
"""``speech_started`` false means the utterance is already running: not a new turn."""
|
||||
session = _session(overlap_policy="barge_in_on_speech")
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.5)}
|
||||
event = {"vad": {"speech_started": False}}
|
||||
|
||||
decision = overlap_policy.decide(session, event, payload, auto_responds=True)
|
||||
|
||||
assert decision["action"] != "barge_in"
|
||||
|
||||
|
||||
def test_barge_in_on_speech_without_vad_still_barges_in_on_speech() -> None:
|
||||
"""No VAD verdict: the policy falls back to its own speech classification."""
|
||||
session = _session(overlap_policy="barge_in_on_speech")
|
||||
payload = {"format": "pcm16", "audio": _pcm16(0.5)}
|
||||
|
||||
decision = overlap_policy.decide(session, {}, payload, auto_responds=True)
|
||||
|
||||
assert decision["action"] == "barge_in"
|
||||
assert decision["reason"] == "barge_in_on_speech"
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,379 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The engine-side Silero VAD: backend selection and endpoint rules.
|
||||
|
||||
The endpoint rules are the part worth pinning hardest. They were reimplemented
|
||||
when turn detection moved into the engine, and drifted from upstream's
|
||||
``ThresholdEndpointPolicy`` in two ways that only show up mid-utterance: a loud
|
||||
frame used to cancel an in-progress silence timer, and ``audio_end_ms`` used to
|
||||
exclude the trailing silence OpenAI says it should include. The parity cases
|
||||
below hold upstream's own outputs, so neither can come back unnoticed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.turn_detection import validate_realtime_turn_detection
|
||||
from vllm_omni.engine.duplex.vad import (
|
||||
SILERO_VAD_SHA256,
|
||||
ServerVADUnavailableError,
|
||||
SileroStreamingVAD,
|
||||
SileroVADBackend,
|
||||
SileroVADBackendProvider,
|
||||
SileroVADConfig,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
FRAME = 512
|
||||
SAMPLE_RATE_HZ = 16_000
|
||||
|
||||
|
||||
def _drive(config: SileroVADConfig, probabilities: list[float]) -> list[tuple[str, int | None]]:
|
||||
"""Feed one probability per frame and collect the endpoint events."""
|
||||
scores = iter(probabilities)
|
||||
vad = SileroStreamingVAD(config, frame_scorer=lambda _frame: next(scores))
|
||||
events: list[tuple[str, int | None]] = []
|
||||
for _ in probabilities:
|
||||
result = vad.process(np.zeros(FRAME, dtype=np.float32))
|
||||
if result.speech_started:
|
||||
events.append(("start", result.speech_start_ms))
|
||||
if result.speech_stopped:
|
||||
events.append(("stop", result.speech_end_ms))
|
||||
return events
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Parity with the serving-side policy this was ported from #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
#: Probability sequences with the endpoint events upstream's
|
||||
#: ``ThresholdEndpointPolicy`` produced for them, captured from
|
||||
#: ``entrypoints/duplex/server_vad.py`` at `e2d2617f` before that module was
|
||||
#: removed. Golden rather than computed because the oracle no longer ships: the
|
||||
#: serving layer is transport only, so the VAD it used to carry is gone. Any
|
||||
#: drift in the engine's rules still breaks these.
|
||||
_PARITY_CASES = {
|
||||
"one utterance": (
|
||||
[0.0, 0.9, 0.9, 0.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
|
||||
{"threshold": 0.5, "prefix_padding_ms": 0, "silence_duration_ms": 100, "min_speech_duration_ms": 32},
|
||||
[("start", 32), ("stop", 256)],
|
||||
),
|
||||
"prefix padding reaches back": (
|
||||
[0.0, 0.9, 0.9, 0.0, 0.0, 0.0, 0.0],
|
||||
{"threshold": 0.5, "prefix_padding_ms": 300, "silence_duration_ms": 64, "min_speech_duration_ms": 32},
|
||||
[("start", 0), ("stop", 160)],
|
||||
),
|
||||
"loud frames inside the silence timer": (
|
||||
[0.9, 0.9, 0.1, 0.45, 0.1, 0.1, 0.1],
|
||||
{"threshold": 0.5, "prefix_padding_ms": 0, "silence_duration_ms": 96, "min_speech_duration_ms": 32},
|
||||
[("start", 0), ("stop", 160)],
|
||||
),
|
||||
"min speech duration rejects a blip": (
|
||||
[0.9, 0.0, 0.9, 0.9, 0.9, 0.0, 0.0, 0.0, 0.0],
|
||||
{"threshold": 0.5, "prefix_padding_ms": 0, "silence_duration_ms": 64, "min_speech_duration_ms": 96},
|
||||
[("start", 64), ("stop", 224)],
|
||||
),
|
||||
"negative threshold floor at a low threshold": (
|
||||
[0.9, 0.9, 0.1, 0.1, 0.1, 0.1],
|
||||
{"threshold": 0.16, "prefix_padding_ms": 0, "silence_duration_ms": 64, "min_speech_duration_ms": 32},
|
||||
[("start", 0)],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("probabilities", "options", "expected"), _PARITY_CASES.values(), ids=list(_PARITY_CASES))
|
||||
def test_endpoint_decisions_match_the_serving_side_policy(
|
||||
probabilities: list[float],
|
||||
options: dict[str, object],
|
||||
expected: list[tuple[str, int | None]],
|
||||
) -> None:
|
||||
"""The engine detector reproduces the policy server VAD shipped with upstream."""
|
||||
assert _drive(SileroVADConfig(**options), probabilities) == expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The two rules that had drifted #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_a_loud_frame_does_not_restart_the_silence_timer() -> None:
|
||||
"""Silero v6.2 hysteresis: once silence is running, a loud frame only delays it.
|
||||
|
||||
The frame at 0.45 sits above the negative threshold (0.35) but below the
|
||||
activation threshold. It must not cancel the pending endpoint, or a speaker
|
||||
who trails off unevenly never gets their turn committed.
|
||||
"""
|
||||
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=96, min_speech_duration_ms=32)
|
||||
assert _drive(config, [0.9, 0.9, 0.1, 0.45, 0.1, 0.1]) == [("start", 0), ("stop", 160)]
|
||||
|
||||
|
||||
def test_audio_end_ms_includes_the_trailing_silence() -> None:
|
||||
"""OpenAI defines audio_end_ms as the end of the audio sent to the model.
|
||||
|
||||
That includes the silence spent deciding the turn was over, so it is the
|
||||
frame boundary the detector stopped on, not where speech last was.
|
||||
"""
|
||||
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=0, silence_duration_ms=100, min_speech_duration_ms=32)
|
||||
events = _drive(config, [0.0, 0.9, 0.9, 0.9, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
|
||||
stop_ms = next(ms for kind, ms in events if kind == "stop")
|
||||
# 8 frames consumed when the timer expires: 8 * 512 / 16 kHz = 256 ms.
|
||||
assert stop_ms == 256
|
||||
|
||||
|
||||
def test_reset_keeps_the_session_clock_and_clamps_the_prefix() -> None:
|
||||
"""A barge-in resets the detector but not the session's timeline.
|
||||
|
||||
Timestamps have to stay comparable with the rest of the session, and prefix
|
||||
padding must not reach back into audio the reset discarded.
|
||||
"""
|
||||
config = SileroVADConfig(threshold=0.5, prefix_padding_ms=300, silence_duration_ms=64, min_speech_duration_ms=32)
|
||||
scores = iter([0.0] * 10 + [0.9, 0.9, 0.9])
|
||||
vad = SileroStreamingVAD(config, frame_scorer=lambda _frame: next(scores))
|
||||
for _ in range(10):
|
||||
vad.process(np.zeros(FRAME, dtype=np.float32))
|
||||
|
||||
stream_start_ms = round(10 * FRAME * 1000 / SAMPLE_RATE_HZ)
|
||||
vad.reset()
|
||||
result = vad.process(np.zeros(FRAME * 3, dtype=np.float32))
|
||||
|
||||
assert result.speech_started
|
||||
# Without the clock the start would land at 0; without the clamp, 300 ms of
|
||||
# prefix would reach back before the reset.
|
||||
assert result.speech_start_ms == stream_start_ms
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Input handling #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _pcm16_b64(samples: np.ndarray) -> str:
|
||||
return base64.b64encode((samples * 32767).astype("<i2").tobytes()).decode("ascii")
|
||||
|
||||
|
||||
def test_pcm16_input_is_accepted_alongside_float32() -> None:
|
||||
config = SileroVADConfig(threshold=0.5)
|
||||
tone = np.zeros(FRAME * 2, dtype=np.float32)
|
||||
seen: list[int] = []
|
||||
vad = SileroStreamingVAD(config, frame_scorer=lambda frame: seen.append(frame.size) or 0.0)
|
||||
vad.process_base64(_pcm16_b64(tone), fmt="pcm16", sample_rate_hz=SAMPLE_RATE_HZ)
|
||||
assert seen == [FRAME, FRAME]
|
||||
|
||||
|
||||
def test_an_odd_pcm16_byte_count_is_rejected() -> None:
|
||||
vad = SileroStreamingVAD(SileroVADConfig(), frame_scorer=lambda _frame: 0.0)
|
||||
with pytest.raises(ValueError, match="incomplete pcm16"):
|
||||
vad.process_base64(base64.b64encode(b"\x00\x01\x02").decode(), fmt="pcm16", sample_rate_hz=SAMPLE_RATE_HZ)
|
||||
|
||||
|
||||
def test_resampling_keeps_the_frame_grid_from_drifting() -> None:
|
||||
"""The 24 kHz remainder carries across chunks, so frame boundaries cannot drift.
|
||||
|
||||
This is a weaker claim than upstream's ``StreamingAudioResampler``: the
|
||||
interpolation is per chunk, so values near a chunk edge differ slightly.
|
||||
What must hold is the frame *count* and alignment, which is what the
|
||||
endpoint timestamps are derived from.
|
||||
"""
|
||||
source = np.sin(np.linspace(0, 40 * np.pi, 24_000, dtype=np.float32))
|
||||
|
||||
def frames_for(chunk_sizes: list[int]) -> int:
|
||||
frames = 0
|
||||
|
||||
def count(_frame: np.ndarray) -> float:
|
||||
nonlocal frames
|
||||
frames += 1
|
||||
return 0.0
|
||||
|
||||
vad = SileroStreamingVAD(SileroVADConfig(), frame_scorer=count)
|
||||
offset = 0
|
||||
for size in chunk_sizes:
|
||||
vad.process_base64(_pcm16_b64(source[offset : offset + size]), fmt="pcm16", sample_rate_hz=24_000)
|
||||
offset += size
|
||||
assert offset == source.size
|
||||
return frames
|
||||
|
||||
whole = frames_for([source.size])
|
||||
split = frames_for([7, 13, 511, 512, 513, source.size - 1_556])
|
||||
assert whole == split == 16_000 // FRAME
|
||||
|
||||
|
||||
def test_a_sample_rate_change_mid_stream_is_rejected() -> None:
|
||||
vad = SileroStreamingVAD(SileroVADConfig(), frame_scorer=lambda _frame: 0.0)
|
||||
chunk = _pcm16_b64(np.zeros(1_024, dtype=np.float32))
|
||||
vad.process_base64(chunk, fmt="pcm16", sample_rate_hz=24_000)
|
||||
with pytest.raises(ValueError, match="sample rate cannot change"):
|
||||
vad.process_base64(chunk, fmt="pcm16", sample_rate_hz=48_000)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Backend selection #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_a_configured_model_path_that_does_not_exist_is_an_error() -> None:
|
||||
"""An operator who names an artifact gets told it is missing, not a fallback."""
|
||||
provider = SileroVADBackendProvider(model_path="/nonexistent/silero_vad.onnx")
|
||||
with pytest.raises(ServerVADUnavailableError, match="does not exist"):
|
||||
provider.get()
|
||||
|
||||
|
||||
def test_a_configured_model_with_the_wrong_checksum_is_refused(tmp_path) -> None:
|
||||
"""A different Silero revision changes endpointing, so the pin is enforced."""
|
||||
impostor = tmp_path / "silero_vad.onnx"
|
||||
impostor.write_bytes(b"not the pinned model")
|
||||
provider = SileroVADBackendProvider(model_path=str(impostor))
|
||||
with pytest.raises(ServerVADUnavailableError, match="checksum mismatch"):
|
||||
provider.get()
|
||||
assert hashlib.sha256(impostor.read_bytes()).hexdigest() != SILERO_VAD_SHA256
|
||||
|
||||
|
||||
def test_the_onnx_backend_is_shared_by_every_session() -> None:
|
||||
"""One ONNX session for the whole engine: its state is passed in, not held."""
|
||||
provider = SileroVADBackendProvider()
|
||||
try:
|
||||
backend = provider.get()
|
||||
except ServerVADUnavailableError as exc:
|
||||
pytest.skip(f"no Silero backend available here: {exc}")
|
||||
if not isinstance(backend, SileroVADBackend):
|
||||
pytest.skip("no local ONNX artifact; the torch fallback is per-session by design")
|
||||
|
||||
assert provider.get() is backend
|
||||
# State is the caller's, so two sessions on one backend cannot interfere.
|
||||
first, second = backend.new_state(), backend.new_state()
|
||||
assert first is not second
|
||||
probability, next_state = backend.infer(np.zeros(FRAME, dtype=np.float32), first)
|
||||
assert 0.0 <= probability <= 1.0
|
||||
assert next_state is not first
|
||||
|
||||
|
||||
def test_the_onnx_backend_holds_the_upstream_silero_v62_contract(monkeypatch, tmp_path) -> None:
|
||||
"""Pin the ONNX call shape without needing the real artifact or runtime.
|
||||
|
||||
The context carry is the part worth pinning: Silero v6.2 is fed 64 samples
|
||||
of the previous frame ahead of the current 512, and that tail travels in the
|
||||
caller's state rather than inside the session -- which is exactly what lets
|
||||
one session serve every duplex session at once.
|
||||
"""
|
||||
sessions: list[FakeInferenceSession] = []
|
||||
|
||||
class FakeSessionOptions:
|
||||
inter_op_num_threads = 0
|
||||
intra_op_num_threads = 0
|
||||
|
||||
class FakeInferenceSession:
|
||||
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
|
||||
self.providers = providers
|
||||
self.sess_options = sess_options
|
||||
self.calls: list[dict[str, np.ndarray]] = []
|
||||
self.run_barrier: threading.Barrier | None = None
|
||||
sessions.append(self)
|
||||
|
||||
def get_inputs(self) -> list[SimpleNamespace]:
|
||||
return [SimpleNamespace(name=name) for name in ("input", "state", "sr")]
|
||||
|
||||
def run(self, _outputs: object, inputs: dict[str, np.ndarray]) -> list[np.ndarray]:
|
||||
self.calls.append({name: np.array(value, copy=True) for name, value in inputs.items()})
|
||||
if self.run_barrier is not None:
|
||||
self.run_barrier.wait(timeout=5)
|
||||
return [
|
||||
np.asarray([[0.75]], dtype=np.float32),
|
||||
np.full((2, 1, 128), len(self.calls), dtype=np.float32),
|
||||
]
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"onnxruntime",
|
||||
SimpleNamespace(SessionOptions=FakeSessionOptions, InferenceSession=FakeInferenceSession),
|
||||
)
|
||||
|
||||
model_path = tmp_path / "silero_vad.onnx"
|
||||
model_path.write_bytes(b"fake-model")
|
||||
backend = SileroVADBackend(model_path)
|
||||
session = sessions[0]
|
||||
|
||||
# Single-threaded on purpose: many sessions share this one ORT session, so
|
||||
# it must not fan out per call.
|
||||
assert session.providers == ["CPUExecutionProvider"]
|
||||
assert session.sess_options.inter_op_num_threads == 1
|
||||
assert session.sess_options.intra_op_num_threads == 1
|
||||
# Warm-up already ran: 64 context + 512 frame.
|
||||
assert session.calls[0]["input"].shape == (1, 576)
|
||||
assert session.calls[0]["state"].shape == (2, 1, 128)
|
||||
assert session.calls[0]["sr"].item() == SAMPLE_RATE_HZ
|
||||
|
||||
state = backend.new_state()
|
||||
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
|
||||
probability, state = backend.infer(first_frame, state)
|
||||
first_call = session.calls[1]
|
||||
|
||||
assert probability == pytest.approx(0.75)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, : backend.context_samples],
|
||||
np.zeros((1, backend.context_samples), dtype=np.float32),
|
||||
)
|
||||
np.testing.assert_array_equal(first_call["input"][:, backend.context_samples :], first_frame[None, :])
|
||||
|
||||
second_frame = -first_frame
|
||||
backend.infer(second_frame, state)
|
||||
second_call = session.calls[2]
|
||||
# The previous frame's tail becomes this frame's context.
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, : backend.context_samples],
|
||||
first_frame[None, -backend.context_samples :],
|
||||
)
|
||||
np.testing.assert_array_equal(second_call["state"], np.full((2, 1, 128), 2, dtype=np.float32))
|
||||
|
||||
# Two sessions may enter the shared ORT session at the same time.
|
||||
session.run_barrier = threading.Barrier(2)
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [
|
||||
executor.submit(backend.infer, first_frame, backend.new_state()),
|
||||
executor.submit(backend.infer, second_frame, backend.new_state()),
|
||||
]
|
||||
assert [future.result()[0] for future in futures] == pytest.approx([0.75, 0.75])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Session payload validation #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_an_unknown_server_vad_field_is_refused() -> None:
|
||||
"""A misspelled knob that silently does nothing is worse than a refusal.
|
||||
|
||||
The endpointing would still look configured while running on defaults.
|
||||
"""
|
||||
error = validate_realtime_turn_detection({"turn_detection": {"type": "server_vad", "semantic_eagerness": "high"}})
|
||||
assert error is not None
|
||||
assert "semantic_eagerness" in error
|
||||
|
||||
|
||||
def test_the_documented_server_vad_fields_are_all_accepted() -> None:
|
||||
assert (
|
||||
validate_realtime_turn_detection(
|
||||
{
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
"threshold": 0.6,
|
||||
"prefix_padding_ms": 200,
|
||||
"silence_duration_ms": 400,
|
||||
"create_response": True,
|
||||
"interrupt_response": True,
|
||||
"min_speech_duration_ms": 120,
|
||||
},
|
||||
"overlap_policy": "barge_in_on_speech",
|
||||
}
|
||||
)
|
||||
is None
|
||||
)
|
||||
@@ -54,7 +54,7 @@ def test_abort_async_waits_for_ack(mocker: MockerFixture):
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
"vllm_omni.engine.omni_engine_base.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="abort-rpc-1"),
|
||||
)
|
||||
|
||||
@@ -81,7 +81,7 @@ def test_abort_async_raises_on_orchestrator_error(mocker: MockerFixture):
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
"vllm_omni.engine.omni_engine_base.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="abort-rpc-err"),
|
||||
)
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_abort_async_times_out_without_result(mocker: MockerFixture):
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
"vllm_omni.engine.omni_engine_base.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="abort-rpc-timeout"),
|
||||
)
|
||||
|
||||
@@ -139,7 +139,7 @@ def test_abort_async_preserves_request_queue_backpressure(mocker: MockerFixture)
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
"vllm_omni.engine.omni_engine_base.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="blocked-abort"),
|
||||
)
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ Focuses on the critical behavior: when the orchestrator thread dies,
|
||||
subsequent attempts to collect output raise RuntimeError.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@@ -18,16 +17,6 @@ from pytest_mock import MockerFixture
|
||||
|
||||
from vllm_omni.engine.async_engine_utils import weak_shutdown_async_omni_engine
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.duplex.control_client import (
|
||||
DuplexControlClient,
|
||||
DuplexControlRequestError,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
AppendDuplexInputMessage,
|
||||
DuplexControlResultMessage,
|
||||
DuplexFence,
|
||||
SignalDuplexTurnMessage,
|
||||
)
|
||||
from vllm_omni.engine.messages import (
|
||||
CollectiveRPCResultMessage,
|
||||
ErrorMessage,
|
||||
@@ -166,9 +155,9 @@ def test_output_remains_on_shared_output_path(mocker: MockerFixture):
|
||||
raw_queue = queue.Queue()
|
||||
raw_queue.put_nowait(
|
||||
OutputMessage(
|
||||
request_id="legacy-duplex-request",
|
||||
request_id="shared-path-request",
|
||||
stage_id=0,
|
||||
engine_outputs=OmniRequestOutput(request_id="legacy-duplex-request"),
|
||||
engine_outputs=OmniRequestOutput(request_id="shared-path-request"),
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
@@ -176,293 +165,7 @@ def test_output_remains_on_shared_output_path(mocker: MockerFixture):
|
||||
|
||||
output = engine.try_get_output(timeout=0.01)
|
||||
|
||||
assert output.request_id == "legacy-duplex-request"
|
||||
|
||||
|
||||
def test_duplex_control_client_uses_engine_owned_correlated_transport():
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine._duplex_control_client = None
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(queue.Queue(), queue.Queue())
|
||||
|
||||
try:
|
||||
client = engine._get_duplex_control_client()
|
||||
|
||||
assert isinstance(client, DuplexControlClient)
|
||||
assert client._transport is engine._correlated_rpc_client
|
||||
assert not hasattr(engine, "_submit_rpc_and_wait")
|
||||
finally:
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_duplex_control_api_requires_explicit_fence():
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
engine.open_duplex_session("sid")
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
engine.append_duplex_input("sid", mode="append_audio_chunk", payload={})
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
engine.signal_duplex_turn("sid", event="turn.end")
|
||||
with pytest.raises(TypeError, match="fence"):
|
||||
engine.close_duplex_session("sid")
|
||||
|
||||
|
||||
def test_open_duplex_session_waits_for_control_ack(mocker: MockerFixture):
|
||||
request_q = queue.Queue()
|
||||
rpc_q = queue.Queue()
|
||||
control_result = DuplexControlResultMessage(
|
||||
control_id="ctrl-1",
|
||||
fence=DuplexFence("sid"),
|
||||
operation="open",
|
||||
session_id="sid",
|
||||
ok=False,
|
||||
stage_results=[{"stage_id": 0, "replica_id": 0, "result": {"supported": False}}],
|
||||
unsupported_count=1,
|
||||
error_count=0,
|
||||
)
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="ctrl-1"))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(engine.open_duplex_session, "sid", fence=DuplexFence("sid"), timeout=1)
|
||||
msg = request_q.get(timeout=1)
|
||||
assert msg.type == "open_duplex_session"
|
||||
assert msg.control_id == "ctrl-1"
|
||||
rpc_q.put(control_result)
|
||||
with pytest.raises(DuplexControlRequestError) as exc_info:
|
||||
pending.result(timeout=1)
|
||||
|
||||
assert exc_info.value.result["unsupported_count"] == 1
|
||||
assert exc_info.value.result["stage_results"][0]["result"]["supported"] is False
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_signal_duplex_turn_message_carries_next_fence(mocker: MockerFixture):
|
||||
request_q = queue.Queue()
|
||||
rpc_q = queue.Queue()
|
||||
cancelled_fence = DuplexFence("sid-cancel", epoch=4, turn_id=2)
|
||||
next_fence = DuplexFence("sid-cancel", epoch=5, turn_id=2)
|
||||
control_result = DuplexControlResultMessage(
|
||||
control_id="ctrl-signal",
|
||||
fence=cancelled_fence,
|
||||
operation="signal",
|
||||
session_id="sid-cancel",
|
||||
ok=True,
|
||||
stage_results=[],
|
||||
)
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="ctrl-signal"))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(
|
||||
engine.signal_duplex_turn,
|
||||
"sid-cancel",
|
||||
event="input.cancel",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
session_config={"temperature": 0.0},
|
||||
timeout=1,
|
||||
)
|
||||
msg = request_q.get(timeout=1)
|
||||
assert isinstance(msg, SignalDuplexTurnMessage)
|
||||
assert msg.fence == cancelled_fence
|
||||
assert msg.next_fence == next_fence
|
||||
assert msg.session_config == {"temperature": 0.0}
|
||||
rpc_q.put(control_result)
|
||||
assert pending.result(timeout=1)["ok"] is True
|
||||
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_async_append_does_not_stop_executor_callable(mocker: MockerFixture):
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
completed = threading.Event()
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
|
||||
def blocking_append(*args, **kwargs):
|
||||
del args, kwargs
|
||||
entered.set()
|
||||
release.wait(timeout=1)
|
||||
completed.set()
|
||||
return {}
|
||||
|
||||
mocker.patch.object(engine, "append_duplex_input", side_effect=blocking_append)
|
||||
task = asyncio.create_task(
|
||||
engine.append_duplex_input_async(
|
||||
"sid-executor-cancel",
|
||||
mode="append_audio_chunk",
|
||||
payload=b"audio",
|
||||
fence=DuplexFence("sid-executor-cancel"),
|
||||
)
|
||||
)
|
||||
assert await asyncio.to_thread(entered.wait, 1)
|
||||
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
release.set()
|
||||
|
||||
assert await asyncio.to_thread(completed.wait, 1)
|
||||
|
||||
|
||||
def test_duplex_controls_route_out_of_order_without_global_lock():
|
||||
request_q = queue.Queue()
|
||||
rpc_q = queue.Queue()
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
first_fence = DuplexFence("sid", epoch=1)
|
||||
second_fence = DuplexFence("sid", epoch=2)
|
||||
first = AppendDuplexInputMessage(
|
||||
control_id="first",
|
||||
fence=first_fence,
|
||||
session_id="sid",
|
||||
expected_epoch=1,
|
||||
mode="audio",
|
||||
payload=b"first",
|
||||
)
|
||||
second = AppendDuplexInputMessage(
|
||||
control_id="second",
|
||||
fence=second_fence,
|
||||
session_id="sid",
|
||||
expected_epoch=2,
|
||||
mode="audio",
|
||||
payload=b"second",
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
client = engine._get_duplex_control_client()
|
||||
first_result = executor.submit(client.execute, first, timeout=1)
|
||||
second_result = executor.submit(client.execute, second, timeout=1)
|
||||
submitted = {request_q.get(timeout=1).control_id, request_q.get(timeout=1).control_id}
|
||||
assert submitted == {"first", "second"}
|
||||
rpc_q.put(
|
||||
DuplexControlResultMessage(
|
||||
control_id="second",
|
||||
fence=second_fence,
|
||||
operation="append",
|
||||
session_id="sid",
|
||||
ok=True,
|
||||
stage_results=[],
|
||||
)
|
||||
)
|
||||
rpc_q.put(
|
||||
DuplexControlResultMessage(
|
||||
control_id="first",
|
||||
fence=first_fence,
|
||||
operation="append",
|
||||
session_id="sid",
|
||||
ok=True,
|
||||
stage_results=[],
|
||||
)
|
||||
)
|
||||
|
||||
assert first_result.result(timeout=1)["fence"] == first_fence
|
||||
assert second_result.result(timeout=1)["fence"] == second_fence
|
||||
|
||||
assert not hasattr(engine, "_rpc_lock")
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_terminal_rpc_router_rejects_without_enqueuing_new_control():
|
||||
request_q = queue.Queue()
|
||||
rpc_q = queue.Queue()
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
pending_message = AppendDuplexInputMessage(
|
||||
control_id="pending",
|
||||
fence=DuplexFence("sid"),
|
||||
session_id="sid",
|
||||
mode="audio",
|
||||
payload=b"audio",
|
||||
)
|
||||
message = AppendDuplexInputMessage(
|
||||
control_id="after-fatal",
|
||||
fence=DuplexFence("sid"),
|
||||
session_id="sid",
|
||||
mode="audio",
|
||||
payload=b"audio",
|
||||
)
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(engine._get_duplex_control_client().execute, pending_message, timeout=1)
|
||||
assert request_q.get(timeout=1).control_id == "pending"
|
||||
rpc_q.put(ErrorMessage(error="orchestrator failed", fatal=True))
|
||||
with pytest.raises(RuntimeError, match="orchestrator failed"):
|
||||
pending.result(timeout=1)
|
||||
|
||||
with pytest.raises(RuntimeError, match="orchestrator failed"):
|
||||
engine._get_duplex_control_client().execute(message, timeout=1)
|
||||
|
||||
assert request_q.empty()
|
||||
finally:
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_collective_and_duplex_results_share_router_without_swapping(mocker: MockerFixture):
|
||||
request_q = queue.Queue()
|
||||
rpc_q = queue.Queue()
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
fence = DuplexFence("sid")
|
||||
duplex = AppendDuplexInputMessage(
|
||||
control_id="duplex-control",
|
||||
fence=fence,
|
||||
session_id="sid",
|
||||
mode="audio",
|
||||
payload=b"audio",
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="collective-rpc"))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
duplex_result = executor.submit(engine._get_duplex_control_client().execute, duplex, timeout=1)
|
||||
collective_result = executor.submit(engine.collective_rpc, "health", timeout=1)
|
||||
submitted_types = {request_q.get(timeout=1).type, request_q.get(timeout=1).type}
|
||||
assert submitted_types == {"append_duplex_input", "collective_rpc"}
|
||||
rpc_q.put(
|
||||
CollectiveRPCResultMessage(
|
||||
rpc_id="collective-rpc",
|
||||
method="health",
|
||||
stage_ids=[0],
|
||||
results=["healthy"],
|
||||
)
|
||||
)
|
||||
rpc_q.put(
|
||||
DuplexControlResultMessage(
|
||||
control_id="duplex-control",
|
||||
fence=fence,
|
||||
operation="append",
|
||||
session_id="sid",
|
||||
ok=True,
|
||||
stage_results=[],
|
||||
)
|
||||
)
|
||||
|
||||
assert collective_result.result(timeout=1) == ["healthy"]
|
||||
assert duplex_result.result(timeout=1)["fence"] == fence
|
||||
|
||||
engine._correlated_rpc_client.close()
|
||||
assert output.request_id == "shared-path-request"
|
||||
|
||||
|
||||
def test_collective_rpc_preserves_request_queue_backpressure(mocker: MockerFixture):
|
||||
@@ -483,8 +186,7 @@ def test_collective_rpc_preserves_request_queue_backpressure(mocker: MockerFixtu
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="blocked-rpc"))
|
||||
mocker.patch("vllm_omni.engine.omni_engine_base.uuid.uuid4", return_value=SimpleNamespace(hex="blocked-rpc"))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(engine.collective_rpc, "health", timeout=1)
|
||||
@@ -506,36 +208,6 @@ def test_collective_rpc_preserves_request_queue_backpressure(mocker: MockerFixtu
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_open_duplex_session_raises_on_stage_control_error(mocker: MockerFixture):
|
||||
request_q = queue.Queue()
|
||||
rpc_q = queue.Queue()
|
||||
control_result = DuplexControlResultMessage(
|
||||
control_id="ctrl-error",
|
||||
fence=DuplexFence("sid"),
|
||||
operation="open",
|
||||
session_id="sid",
|
||||
ok=False,
|
||||
stage_results=[{"stage_id": 0, "replica_id": 0, "result": {"supported": False, "error": "boom"}}],
|
||||
unsupported_count=1,
|
||||
error_count=1,
|
||||
)
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
engine._duplex_control_client = None
|
||||
mocker.patch("vllm_omni.engine.async_omni_engine.uuid.uuid4", return_value=SimpleNamespace(hex="ctrl-error"))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
pending = executor.submit(engine.open_duplex_session, "sid", fence=DuplexFence("sid"), timeout=1)
|
||||
request_q.get(timeout=1)
|
||||
rpc_q.put(control_result)
|
||||
with pytest.raises(RuntimeError, match="duplex open failed"):
|
||||
pending.result(timeout=1)
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_shutdown_does_not_race_runtime_cleanup_with_live_orchestrator(mocker: MockerFixture):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine._shutdown_called = False
|
||||
|
||||
@@ -12,7 +12,7 @@ import types
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.data import AttentionConfig
|
||||
from vllm_omni.engine import async_omni_engine as async_omni_engine_module
|
||||
from vllm_omni.engine import omni_engine_base as async_omni_engine_module
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.stage_init_utils import (
|
||||
LogicalStageInitPlan,
|
||||
@@ -188,7 +188,7 @@ def test_stage_engine_core_client_module_reload_keeps_forward_refs_deferred():
|
||||
|
||||
|
||||
def test_async_omni_engine_initialize_stages_passes_log_stats_to_runtime(monkeypatch):
|
||||
import vllm_omni.engine.async_omni_engine as engine_mod
|
||||
import vllm_omni.engine.omni_engine_base as engine_mod
|
||||
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.stage_configs = [types.SimpleNamespace()]
|
||||
|
||||
@@ -33,7 +33,7 @@ def _resolve(monkeypatch, kwargs: dict[str, Any]) -> dict[str, Any]:
|
||||
return "config.yaml", [], None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.engine.async_omni_engine.load_and_resolve_stage_configs",
|
||||
"vllm_omni.engine.omni_engine_base.load_and_resolve_stage_configs",
|
||||
_fake_load,
|
||||
)
|
||||
engine = SimpleNamespace(
|
||||
|
||||
@@ -45,27 +45,62 @@ def test_isolated_import_ignores_conflicting_gpu_visibility(monkeypatch: pytest.
|
||||
_assert_isolated_import_succeeds("import vllm_omni.outputs")
|
||||
|
||||
|
||||
def test_stable_engine_imports_load_duplex_kernel_eagerly() -> None:
|
||||
# The duplex kernel is imported eagerly by the stable engine modules.
|
||||
# Model-specific duplex adapters must still load only via the
|
||||
# dotted-string plugin paths, and the client-side vllm_omni.clients
|
||||
# package must never enter the stable runtime import graph.
|
||||
def test_turn_based_stack_has_no_duplex_vocabulary() -> None:
|
||||
# The generic bases and the turn-based siblings (AsyncOmni / AsyncOmniEngine /
|
||||
# Orchestrator) carry no duplex code: importing them must not load the
|
||||
# engine-side duplex package, the model plugins or the client library.
|
||||
_assert_isolated_import_succeeds("""
|
||||
import sys
|
||||
|
||||
import vllm_omni.engine.async_omni_engine
|
||||
import vllm_omni.engine.omni_engine_base
|
||||
import vllm_omni.engine.orchestrator
|
||||
import vllm_omni.entrypoints.async_omni
|
||||
import vllm_omni.entrypoints.async_omni_base
|
||||
|
||||
forbidden_prefixes = (
|
||||
"vllm_omni.engine.duplex",
|
||||
"vllm_omni.engine.duplex_omni_engine",
|
||||
"vllm_omni.engine.duplex_orchestrator",
|
||||
"vllm_omni.entrypoints.duplex",
|
||||
"vllm_omni.entrypoints.duplex_omni",
|
||||
"vllm_omni.clients",
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
|
||||
"vllm_omni.model_executor.models.nemotron_voicechat.duplex",
|
||||
"vllm_omni.model_executor.models.personaplex.duplex",
|
||||
)
|
||||
loaded = sorted(
|
||||
name
|
||||
for name in sys.modules
|
||||
if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
|
||||
)
|
||||
if loaded:
|
||||
raise SystemExit("turn-based imports loaded duplex modules: " + ", ".join(loaded))
|
||||
""")
|
||||
|
||||
|
||||
def test_duplex_stack_loads_the_framework_but_not_the_model_plugins() -> None:
|
||||
# The duplex siblings import the engine-side duplex framework eagerly;
|
||||
# model plugins still load only through PipelineConfig.duplex_plugin, and
|
||||
# the client-side package never enters the server import graph.
|
||||
_assert_isolated_import_succeeds("""
|
||||
import sys
|
||||
|
||||
import vllm_omni.engine.duplex_omni_engine
|
||||
import vllm_omni.engine.duplex_orchestrator
|
||||
import vllm_omni.entrypoints.duplex_omni
|
||||
import vllm_omni.entrypoints.duplex.serving
|
||||
|
||||
expected_eager = (
|
||||
"vllm_omni.engine.duplex.contracts",
|
||||
"vllm_omni.engine.duplex.control_plane",
|
||||
"vllm_omni.entrypoints.duplex_request_client",
|
||||
"vllm_omni.outputs.duplex",
|
||||
"vllm_omni.engine.duplex.commands",
|
||||
"vllm_omni.engine.duplex.events",
|
||||
"vllm_omni.engine.duplex.session.engine_session",
|
||||
"vllm_omni.engine.duplex.session.manager",
|
||||
"vllm_omni.engine.duplex.plugin",
|
||||
)
|
||||
missing = sorted(name for name in expected_eager if name not in sys.modules)
|
||||
if missing:
|
||||
raise SystemExit("duplex kernel modules not imported eagerly: " + ", ".join(missing))
|
||||
raise SystemExit("duplex framework modules not imported eagerly: " + ", ".join(missing))
|
||||
|
||||
forbidden_prefixes = (
|
||||
"vllm_omni.clients",
|
||||
@@ -79,7 +114,7 @@ loaded = sorted(
|
||||
if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden_prefixes)
|
||||
)
|
||||
if loaded:
|
||||
raise SystemExit("stable imports loaded plugin-only duplex modules: " + ", ".join(loaded))
|
||||
raise SystemExit("duplex imports loaded plugin-only or client modules: " + ", ".join(loaded))
|
||||
""")
|
||||
|
||||
|
||||
@@ -190,16 +225,32 @@ def test_engine_duplex_uses_canonical_contract_module_names() -> None:
|
||||
core_dir = REPO_ROOT / "vllm_omni" / "experimental" / "fullduplex" / "core"
|
||||
|
||||
for name in (
|
||||
"commands.py",
|
||||
"contracts.py",
|
||||
"lease.py",
|
||||
"events.py",
|
||||
"messages.py",
|
||||
"session.py",
|
||||
"control_plane.py",
|
||||
"control_client.py",
|
||||
"runtime.py",
|
||||
"plugin.py",
|
||||
"intermediate.py",
|
||||
):
|
||||
assert (engine_dir / name).is_file()
|
||||
# one session and everything that runs it lives in the session subpackage
|
||||
for name in (
|
||||
"__init__.py",
|
||||
"engine_session.py",
|
||||
"manager.py",
|
||||
"runner.py",
|
||||
"context.py",
|
||||
"emitter.py",
|
||||
"model_channel.py",
|
||||
"control.py",
|
||||
"append_task.py",
|
||||
"helpers.py",
|
||||
"lease.py",
|
||||
):
|
||||
assert (engine_dir / "session" / name).is_file()
|
||||
assert not (engine_dir / "session.py").exists()
|
||||
for removed in ("control_plane.py", "control_client.py", "runtime.py"):
|
||||
assert not (engine_dir / removed).exists()
|
||||
# the duplex_ prefix is dropped inside the duplex package
|
||||
assert not (engine_dir / "duplex_session.py").exists()
|
||||
assert not (engine_dir / "duplex_lease.py").exists()
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``DuplexOmniEngine``: the session message surface over the generic engine base."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.commands import Heartbeat
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
CloseDuplexSessionMessage,
|
||||
DuplexControlResultMessage,
|
||||
DuplexSessionCommandMessage,
|
||||
DuplexSessionError,
|
||||
OpenDuplexSessionMessage,
|
||||
ResumeDuplexSessionMessage,
|
||||
TouchDuplexSessionMessage,
|
||||
)
|
||||
from vllm_omni.engine.duplex_omni_engine import DuplexOmniEngine
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class _FakeRpcClient:
|
||||
def __init__(self, result: Any) -> None:
|
||||
self.result = result
|
||||
self.calls: list[tuple[tuple[str, str], Any, float | None]] = []
|
||||
#: Control RPCs must block rather than raise a raw queue.Full.
|
||||
self.block_on_submit_flags: list[bool] = []
|
||||
|
||||
def execute(self, key, message, *, timeout=None, timeout_message=None, block_on_submit=False):
|
||||
self.calls.append((key, message, timeout))
|
||||
self.block_on_submit_flags.append(block_on_submit)
|
||||
if isinstance(self.result, BaseException):
|
||||
raise self.result
|
||||
return self.result
|
||||
|
||||
|
||||
class _FakeSyncQueue:
|
||||
def __init__(self, *, full: bool = False) -> None:
|
||||
self.items: list[Any] = []
|
||||
self.full = full
|
||||
|
||||
def put(self, item: Any, timeout: float | None = None) -> None:
|
||||
if self.full:
|
||||
raise queue.Full
|
||||
self.items.append(item)
|
||||
|
||||
|
||||
def _engine(result: Any, *, alive: bool = True, full: bool = False) -> DuplexOmniEngine:
|
||||
engine = object.__new__(DuplexOmniEngine)
|
||||
engine._correlated_rpc_client = _FakeRpcClient(result)
|
||||
engine.request_queue = SimpleNamespace(sync_q=_FakeSyncQueue(full=full))
|
||||
engine.is_alive = lambda: alive
|
||||
engine.plugin = None
|
||||
return engine
|
||||
|
||||
|
||||
def _ok(operation: str, **fields: Any) -> DuplexControlResultMessage:
|
||||
return DuplexControlResultMessage(control_id="c", operation=operation, session_id="sid", ok=True, **fields)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_session_runs_the_correlated_rpc_with_the_typed_config() -> None:
|
||||
result = _ok("open", capabilities=DuplexCapabilities(), public_session={"id": "sid"}, lease_generation=0)
|
||||
engine = _engine(result)
|
||||
config = DuplexSessionConfig(model="m")
|
||||
|
||||
returned = await engine.open_session_async("sid", config, timeout=3.0)
|
||||
|
||||
assert returned is result
|
||||
((key, message, timeout),) = engine.rpc_client.calls
|
||||
assert key == ("duplex", message.control_id)
|
||||
assert isinstance(message, OpenDuplexSessionMessage)
|
||||
assert message.session_id == "sid" and message.session_config is config
|
||||
assert timeout == 3.0
|
||||
# A momentarily full request queue is backpressure, not a failed open: the
|
||||
# control RPC blocks rather than raising an untyped queue.Full at the caller.
|
||||
assert engine.rpc_client.block_on_submit_flags == [True]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_resume_and_touch_build_their_messages_without_incarnation() -> None:
|
||||
engine = _engine(_ok("close"))
|
||||
await engine.close_session_async("sid", reason="bye")
|
||||
await engine.resume_session_async("sid", expected_lease_generation=4)
|
||||
await engine.touch_session_async("sid", activity="detach")
|
||||
|
||||
close, resume, touch = (call[1] for call in engine.rpc_client.calls)
|
||||
assert isinstance(close, CloseDuplexSessionMessage) and close.reason == "bye"
|
||||
assert isinstance(resume, ResumeDuplexSessionMessage) and resume.expected_lease_generation == 4
|
||||
assert isinstance(touch, TouchDuplexSessionMessage) and touch.activity == "detach"
|
||||
for message in (close, resume, touch):
|
||||
assert message.session_id == "sid"
|
||||
assert not hasattr(message, "incarnation")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_control_result_maps_to_a_typed_session_error() -> None:
|
||||
engine = _engine(
|
||||
DuplexControlResultMessage(
|
||||
control_id="c",
|
||||
operation="open",
|
||||
session_id="sid",
|
||||
ok=False,
|
||||
error_code="resource_exhausted",
|
||||
error_message="duplex_session_capacity_exhausted: limit=1",
|
||||
error_retryable=True,
|
||||
)
|
||||
)
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await engine.open_session_async("sid", DuplexSessionConfig())
|
||||
assert excinfo.value.code == "resource_exhausted"
|
||||
assert excinfo.value.retryable is True
|
||||
assert excinfo.value.session_id == "sid"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_error_and_timeout_map_to_session_errors() -> None:
|
||||
# The correlated RPC client raises RuntimeError for a closed router or a
|
||||
# terminal orchestrator error; both become a typed engine_error.
|
||||
engine = _engine(RuntimeError("orchestrator exploded"))
|
||||
with pytest.raises(DuplexSessionError, match="orchestrator exploded") as excinfo:
|
||||
await engine.close_session_async("sid")
|
||||
assert excinfo.value.code == "engine_error"
|
||||
|
||||
engine = _engine(TimeoutError("duplex open timed out for session sid"))
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await engine.open_session_async("sid", DuplexSessionConfig())
|
||||
assert excinfo.value.code == "timeout" and excinfo.value.retryable is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_submit_command_is_one_way_on_the_request_queue() -> None:
|
||||
engine = _engine(None)
|
||||
command = Heartbeat(event_id="evt-1")
|
||||
|
||||
await engine.submit_command_async("sid", command)
|
||||
|
||||
(message,) = engine.request_queue.sync_q.items
|
||||
assert isinstance(message, DuplexSessionCommandMessage)
|
||||
assert message.session_id == "sid" and message.command is command
|
||||
assert engine.rpc_client.calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_submit_command_rejects_dead_engine_and_full_queue() -> None:
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await _engine(None, alive=False).submit_command_async("sid", Heartbeat())
|
||||
assert excinfo.value.code == "engine_dead"
|
||||
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await _engine(None, full=True).submit_command_async("sid", Heartbeat())
|
||||
assert excinfo.value.code == "engine_backpressure" and excinfo.value.retryable is True
|
||||
|
||||
|
||||
def test_duplex_capabilities_come_from_the_loaded_plugin() -> None:
|
||||
engine = object.__new__(DuplexOmniEngine)
|
||||
engine.plugin = None
|
||||
with pytest.raises(RuntimeError, match="not loaded"):
|
||||
engine.duplex_capabilities
|
||||
capabilities = DuplexCapabilities(supports_barge_in=False)
|
||||
engine.plugin = SimpleNamespace(capabilities=lambda *, max_sessions: capabilities)
|
||||
engine.duplex_session_config = SimpleNamespace(max_sessions=3)
|
||||
assert engine.duplex_capabilities is capabilities
|
||||
|
||||
|
||||
def test_validate_deployment_requires_a_plugin_and_duplex_session_mode() -> None:
|
||||
engine = object.__new__(DuplexOmniEngine)
|
||||
engine.model = "plain-model"
|
||||
engine.pipeline_config = SimpleNamespace(duplex_plugin=None)
|
||||
engine.deploy_config = SimpleNamespace(session_mode="duplex")
|
||||
with pytest.raises(ValueError, match="not a duplex model"):
|
||||
engine._validate_deployment()
|
||||
|
||||
engine.pipeline_config = SimpleNamespace(duplex_plugin="pkg.mod.Plugin")
|
||||
engine.deploy_config = SimpleNamespace(session_mode="turn")
|
||||
with pytest.raises(ValueError, match="session_mode: duplex"):
|
||||
engine._validate_deployment()
|
||||
|
||||
engine.deploy_config = None
|
||||
with pytest.raises(ValueError, match="none resolved"):
|
||||
engine._validate_deployment()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_wrappers_run_the_blocking_calls_off_the_event_loop() -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
engine = _engine(_ok("touch"))
|
||||
seen: list[bool] = []
|
||||
original = engine.rpc_client.execute
|
||||
|
||||
def execute(*args, **kwargs):
|
||||
# Called from the default executor thread, never on the event loop thread.
|
||||
try:
|
||||
seen.append(asyncio.get_running_loop() is loop)
|
||||
except RuntimeError:
|
||||
seen.append(False)
|
||||
return original(*args, **kwargs)
|
||||
|
||||
engine.rpc_client.execute = execute
|
||||
await engine.touch_session_async("sid", activity="heartbeat")
|
||||
assert seen == [False]
|
||||
@@ -0,0 +1,517 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``DuplexOrchestrator``: the template seams and the stage port around ``OrchestratorBase``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import struct
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from tests.engine.test_orchestrator import (
|
||||
FakeOutputProcessor,
|
||||
FakeRunningCounter,
|
||||
FakeStageClient,
|
||||
_build_stage_pools,
|
||||
)
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex import commands
|
||||
from vllm_omni.engine.duplex.config import DuplexSessionConfig, DuplexSessionState
|
||||
from vllm_omni.engine.duplex.contracts import DuplexFence, duplex_resource_request_id
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
CloseDuplexSessionMessage,
|
||||
DuplexControlResultMessage,
|
||||
DuplexSessionCommandMessage,
|
||||
OpenDuplexSessionMessage,
|
||||
)
|
||||
from vllm_omni.engine.duplex_orchestrator import DuplexOrchestrator, DuplexOrchestratorRequestState
|
||||
from vllm_omni.engine.messages import AbortRequestMessage, ShutdownRequestMessage
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.plugin import MiniCPMO45DuplexPlugin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
SESSION_ID = "duplex-orch"
|
||||
|
||||
|
||||
def _encode_audio(audio: object, sample_rate_hz: int, response_format: str, speed: float | None) -> str | None:
|
||||
del sample_rate_hz, response_format, speed
|
||||
samples = int(np.asarray(audio, dtype=np.float32).size) if audio is not None else 0
|
||||
return f"wav-{samples}" if samples > 0 else None
|
||||
|
||||
|
||||
def _stage_configs(count: int) -> list[object]:
|
||||
return [SimpleNamespace(model_config=SimpleNamespace(max_model_len=4096)) for _ in range(count)]
|
||||
|
||||
|
||||
def _build(
|
||||
*,
|
||||
stages: int = 1,
|
||||
running_counter: FakeRunningCounter | None = None,
|
||||
runtime_config: DuplexSessionRuntimeConfig | None = None,
|
||||
) -> tuple[DuplexOrchestrator, list[FakeStageClient], asyncio.Queue, asyncio.Queue]:
|
||||
clients = [FakeStageClient(stage_type="llm", final_output=index == stages - 1) for index in range(stages)]
|
||||
pools = _build_stage_pools(
|
||||
[[client] for client in clients],
|
||||
output_processors=[FakeOutputProcessor() for _ in clients],
|
||||
stage_vllm_configs=_stage_configs(stages),
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
output_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = DuplexOrchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=output_q,
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=pools,
|
||||
running_counter=running_counter,
|
||||
plugin=MiniCPMO45DuplexPlugin(_encode_audio),
|
||||
duplex_session_config=runtime_config or DuplexSessionRuntimeConfig(reaper_interval_s=0.01),
|
||||
model_config=None,
|
||||
)
|
||||
return orchestrator, clients, rpc_q, output_q
|
||||
|
||||
|
||||
def _open_message(extra_body: dict[str, object] | None = None) -> OpenDuplexSessionMessage:
|
||||
return OpenDuplexSessionMessage(
|
||||
control_id=f"open-{SESSION_ID}",
|
||||
session_id=SESSION_ID,
|
||||
session_config=DuplexSessionConfig(
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
modalities=["text"],
|
||||
instructions="public instructions",
|
||||
extra_body={"auto_response": True, **(extra_body or {})},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _open(orchestrator: DuplexOrchestrator, rpc_q: asyncio.Queue, **kwargs: Any) -> DuplexControlResultMessage:
|
||||
await orchestrator._dispatch_message(_open_message(**kwargs))
|
||||
result = await asyncio.wait_for(rpc_q.get(), timeout=2.0)
|
||||
assert isinstance(result, DuplexControlResultMessage)
|
||||
return result
|
||||
|
||||
|
||||
async def _close(orchestrator: DuplexOrchestrator, rpc_q: asyncio.Queue) -> DuplexControlResultMessage:
|
||||
await orchestrator._dispatch_message(
|
||||
CloseDuplexSessionMessage(control_id=f"close-{SESSION_ID}", session_id=SESSION_ID, reason="test")
|
||||
)
|
||||
result = await asyncio.wait_for(rpc_q.get(), timeout=2.0)
|
||||
await _settle(orchestrator)
|
||||
return result
|
||||
|
||||
|
||||
def _stage0_request_id(epoch: int = 0) -> str:
|
||||
return duplex_resource_request_id(DuplexFence(SESSION_ID, epoch=epoch), "stage0")
|
||||
|
||||
|
||||
def _append_audio(samples: int = 16000) -> commands.AppendAudio:
|
||||
return commands.AppendAudio(
|
||||
audio=struct.pack(f"<{samples}f", *([0.05] * samples)),
|
||||
format="pcm_f32le",
|
||||
sample_rate_hz=16000,
|
||||
is_speech=True,
|
||||
)
|
||||
|
||||
|
||||
async def _submit(orchestrator: DuplexOrchestrator, command: commands.DuplexCommand) -> None:
|
||||
assert await orchestrator._dispatch_message(DuplexSessionCommandMessage(session_id=SESSION_ID, command=command))
|
||||
await _settle(orchestrator)
|
||||
|
||||
|
||||
async def _settle(orchestrator: DuplexOrchestrator, *, timeout_s: float = 3.0) -> None:
|
||||
"""Let the session runner drain its mailbox and append tasks."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout_s
|
||||
quiet = 0
|
||||
while loop.time() < deadline:
|
||||
runner = orchestrator.session_manager.runners.get(SESSION_ID)
|
||||
busy = runner is not None and (
|
||||
not runner._mailbox.empty()
|
||||
or any(not task.done() for task in runner.tasks.append_tasks)
|
||||
or any(not task.done() for task in runner._background_tasks)
|
||||
)
|
||||
busy = busy or any(not task.done() for task in orchestrator.session_manager._dispatched_control_tasks)
|
||||
quiet = 0 if busy else quiet + 1
|
||||
if quiet >= 5:
|
||||
return
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
def _tts_output(request_id: str, *, samples: int = 24000, text: str = "hello") -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
request_id=request_id,
|
||||
finished=False,
|
||||
outputs=[SimpleNamespace(text=text, token_ids=[], multimodal_output={})],
|
||||
multimodal_output={
|
||||
"audio": np.zeros(samples, dtype=np.float32),
|
||||
"sr": 24000,
|
||||
"meta.duplex_turn_id": np.array([0], dtype=np.int32),
|
||||
"meta.duplex_epoch": np.array([0], dtype=np.int32),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Message dispatch and session lifecycle #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_turn_based_orchestrator_has_no_session_manager() -> None:
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=asyncio.Queue(),
|
||||
stage_pools=[],
|
||||
)
|
||||
assert not hasattr(orchestrator, "session_manager")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_messages_are_dispatched_and_generic_ones_are_not() -> None:
|
||||
orchestrator, _, rpc_q, _ = _build()
|
||||
assert await orchestrator._dispatch_message(AbortRequestMessage(request_ids=["x"])) is False
|
||||
assert await orchestrator._dispatch_message(ShutdownRequestMessage()) is False
|
||||
assert await orchestrator._dispatch_message(_open_message()) is True
|
||||
assert (await asyncio.wait_for(rpc_q.get(), timeout=2.0)).ok is True
|
||||
await orchestrator.session_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_preregisters_the_stage0_request_and_close_releases_it() -> None:
|
||||
counter = FakeRunningCounter()
|
||||
orchestrator, clients, rpc_q, _ = _build(running_counter=counter)
|
||||
result = await _open(orchestrator, rpc_q)
|
||||
assert result.ok is True
|
||||
assert result.public_session["id"] == SESSION_ID
|
||||
assert result.capabilities is not None and result.capabilities.supports_input_append
|
||||
|
||||
request_id = _stage0_request_id()
|
||||
request_state = orchestrator.request_states[request_id]
|
||||
assert isinstance(request_state, DuplexOrchestratorRequestState)
|
||||
assert request_state.session_owned is True
|
||||
assert request_state.session_id == SESSION_ID
|
||||
assert request_state.fence == DuplexFence(SESSION_ID)
|
||||
assert request_state.streaming.enabled is True
|
||||
# Preregistration reserves the id; nothing is running until an append submits.
|
||||
assert counter.value == 0
|
||||
assert clients[0].add_request_calls == []
|
||||
assert orchestrator.session_manager.runner_for_request_id(request_id) is not None
|
||||
|
||||
closed = await _close(orchestrator, rpc_q)
|
||||
assert closed.ok is True
|
||||
assert request_id not in orchestrator.request_states
|
||||
assert orchestrator.session_manager.runner_for_request_id(request_id) is None
|
||||
assert counter.value == 0
|
||||
assert orchestrator.session_manager.active_count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_failure_rolls_back_session_and_reserved_request() -> None:
|
||||
orchestrator, _, rpc_q, _ = _build()
|
||||
# Server-owned runtime keys may not come from the client.
|
||||
result = await _open(orchestrator, rpc_q, extra_body={"duplex_stage_max_tokens": {"0": 99}})
|
||||
|
||||
assert result.ok is False
|
||||
assert result.error_code == "invalid_duplex_runtime_config"
|
||||
assert orchestrator.request_states == {}
|
||||
assert orchestrator.session_manager.active_count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_fails_cleanly_when_a_stage_has_no_live_replica() -> None:
|
||||
orchestrator, _, rpc_q, _ = _build(stages=2)
|
||||
orchestrator.stage_pools[1].evict_replica(0)
|
||||
|
||||
result = await _open(orchestrator, rpc_q)
|
||||
|
||||
assert result.ok is False
|
||||
assert "stage 1 has no live replica" in (result.error_message or "")
|
||||
assert orchestrator.request_states == {}
|
||||
assert orchestrator.session_manager.active_count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_state_keeps_public_and_runtime_config_separate() -> None:
|
||||
orchestrator, _, rpc_q, _ = _build()
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_state = orchestrator.request_states[_stage0_request_id()]
|
||||
bridge = request_state.streaming.bridge_states["duplex"]
|
||||
|
||||
assert bridge["session_id"] == SESSION_ID
|
||||
assert bridge["fence"] == DuplexFence(SESSION_ID)
|
||||
assert (bridge["epoch"], bridge["turn_id"], bridge["model_turn_id"]) == (0, 0, 0)
|
||||
assert "incarnation" not in bridge
|
||||
assert bridge["session_config"]["instructions"] == "public instructions"
|
||||
assert "duplex_stage_sampling_params" not in bridge["session_config"]
|
||||
assert bridge["runtime_config"]["instructions"] == "public instructions"
|
||||
assert bridge["runtime_config"]["duplex_stage_max_tokens"] == {"0": 20, "1": 8192}
|
||||
# The plugin's sampling policy is applied to the preregistered request.
|
||||
assert request_state.sampling_params_list[0].max_tokens == 20
|
||||
await orchestrator.session_manager.shutdown()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Stage port #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_append_submits_the_resumable_stage0_request_and_counts_it_running() -> None:
|
||||
counter = FakeRunningCounter()
|
||||
orchestrator, clients, rpc_q, _ = _build(running_counter=counter)
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_id = _stage0_request_id()
|
||||
|
||||
await _submit(orchestrator, _append_audio())
|
||||
assert len(clients[0].add_request_calls) == 1
|
||||
submitted = clients[0].add_request_calls[0][0]
|
||||
assert submitted.request_id == request_id
|
||||
assert submitted.resumable is True
|
||||
assert submitted.sampling_params.max_tokens == 20
|
||||
request_state = orchestrator.request_states[request_id]
|
||||
assert request_state.stage_fences[0] == DuplexFence(SESSION_ID)
|
||||
assert 0 in request_state.stage_submit_ts
|
||||
assert counter.value == 1
|
||||
session = orchestrator.session_manager.get(SESSION_ID)
|
||||
assert session is not None and session.stage_request_submitted(0, request_id)
|
||||
|
||||
# Later units update the same resumable request; the counter stays at one.
|
||||
await _submit(orchestrator, _append_audio())
|
||||
assert len(clients[0].add_request_calls) == 2
|
||||
assert clients[0].add_request_calls[1][0].request_id == request_id
|
||||
assert counter.value == 1
|
||||
|
||||
await _close(orchestrator, rpc_q)
|
||||
assert clients[0].abort_calls == [[request_id]]
|
||||
assert counter.value == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_update_refreshes_the_next_append_sampling_params() -> None:
|
||||
orchestrator, clients, rpc_q, _ = _build()
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_state = orchestrator.request_states[_stage0_request_id()]
|
||||
assert request_state.sampling_params_list[0].max_tokens == 20
|
||||
|
||||
await _submit(orchestrator, commands.UpdateSession(patch={"max_output_tokens": 7}))
|
||||
await _submit(orchestrator, _append_audio())
|
||||
|
||||
assert request_state.sampling_params_list[0].max_tokens == 7
|
||||
assert clients[0].add_request_calls[-1][0].sampling_params.max_tokens == 7
|
||||
await orchestrator.session_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forwarded_stage_requests_are_bound_and_barge_in_aborts_them() -> None:
|
||||
orchestrator, clients, rpc_q, _ = _build(stages=2)
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_id = _stage0_request_id()
|
||||
await _submit(orchestrator, _append_audio())
|
||||
request_state = orchestrator.request_states[request_id]
|
||||
session = orchestrator.session_manager.get(SESSION_ID)
|
||||
assert session is not None
|
||||
|
||||
# The base forwards Stage0 text to the TTS stage and reports the submission.
|
||||
forwarded = SimpleNamespace(request_id=request_id, prompt_token_ids=[1, 2], resumable=True)
|
||||
replica_id = await orchestrator.stage_pools[1].submit_initial(request_id, request_state, forwarded)
|
||||
orchestrator._on_stage_submitted(1, request_id, replica_id, request_state)
|
||||
assert request_state.stage_fences[1] == DuplexFence(SESSION_ID)
|
||||
assert session.stage_request_submitted(1, request_id)
|
||||
|
||||
await _submit(orchestrator, commands.BargeIn())
|
||||
assert session.epoch == 1
|
||||
assert clients[0].abort_calls == [[request_id]]
|
||||
assert clients[1].abort_calls == [[request_id]]
|
||||
assert request_id not in orchestrator.request_states
|
||||
assert session.resource_request_ids() == []
|
||||
await orchestrator.session_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_owned_outputs_reach_the_runner_and_never_the_client_queue() -> None:
|
||||
orchestrator, _, rpc_q, output_q = _build(stages=2)
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_id = _stage0_request_id()
|
||||
await _submit(orchestrator, _append_audio())
|
||||
request_state = orchestrator.request_states[request_id]
|
||||
while not output_q.empty():
|
||||
output_q.get_nowait()
|
||||
|
||||
consumed = await orchestrator._intercept_stage_output(1, 0, _tts_output(request_id), request_state, None, None)
|
||||
await _settle(orchestrator)
|
||||
|
||||
assert consumed is True
|
||||
session = orchestrator.session_manager.get(SESSION_ID)
|
||||
assert session is not None and session.active_response_id is not None
|
||||
types = [message.event.type for message in [output_q.get_nowait() for _ in range(output_q.qsize())]]
|
||||
assert "response.created" in types and "response.output_audio.delta" in types
|
||||
|
||||
orphan = DuplexOrchestratorRequestState(
|
||||
request_id="duplex-s.b3RoZXI.e.0.r.stage0",
|
||||
prompt=None,
|
||||
sampling_params_list=[],
|
||||
final_stage_id=1,
|
||||
session_owned=True,
|
||||
session_id="other",
|
||||
fence=DuplexFence("other"),
|
||||
)
|
||||
assert await orchestrator._intercept_stage_output(1, 0, _tts_output(orphan.request_id), orphan, None, None)
|
||||
await orchestrator.session_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forward_failure_closes_the_owning_session() -> None:
|
||||
orchestrator, clients, rpc_q, output_q = _build(stages=2)
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_id = _stage0_request_id()
|
||||
await _submit(orchestrator, _append_audio())
|
||||
request_state = orchestrator.request_states[request_id]
|
||||
session = orchestrator.session_manager.get(SESSION_ID)
|
||||
assert session is not None
|
||||
|
||||
absorbed = await orchestrator._handle_forward_failure(request_id, 1, request_state, ValueError("bad thinker"))
|
||||
await _settle(orchestrator)
|
||||
|
||||
assert absorbed is True
|
||||
assert request_id not in orchestrator.request_states
|
||||
assert clients[0].abort_calls == [[request_id]]
|
||||
assert SESSION_ID not in orchestrator.session_manager.runners
|
||||
assert session.state == DuplexSessionState.CLOSED
|
||||
assert orchestrator.session_manager.active_count() == 0
|
||||
types = [message.event.type for message in [output_q.get_nowait() for _ in range(output_q.qsize())]]
|
||||
# The stage failure is reported before the session expires, never after.
|
||||
assert types.index("error") < types.index("session.expired")
|
||||
assert types[-1] == "session.expired"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_cleanup_failure_is_deferred_and_retried_by_the_reaper(monkeypatch) -> None:
|
||||
orchestrator, clients, rpc_q, _ = _build()
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_id = _stage0_request_id()
|
||||
await _submit(orchestrator, _append_audio())
|
||||
manager = orchestrator.session_manager
|
||||
|
||||
async def failing_abort(request_ids: list[str]) -> None:
|
||||
raise RuntimeError("stage abort failed")
|
||||
|
||||
monkeypatch.setattr(clients[0], "abort_requests_async", failing_abort)
|
||||
with pytest.raises(RuntimeError, match="stage abort failed"):
|
||||
await orchestrator._cleanup_request_ids([request_id], abort=True, release_owners=True)
|
||||
await _settle(orchestrator)
|
||||
|
||||
assert SESSION_ID not in manager.runners
|
||||
assert SESSION_ID in manager._closing # admission slot retained
|
||||
assert manager._pending_request_cleanups and not manager._request_cleanups_in_progress
|
||||
assert manager.active_count() == 0 and manager._admission_count() == 1
|
||||
|
||||
monkeypatch.undo()
|
||||
assert await manager.reap_expired() >= 1
|
||||
assert manager._pending_request_cleanups == {}
|
||||
assert manager._admission_count() == 0
|
||||
assert request_id not in orchestrator.request_states
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reaper_runs_as_a_background_task_and_survives_one_failure(monkeypatch) -> None:
|
||||
orchestrator, _, _, _ = _build(runtime_config=DuplexSessionRuntimeConfig(reaper_interval_s=0.01))
|
||||
calls = {"count": 0}
|
||||
|
||||
async def reap_expired(now: float | None = None) -> int:
|
||||
calls["count"] += 1
|
||||
if calls["count"] == 1:
|
||||
raise RuntimeError("transient cleanup failure")
|
||||
return 0
|
||||
|
||||
monkeypatch.setattr(orchestrator.session_manager, "reap_expired", reap_expired)
|
||||
tasks = orchestrator._background_tasks()
|
||||
assert len(tasks) == 1
|
||||
task = asyncio.create_task(tasks[0])
|
||||
await asyncio.sleep(0.05)
|
||||
orchestrator._shutdown_event.set()
|
||||
await task
|
||||
assert calls["count"] >= 2
|
||||
await orchestrator._shutdown_extensions()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The hierarchy the two surfaces depend on #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_the_duplex_stack_extends_the_turn_based_one() -> None:
|
||||
"""One engine serves a session and an ordinary request, so duplex must extend turn-based.
|
||||
|
||||
Pinned because it is the kind of relationship a later tidy-up turns back
|
||||
into siblings over a shared base. If it does, a duplex server silently
|
||||
stops serving /v1/chat/completions: the chat service needs a real
|
||||
EngineClient with generate(), which only the turn-based side provides.
|
||||
"""
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.duplex_omni_engine import DuplexOmniEngine
|
||||
from vllm_omni.engine.orchestrator import Orchestrator
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
|
||||
assert issubclass(DuplexOrchestrator, Orchestrator)
|
||||
assert issubclass(DuplexOmniEngine, AsyncOmniEngine)
|
||||
assert issubclass(DuplexOmni, AsyncOmni)
|
||||
# What the chat route actually needs off the engine client.
|
||||
assert hasattr(DuplexOmni, "generate")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unrecognised_message_reaches_the_turn_based_handler() -> None:
|
||||
"""The duplex orchestrator consumes session messages and passes the rest down.
|
||||
|
||||
Without the fall-through, an ordinary add_request on a duplex engine is
|
||||
silently dropped and the chat request that sent it hangs forever.
|
||||
"""
|
||||
orchestrator, _clients, _rpc_q, _output_q = _build()
|
||||
handled: list[str] = []
|
||||
|
||||
async def _turn_based(msg: object) -> None:
|
||||
handled.append(getattr(msg, "type", ""))
|
||||
|
||||
orchestrator._handle_add_request = _turn_based # type: ignore[method-assign]
|
||||
msg = SimpleNamespace(type="add_request")
|
||||
|
||||
assert await orchestrator._dispatch_message(msg) is True
|
||||
assert handled == ["add_request"], "an add_request must reach Orchestrator, not be dropped"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_dead_replica_closes_the_sessions_it_was_serving() -> None:
|
||||
"""A session outlives its stage request, so replica death has to release the owner.
|
||||
|
||||
``_handle_dead_replica`` cleaned the stage request but not the runner that
|
||||
owned it: the session stayed alive holding an admission slot, and the client
|
||||
was told nothing — the request-scoped ``ErrorMessage`` it emits has no
|
||||
frontend ``request_states`` entry to land in for a session-owned request, so
|
||||
the frontend drops it. Heartbeats would then keep an unusable session
|
||||
occupying capacity.
|
||||
"""
|
||||
orchestrator, clients, rpc_q, output_q = _build()
|
||||
await _open(orchestrator, rpc_q)
|
||||
request_id = _stage0_request_id()
|
||||
await _submit(orchestrator, _append_audio())
|
||||
session = orchestrator.session_manager.get(SESSION_ID)
|
||||
assert session is not None
|
||||
assert orchestrator.session_manager.active_count() == 1
|
||||
|
||||
await orchestrator._handle_dead_replica(0, 0, EngineDeadError("stage 0 replica 0 died"))
|
||||
await _settle(orchestrator)
|
||||
|
||||
assert request_id not in orchestrator.request_states
|
||||
assert SESSION_ID not in orchestrator.session_manager.runners
|
||||
assert session.state == DuplexSessionState.CLOSED
|
||||
assert orchestrator.session_manager.active_count() == 0, "the admission slot must come back"
|
||||
messages = [output_q.get_nowait() for _ in range(output_q.qsize())]
|
||||
types = [getattr(getattr(m, "event", None), "type", type(m).__name__) for m in messages]
|
||||
assert types[-1] in {"session.expired", "session.closed"}, f"the client needs a terminal event, got {types}"
|
||||
@@ -21,20 +21,6 @@ from vllm.v1.engine import EngineCoreOutput, EngineCoreOutputs, FinishReason
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
from vllm_omni.engine.duplex.control_plane import DuplexControlPlane
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
AppendDuplexInputMessage,
|
||||
CloseDuplexSessionMessage,
|
||||
DuplexFence,
|
||||
OpenDuplexSessionMessage,
|
||||
SignalDuplexTurnMessage,
|
||||
)
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexInputMode,
|
||||
DuplexRuntimeCapabilities,
|
||||
DuplexSessionRuntimeState,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.engine.messages import (
|
||||
AbortRequestMessage,
|
||||
AbortResultMessage,
|
||||
@@ -54,7 +40,6 @@ from vllm_omni.engine.orchestrator import (
|
||||
)
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.runtime import MiniCPMO45DuplexRuntimeExtension
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -1118,778 +1103,6 @@ async def test_handle_abort_emits_error_result_on_failure() -> None:
|
||||
assert "cleanup exploded" in (result.error or "")
|
||||
|
||||
|
||||
def _duplex_open_message(
|
||||
session_id: str,
|
||||
*,
|
||||
incarnation: int = 0,
|
||||
session_config: dict[str, object] | None = None,
|
||||
runtime_config: dict[str, object] | None = None,
|
||||
) -> OpenDuplexSessionMessage:
|
||||
return OpenDuplexSessionMessage(
|
||||
control_id=f"open-{session_id}",
|
||||
fence=DuplexFence(session_id, incarnation=incarnation),
|
||||
session_id=session_id,
|
||||
capabilities={
|
||||
"input_modes": [DuplexInputMode.APPEND_AUDIO_CHUNK.value],
|
||||
"implementation_level": "model_native_duplex",
|
||||
},
|
||||
session_config=session_config or {},
|
||||
runtime_config=runtime_config or {},
|
||||
)
|
||||
|
||||
|
||||
async def _handle_duplex(orchestrator: Orchestrator, message: object) -> None:
|
||||
await orchestrator._require_duplex_control_plane().handle(message)
|
||||
|
||||
|
||||
def _duplex_request_state(
|
||||
orchestrator: Orchestrator,
|
||||
session: DuplexSessionRuntimeState,
|
||||
*,
|
||||
stage_id: int,
|
||||
) -> OrchestratorRequestState | None:
|
||||
context = orchestrator._require_duplex_control_plane().ensure_stage_request(
|
||||
session,
|
||||
stage_id=stage_id,
|
||||
)
|
||||
if context is None:
|
||||
return None
|
||||
return orchestrator.request_states.get(context.request_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_control_plane_keeps_public_and_runtime_config_separate() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
open_message = _duplex_open_message(
|
||||
"sid-config-channels",
|
||||
incarnation=3,
|
||||
session_config={
|
||||
"instructions": "public instructions",
|
||||
"extra_body": {"duplex_stage_max_tokens": {"0": 99}},
|
||||
},
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 3},
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151705]}},
|
||||
},
|
||||
)
|
||||
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
session = orchestrator.duplex_sessions.require(open_message.session_id)
|
||||
request_state = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert request_state.sampling_params_list[0].max_tokens == 3
|
||||
assert request_state.sampling_params_list[0].stop_token_ids == [151705]
|
||||
bridge = request_state.streaming.bridge_states["duplex"]
|
||||
assert bridge["incarnation"] == open_message.fence.incarnation
|
||||
assert bridge["session_config"] == open_message.session_config
|
||||
assert bridge["runtime_config"] == open_message.runtime_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_control_plane_preserves_turn_commit_without_model_extension() -> None:
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=[],
|
||||
duplex_runtime_extension=None,
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
assert isinstance(orchestrator.duplex_control_plane, DuplexControlPlane)
|
||||
|
||||
message = OpenDuplexSessionMessage(
|
||||
control_id="open-turn-commit",
|
||||
fence=DuplexFence("sid-turn-commit"),
|
||||
session_id="sid-turn-commit",
|
||||
capabilities={"input_modes": [DuplexInputMode.TURN_COMMIT_ONLY.value]},
|
||||
session_config={},
|
||||
)
|
||||
await _handle_duplex(orchestrator, message)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
assert result.stage_results[0]["result"]["scheduler_request_context"] is False
|
||||
|
||||
|
||||
def test_ordinary_orchestrator_bypasses_duplex_control_plane() -> None:
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=asyncio.Queue(),
|
||||
stage_pools=[],
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
)
|
||||
|
||||
assert orchestrator.duplex_control_plane is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_close_cleans_preregistered_request_without_append() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
running_counter = FakeRunningCounter()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
running_counter=running_counter,
|
||||
)
|
||||
open_message = _duplex_open_message("sid-preregister-close")
|
||||
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
open_result = rpc_q.get_nowait()
|
||||
request_id = open_result.stage_results[0]["result"]["request_id"]
|
||||
assert request_id in orchestrator.request_states
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
CloseDuplexSessionMessage(
|
||||
control_id="close-preregistered",
|
||||
fence=open_message.fence,
|
||||
session_id=open_message.session_id,
|
||||
),
|
||||
)
|
||||
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert request_id not in orchestrator.request_states
|
||||
assert orchestrator.duplex_sessions.get(open_message.session_id) is None
|
||||
assert running_counter.value == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_open_failure_rolls_back_session_and_reserved_request() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
open_message = _duplex_open_message(
|
||||
"sid-open-rollback",
|
||||
runtime_config={
|
||||
"duplex_stage_sampling_params": {
|
||||
"0": {"stop_token_ids": 123},
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
|
||||
assert rpc_q.get_nowait().ok is False
|
||||
assert orchestrator.duplex_sessions.get(open_message.session_id) is None
|
||||
assert orchestrator.request_states == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_running_counter_tracks_only_submitted_request() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
running_counter = FakeRunningCounter()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
running_counter=running_counter,
|
||||
)
|
||||
open_message = _duplex_open_message("sid-duplex-counter")
|
||||
await _handle_duplex(orchestrator, open_message)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert running_counter.value == 0
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-duplex-counter",
|
||||
fence=open_message.fence,
|
||||
session_id=open_message.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert running_counter.value == 1
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
CloseDuplexSessionMessage(
|
||||
control_id="close-duplex-counter",
|
||||
fence=open_message.fence,
|
||||
session_id=open_message.session_id,
|
||||
),
|
||||
)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert running_counter.value == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_failed_append_does_not_advance_sequence_or_fence() -> None:
|
||||
class FailingPlanExtension(MiniCPMO45DuplexRuntimeExtension):
|
||||
def plan_append(self, **kwargs):
|
||||
raise RuntimeError("planned append failure")
|
||||
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=FailingPlanExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-append-rollback", turn_id=1)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
DuplexFence("sid-append-rollback"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-rollback",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert session.fence == DuplexFence("sid-append-rollback")
|
||||
assert session.input_seq == 0
|
||||
assert session.input_turn_seq == 0
|
||||
assert stage0.add_request_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_duplicate_append_operation_is_submitted_once() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-idempotent-append")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
for control_id in ("append-first", "append-timeout-retry"):
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id=control_id,
|
||||
operation_id="physical-input-1",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
first = rpc_q.get_nowait()
|
||||
retry = rpc_q.get_nowait()
|
||||
assert first.ok is True
|
||||
assert retry.ok is True
|
||||
assert first.stage_results == retry.stage_results
|
||||
assert len(stage0.add_request_calls) == 1
|
||||
assert session.input_seq == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_barge_in_aborts_bound_stage_requests_before_releasing_fence() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0], [stage1]],
|
||||
output_processors=[FakeOutputProcessor(), FakeOutputProcessor()],
|
||||
stage_vllm_configs=[
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
DuplexFence("sid-stage-signal"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(0, "req-stage0", fence=session.fence)
|
||||
session.bind_stage_request(1, "req-stage1", fence=session.fence)
|
||||
assert stage_pools[0].select_replica_id("req-stage0") == 0
|
||||
assert stage_pools[1].select_replica_id("req-stage1") == 0
|
||||
session.append_input(
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK,
|
||||
fence=session.fence,
|
||||
)
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="ctrl-signal",
|
||||
fence=session.fence,
|
||||
next_fence=DuplexFence("sid-stage-signal", epoch=1),
|
||||
session_id="sid-stage-signal",
|
||||
event="barge_in",
|
||||
),
|
||||
)
|
||||
|
||||
assert stage0.abort_calls == [["req-stage0"]]
|
||||
assert stage1.abort_calls == [["req-stage1"]]
|
||||
assert stage0.collective_rpc_calls == []
|
||||
assert stage1.collective_rpc_calls == []
|
||||
assert session.stage_bindings == {}
|
||||
assert session.fence == DuplexFence("sid-stage-signal", epoch=1)
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_late_barge_in_releases_only_cancelled_fence_bindings() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=False)
|
||||
stage1 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0], [stage1]],
|
||||
output_processors=[FakeOutputProcessor(), FakeOutputProcessor()],
|
||||
stage_vllm_configs=[
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
cancelled_fence = DuplexFence("sid-late-stage-signal")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
cancelled_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
session.bind_stage_request(0, "req-stage0", fence=cancelled_fence)
|
||||
session.bind_stage_request(1, "req-stage1", fence=cancelled_fence)
|
||||
assert stage_pools[0].select_replica_id("req-stage0") == 0
|
||||
assert stage_pools[1].select_replica_id("req-stage1") == 0
|
||||
current_fence = DuplexFence("sid-late-stage-signal", epoch=1)
|
||||
session.accept_fence(current_fence)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="ctrl-late-signal",
|
||||
fence=cancelled_fence,
|
||||
next_fence=current_fence,
|
||||
session_id="sid-late-stage-signal",
|
||||
event="barge_in",
|
||||
),
|
||||
)
|
||||
|
||||
assert stage0.abort_calls == [["req-stage0"]]
|
||||
assert stage1.abort_calls == [["req-stage1"]]
|
||||
assert session.stage_bindings == {}
|
||||
assert session.fence == current_fence
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_cancel_without_next_fence_is_rejected_without_releasing_bindings() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-cancel-contract")
|
||||
session = orchestrator.duplex_sessions.open_session(fence)
|
||||
session.bind_stage_request(0, "req-live", fence=fence)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="cancel-without-next",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="input.cancel",
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert result.error_count == 1
|
||||
assert "next_fence" in result.stage_results[0]["result"]["error"]
|
||||
assert session.fence == fence
|
||||
assert session.stage_request_ids() == ["req-live"]
|
||||
assert stage0.abort_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_session_update_replaces_runtime_config() -> None:
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=[],
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-update-config")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
session_config={"temperature": 0.7},
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="update-config",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="session.update",
|
||||
session_config={"temperature": 0.0, "instructions": "updated"},
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is True
|
||||
assert session.session_config == {"temperature": 0.0, "instructions": "updated"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_session_update_refreshes_next_append_sampling_params() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-update-policy")
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 2},
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151645]}},
|
||||
},
|
||||
)
|
||||
req_state = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert req_state is not None
|
||||
assert req_state.sampling_params_list[0].max_tokens == 2
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="update-policy",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="session.update",
|
||||
runtime_config={
|
||||
"duplex_stage_max_tokens": {"0": 7},
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151705]}},
|
||||
},
|
||||
),
|
||||
)
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-updated-policy",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
assert rpc_q.get_nowait().ok is True
|
||||
assert req_state.sampling_params_list[0].max_tokens == 7
|
||||
assert req_state.sampling_params_list[0].stop_token_ids == [151705]
|
||||
submitted_request = stage0.add_request_calls[0][0]
|
||||
assert submitted_request.sampling_params.stop_token_ids == [151705]
|
||||
assert submitted_request.sampling_params.max_tokens == 7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_invalid_session_update_preserves_previous_config() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=_build_stage_pools([[stage0]]),
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-invalid-update")
|
||||
original_runtime_config = {
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": [151645]}},
|
||||
}
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
fence,
|
||||
runtime_config=original_runtime_config,
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="invalid-update",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="session.update",
|
||||
runtime_config={
|
||||
"duplex_stage_sampling_params": {"0": {"stop_token_ids": 123}},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert session.runtime_config == original_runtime_config
|
||||
assert session.config_generation == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_arbitrary_non_cancel_signal_is_rejected() -> None:
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=[],
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
fence = DuplexFence("sid-unsupported-signal")
|
||||
orchestrator.duplex_sessions.open_session(fence)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="unsupported-signal",
|
||||
fence=fence,
|
||||
session_id=fence.session_id,
|
||||
event="turn.end",
|
||||
),
|
||||
)
|
||||
|
||||
result = rpc_q.get_nowait()
|
||||
assert result.ok is False
|
||||
assert "unsupported duplex runtime signal" in result.stage_results[0]["result"]["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_cancel_rejects_late_old_append_and_accepts_next_epoch() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
rpc_q: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_q,
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
cancelled_fence = DuplexFence("sid-cancel-late-append")
|
||||
next_fence = DuplexFence("sid-cancel-late-append", epoch=1)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
cancelled_fence,
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
SignalDuplexTurnMessage(
|
||||
control_id="cancel-old-fence",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
session_id=session.session_id,
|
||||
event="input.cancel",
|
||||
),
|
||||
)
|
||||
|
||||
cancel_result = rpc_q.get_nowait()
|
||||
assert cancel_result.ok is True
|
||||
assert session.fence == next_fence
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="late-old-append",
|
||||
fence=cancelled_fence,
|
||||
session_id=session.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
stale_result = rpc_q.get_nowait()
|
||||
assert stale_result.ok is False
|
||||
assert stage0.add_request_calls == []
|
||||
assert session.stage_bindings == {}
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="next-epoch-append",
|
||||
fence=next_fence,
|
||||
session_id=session.session_id,
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
|
||||
next_result = rpc_q.get_nowait()
|
||||
assert next_result.ok is True
|
||||
assert len(stage0.add_request_calls) == 1
|
||||
assert session.stage_bindings[0].fence == next_fence
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_append_updates_bridge_turn_id_on_long_lived_stage0_request() -> None:
|
||||
stage0 = FakeStageClient(stage_type="llm", final_output=True)
|
||||
stage_pools = _build_stage_pools(
|
||||
[[stage0]],
|
||||
output_processors=[FakeOutputProcessor()],
|
||||
stage_vllm_configs=[SimpleNamespace(model_config=SimpleNamespace(max_model_len=64))],
|
||||
)
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=asyncio.Queue(),
|
||||
stage_pools=stage_pools,
|
||||
duplex_runtime_extension=MiniCPMO45DuplexRuntimeExtension(),
|
||||
enable_duplex_control=True,
|
||||
)
|
||||
session = orchestrator.duplex_sessions.open_session(
|
||||
DuplexFence("sid-bridge-turn"),
|
||||
capabilities=DuplexRuntimeCapabilities(
|
||||
input_modes={DuplexInputMode.APPEND_AUDIO_CHUNK},
|
||||
),
|
||||
session_config={
|
||||
"voice": "test",
|
||||
},
|
||||
runtime_config={
|
||||
"duplex_stage_sampling_params": {
|
||||
"0": {"stop_token_ids": [151645]},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-1",
|
||||
fence=DuplexFence("sid-bridge-turn"),
|
||||
session_id="sid-bridge-turn",
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True},
|
||||
),
|
||||
)
|
||||
req_state1 = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert req_state1 is not None
|
||||
duplex_state1 = req_state1.streaming.bridge_states["duplex"]
|
||||
assert duplex_state1["session_id"] == "sid-bridge-turn"
|
||||
assert duplex_state1["epoch"] == 0
|
||||
assert duplex_state1["turn_id"] == 0
|
||||
assert duplex_state1["session_config"] == session.session_config
|
||||
assert req_state1.sampling_params_list[0].stop_token_ids == [151645]
|
||||
submitted_request = stage0.add_request_calls[0][0]
|
||||
assert submitted_request.sampling_params.stop_token_ids == [151645]
|
||||
assert submitted_request.sampling_params.max_tokens == 1
|
||||
|
||||
await _handle_duplex(
|
||||
orchestrator,
|
||||
AppendDuplexInputMessage(
|
||||
control_id="append-2",
|
||||
fence=DuplexFence("sid-bridge-turn", turn_id=1, response_seq=1),
|
||||
session_id="sid-bridge-turn",
|
||||
mode=DuplexInputMode.APPEND_AUDIO_CHUNK.value,
|
||||
payload={"is_speech": True, "new_user_turn": True},
|
||||
),
|
||||
)
|
||||
req_state2 = _duplex_request_state(orchestrator, session, stage_id=0)
|
||||
assert req_state2 is req_state1
|
||||
duplex_state2 = req_state2.streaming.bridge_states["duplex"]
|
||||
assert duplex_state2["turn_id"] == 1
|
||||
assert duplex_state2["epoch"] == 0
|
||||
expected_request_id = duplex_resource_request_id(DuplexFence("sid-bridge-turn"), "stage0")
|
||||
assert [call[0].request_id for call in stage0.add_request_calls] == [
|
||||
expected_request_id,
|
||||
expected_request_id,
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-replica tests
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -2641,61 +1854,6 @@ async def test_stage_pool_process_llm_raw_outputs_mutates_iteration_stats() -> N
|
||||
assert iteration_stats.num_generation_tokens == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_reaper_loop_waits_between_ticks():
|
||||
class _Plane:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def reap_expired(self) -> int:
|
||||
self.calls += 1
|
||||
return 0
|
||||
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.duplex_control_plane = _Plane()
|
||||
orchestrator._duplex_reaper_interval_s = 0.01
|
||||
orchestrator._shutdown_event = asyncio.Event()
|
||||
|
||||
task = asyncio.create_task(orchestrator._duplex_reaper_loop())
|
||||
await asyncio.sleep(0.035)
|
||||
orchestrator._shutdown_event.set()
|
||||
await task
|
||||
|
||||
assert 2 <= orchestrator.duplex_control_plane.calls <= 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("first_cleanup_delay", [0.0, 0.05], ids=["immediate", "delayed"])
|
||||
async def test_duplex_reaper_loop_survives_one_cleanup_failure(first_cleanup_delay: float) -> None:
|
||||
recovered = asyncio.Event()
|
||||
|
||||
class _Plane:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
async def reap_expired(self) -> int:
|
||||
self.calls += 1
|
||||
if self.calls == 1:
|
||||
await asyncio.sleep(first_cleanup_delay)
|
||||
raise RuntimeError("transient cleanup failure")
|
||||
recovered.set()
|
||||
return 0
|
||||
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.duplex_control_plane = _Plane()
|
||||
orchestrator._duplex_reaper_interval_s = 0.01
|
||||
orchestrator._shutdown_event = asyncio.Event()
|
||||
|
||||
task = asyncio.create_task(orchestrator._duplex_reaper_loop())
|
||||
try:
|
||||
# Wait for recovery itself, including when cleanup exceeds the old 35 ms window.
|
||||
await asyncio.wait_for(recovered.wait(), timeout=5.0)
|
||||
assert orchestrator.duplex_control_plane.calls >= 2
|
||||
finally:
|
||||
orchestrator._shutdown_event.set()
|
||||
await asyncio.wait_for(task, timeout=5.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_retry_does_not_repeat_successful_stage_abort():
|
||||
class _Pool:
|
||||
@@ -2733,48 +1891,3 @@ async def test_abort_retry_does_not_repeat_successful_stage_abort():
|
||||
assert first.physical_abort_calls == 1
|
||||
assert second.physical_abort_calls == 2
|
||||
assert second.bound == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_cleanup_failure_is_deferred_to_control_plane():
|
||||
class _FailingPool:
|
||||
async def abort_requests(self, request_ids: list[str]) -> None:
|
||||
raise RuntimeError("stage abort failed")
|
||||
|
||||
def release_bindings(self, request_ids: list[str]) -> None:
|
||||
pass
|
||||
|
||||
class _Plane:
|
||||
def __init__(self) -> None:
|
||||
self.deferred: list[str] = []
|
||||
self.finalized: list[str] = []
|
||||
|
||||
def close_sessions_for_request_ids(self, request_ids: list[str], **kwargs):
|
||||
assert kwargs == {"abort": True, "cleanup_in_progress": True}
|
||||
return {"sid-cleanup": ["req-a"]}
|
||||
|
||||
def defer_request_cleanups(self, session_ids: list[str]) -> None:
|
||||
self.deferred.extend(session_ids)
|
||||
|
||||
def finalize_closed_sessions(self, session_ids: list[str]) -> None:
|
||||
self.finalized.extend(session_ids)
|
||||
|
||||
from vllm_omni.engine.cfg_companion_tracker import CfgCompanionTracker
|
||||
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
orchestrator.stage_pools = [_FailingPool()]
|
||||
orchestrator.duplex_control_plane = _Plane()
|
||||
orchestrator._pd_kv_params = {}
|
||||
orchestrator.request_states = {}
|
||||
orchestrator._running_counter = None
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
|
||||
with pytest.raises(RuntimeError, match="stage abort failed"):
|
||||
await orchestrator._cleanup_request_ids(
|
||||
["req-a"],
|
||||
abort=True,
|
||||
close_duplex_sessions=True,
|
||||
)
|
||||
|
||||
assert orchestrator.duplex_control_plane.deferred == ["sid-cleanup"]
|
||||
assert orchestrator.duplex_control_plane.finalized == []
|
||||
@@ -1054,7 +1054,7 @@ async def test_streaming_input_processor_client_error_does_not_forward_terminal_
|
||||
final_stage_id=1,
|
||||
)
|
||||
state.streaming.enabled = True
|
||||
assert state.duplex_identity is None
|
||||
assert state.session_owned is False
|
||||
orchestrator.request_states[state.request_id] = state
|
||||
try:
|
||||
await orchestrator._route_output(0, 0, _build_request_output(state.request_id), state, stage_metrics=None)
|
||||
@@ -1074,34 +1074,14 @@ async def test_streaming_input_processor_client_error_does_not_forward_terminal_
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_input_processor_failure_is_request_scoped(orchestrator_factory, monkeypatch) -> None:
|
||||
class FailingStage(FakeStageClient):
|
||||
def process_engine_inputs(self, *_args, **_kwargs):
|
||||
raise ValueError("No latent or hidden_states found in thinker output")
|
||||
|
||||
fixture = orchestrator_factory(
|
||||
[FakeStageClient(stage_type="llm"), FailingStage(stage_type="llm", final_output=True)]
|
||||
)
|
||||
state = OrchestratorRequestState(
|
||||
request_id="bad",
|
||||
prompt=SimpleNamespace(request_id="bad", prompt_token_ids=[1]),
|
||||
sampling_params_list=[_sampling_params(), _sampling_params()],
|
||||
final_stage_id=1,
|
||||
duplex_identity=SimpleNamespace(),
|
||||
)
|
||||
|
||||
async def no_cleanup(*_args, **_kwargs):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(fixture.orchestrator, "_cleanup_request_ids", no_cleanup)
|
||||
try:
|
||||
await fixture.orchestrator._forward_to_next_stage_unguarded("bad", 0, _build_request_output("raw"), state)
|
||||
error = await _wait_for_error_message(fixture, request_id="bad")
|
||||
assert error.fatal is False
|
||||
finally:
|
||||
fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
|
||||
fixture.thread.join(timeout=5)
|
||||
# ``test_duplex_input_processor_failure_is_request_scoped`` lived here upstream,
|
||||
# where one Orchestrator served both turn-based and duplex requests. Duplex
|
||||
# request ownership now belongs to DuplexOrchestrator, and the base
|
||||
# ``_handle_forward_failure`` deliberately absorbs nothing, so the equivalent
|
||||
# assertion is
|
||||
# ``tests/engine/test_duplex_orchestrator.py::test_forward_failure_closes_the_owning_session``
|
||||
# -- which also checks the stage abort, the session teardown and the
|
||||
# error-before-session.expired ordering.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -2,10 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Streaming segment-boundary state must be keyed by stage.
|
||||
|
||||
``Orchestrator._orchestration_loop`` polls every stage into the same
|
||||
``OrchestratorBase._orchestration_loop`` polls every stage into the same
|
||||
``OrchestratorRequestState``. While the segment-boundary fields lived in a single
|
||||
flat slot per request, whichever stage polled most recently overwrote the others,
|
||||
and the per-stage consumers then read back a different stage's boundary.
|
||||
|
||||
The duplex consumer of the per-stage segment (``DuplexOrchestrator``'s output
|
||||
context) is covered in ``test_duplex_orchestrator.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,8 +18,6 @@ from dataclasses import dataclass, field
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.engine.duplex.contracts import DuplexRequestIdentity
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.orchestrator import (
|
||||
Orchestrator,
|
||||
OrchestratorRequestState,
|
||||
@@ -54,22 +55,19 @@ class _RecordingPool:
|
||||
return _StageMetrics()
|
||||
|
||||
|
||||
def _duplex_req_state(request_id: str = "req-duplex") -> OrchestratorRequestState:
|
||||
def _streaming_req_state(request_id: str = "req-streaming") -> OrchestratorRequestState:
|
||||
req_state = OrchestratorRequestState(
|
||||
request_id=request_id,
|
||||
sampling_params_list=[SamplingParams(max_tokens=1), SamplingParams(max_tokens=1)],
|
||||
final_stage_id=TALKER_STAGE,
|
||||
session_owned=True,
|
||||
)
|
||||
req_state.streaming.enabled = True
|
||||
req_state.duplex_identity = DuplexRequestIdentity(
|
||||
session_id="session-1",
|
||||
fence=DuplexFence(session_id="session-1"),
|
||||
)
|
||||
return req_state
|
||||
|
||||
|
||||
def test_duplex_output_context_reads_only_its_own_stage_segment() -> None:
|
||||
req_state = _duplex_req_state()
|
||||
def test_segment_accessor_reads_only_its_own_stage() -> None:
|
||||
req_state = _streaming_req_state()
|
||||
req_state.streaming.segments[THINKER_STAGE] = StreamingSegmentState(
|
||||
finished=True,
|
||||
token_ids=[11, 22],
|
||||
@@ -78,26 +76,28 @@ def test_duplex_output_context_reads_only_its_own_stage_segment() -> None:
|
||||
# The talker polls after the thinker and is still mid-segment.
|
||||
req_state.streaming.segments[TALKER_STAGE] = StreamingSegmentState(finished=False)
|
||||
|
||||
thinker = Orchestrator._duplex_output_context(req_state, stage_id=THINKER_STAGE)
|
||||
talker = Orchestrator._duplex_output_context(req_state, stage_id=TALKER_STAGE)
|
||||
|
||||
assert thinker is not None
|
||||
assert talker is not None
|
||||
thinker = req_state.streaming.segment(THINKER_STAGE)
|
||||
talker = req_state.streaming.segment(TALKER_STAGE)
|
||||
|
||||
# These are the assertions that fail on the flat field: the talker is written
|
||||
# second, so the thinker loses its boundary to whichever stage polled last.
|
||||
assert thinker.segment_finished is True
|
||||
assert thinker.segment_token_ids == (11, 22)
|
||||
assert thinker.segment_output_metadata == {"stage": "thinker"}
|
||||
assert thinker.finished is True
|
||||
assert thinker.token_ids == [11, 22]
|
||||
assert thinker.output_metadata == {"stage": "thinker"}
|
||||
|
||||
assert talker.segment_finished is False
|
||||
assert talker.segment_token_ids == ()
|
||||
assert talker.segment_output_metadata == {}
|
||||
assert talker.finished is False
|
||||
assert talker.token_ids == []
|
||||
assert talker.output_metadata == {}
|
||||
|
||||
# A stage that never reported a boundary gets a fresh, unshared segment.
|
||||
unreported = req_state.streaming.segment(2)
|
||||
unreported.output_metadata["poisoned"] = True
|
||||
assert req_state.streaming.segment(2).output_metadata == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_segment_boundary_on_one_stage_does_not_build_metrics_on_another() -> None:
|
||||
req_state = _duplex_req_state()
|
||||
req_state = _streaming_req_state()
|
||||
req_state.streaming.segments[THINKER_STAGE] = StreamingSegmentState(finished=True)
|
||||
|
||||
talker_pool = _RecordingPool()
|
||||
|
||||
@@ -7,23 +7,17 @@ import asyncio
|
||||
import queue
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import janus
|
||||
import pytest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.engine.duplex.contracts import (
|
||||
DuplexStageRequestContext,
|
||||
DuplexStageSubmission,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.orchestrator import (
|
||||
Orchestrator,
|
||||
OrchestratorRequestState,
|
||||
StreamingSegmentState,
|
||||
_OrchestratorDuplexStagePort,
|
||||
)
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
|
||||
@@ -131,64 +125,6 @@ class FakePrewarmPool:
|
||||
return self.stage_client if self.get_bound_replica_id(request_id) is not None else None
|
||||
|
||||
|
||||
def _duplex_stage_port_submission():
|
||||
stage_pools = []
|
||||
for stage_id in range(3):
|
||||
pool = SimpleNamespace(
|
||||
stage_client=SimpleNamespace(default_sampling_params=SamplingParams(max_tokens=1)),
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
submit_initial=AsyncMock(return_value=stage_id + 10),
|
||||
submit_update=AsyncMock(return_value=stage_id + 20),
|
||||
)
|
||||
stage_pools.append(pool)
|
||||
request_states: dict[str, OrchestratorRequestState] = {}
|
||||
prewarm = AsyncMock()
|
||||
port = _OrchestratorDuplexStagePort(
|
||||
stage_pools=stage_pools,
|
||||
request_states=request_states,
|
||||
running_counter=None,
|
||||
cleanup_request_ids=AsyncMock(),
|
||||
async_chunk=True,
|
||||
prewarm_async_chunk_stages=prewarm,
|
||||
)
|
||||
context = DuplexStageRequestContext(
|
||||
request_id="req-duplex",
|
||||
session_id="session-duplex",
|
||||
fence=DuplexFence("session-duplex"),
|
||||
stage_id=0,
|
||||
final_stage_id=2,
|
||||
config_generation=0,
|
||||
sampling_params=tuple(SamplingParams(max_tokens=1) for _ in range(3)),
|
||||
)
|
||||
port.ensure_request(context)
|
||||
submission = DuplexStageSubmission(
|
||||
context=context,
|
||||
prompt={"prompt_token_ids": [1, 2]},
|
||||
already_submitted=False,
|
||||
)
|
||||
return port, stage_pools, request_states, prewarm, submission
|
||||
|
||||
|
||||
def test_duplex_bridge_state_catches_up_after_serving_turn_boundary() -> None:
|
||||
port, _, request_states, _, submission = _duplex_stage_port_submission()
|
||||
request_state = request_states[submission.context.request_id]
|
||||
bridge_state = request_state.streaming.bridge_states["duplex"]
|
||||
bridge_state["model_turn_id"] = 0
|
||||
|
||||
next_context = DuplexStageRequestContext(
|
||||
request_id=submission.context.request_id,
|
||||
session_id=submission.context.session_id,
|
||||
fence=DuplexFence(submission.context.session_id, turn_id=1),
|
||||
stage_id=submission.context.stage_id,
|
||||
final_stage_id=submission.context.final_stage_id,
|
||||
config_generation=submission.context.config_generation,
|
||||
sampling_params=submission.context.sampling_params,
|
||||
)
|
||||
port.ensure_request(next_context)
|
||||
|
||||
assert bridge_state["model_turn_id"] == 1
|
||||
|
||||
|
||||
def _request_output(request_id: str) -> RequestOutput:
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
@@ -279,13 +215,12 @@ async def test_async_prewarm_skips_outgoing_only_stage(payload_sender_info) -> N
|
||||
stage1.stage_client.get_payload_sender_info = MagicMock(return_value=payload_sender_info)
|
||||
orchestrator.stage_pools = [stage0, stage1, stage2]
|
||||
orchestrator._emit_tx_edge = lambda **_kwargs: None
|
||||
orchestrator._record_duplex_stage_submission = MagicMock()
|
||||
orchestrator._on_stage_submitted = MagicMock()
|
||||
req_state = OrchestratorRequestState(
|
||||
request_id="req-prewarm",
|
||||
prompt={"prompt_token_ids": [1, 2]},
|
||||
sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(3)],
|
||||
final_stage_id=2,
|
||||
duplex_identity=SimpleNamespace(),
|
||||
)
|
||||
|
||||
prewarmed = await orchestrator._prewarm_async_chunk_stages(
|
||||
@@ -306,7 +241,7 @@ async def test_async_prewarm_skips_outgoing_only_stage(payload_sender_info) -> N
|
||||
assert stage2.submitted[0].resumable is True
|
||||
assert 1 not in req_state.stage_submit_ts
|
||||
assert 2 in req_state.stage_submit_ts
|
||||
orchestrator._record_duplex_stage_submission.assert_called_once_with(
|
||||
orchestrator._on_stage_submitted.assert_called_once_with(
|
||||
2,
|
||||
"req-prewarm",
|
||||
0,
|
||||
@@ -314,42 +249,6 @@ async def test_async_prewarm_skips_outgoing_only_stage(payload_sender_info) -> N
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_prewarm_runs_after_first_stage0_submission() -> None:
|
||||
port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission()
|
||||
prewarm.return_value = True
|
||||
|
||||
result = await port.submit(submission)
|
||||
|
||||
assert result.stage_id == 0
|
||||
stage_pools[0].submit_initial.assert_awaited_once()
|
||||
prewarm.assert_awaited_once_with("req-duplex", ANY, request_states["req-duplex"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_submit_bails_out_when_prewarm_failed_the_request() -> None:
|
||||
"""A failed prewarm has already aborted the request and popped its state.
|
||||
|
||||
Returning a success result here would hand the control plane a replica for a
|
||||
request that no longer exists, and the trailing bookkeeping would re-register
|
||||
a running counter the cleanup just released -- a leak that never decrements.
|
||||
"""
|
||||
port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission()
|
||||
counter = MagicMock()
|
||||
port._running_counter = counter
|
||||
prewarm.return_value = False
|
||||
request_state = request_states["req-duplex"]
|
||||
|
||||
with pytest.raises(RuntimeError, match="prewarm failed"):
|
||||
await port.submit(submission)
|
||||
|
||||
prewarm.assert_awaited_once()
|
||||
assert request_state.duplex_stage_fences == {}
|
||||
assert request_state.stage_submit_ts == {}
|
||||
assert request_state.running_counter_registered is False
|
||||
counter.increment.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_route_forwards_to_outgoing_only_stage() -> None:
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine import FinishReason
|
||||
|
||||
from vllm_omni.outputs.mm_outputs import MultimodalPayload
|
||||
from vllm_omni.outputs.output_modality import OutputModality, TensorAccumulationStrategy
|
||||
from vllm_omni.outputs.output_modality import OutputModality
|
||||
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -58,7 +58,7 @@ def test_rate_is_latest_snapshot_before_consolidation(key, representation):
|
||||
assert payload.get(key) is payload.to_dict()[key]
|
||||
payload = payload.merged_with(MultimodalPayload(tensors={"audio": torch.full((2,), 3)}))
|
||||
assert int(payload[key]) == 48000 # Missing metadata means retain, not clear.
|
||||
payload.consolidate_tensors(TensorAccumulationStrategy.CONCAT_LAST)
|
||||
payload.consolidate_tensors(OutputModality.AUDIO)
|
||||
payload.consolidate_metadata()
|
||||
torch.testing.assert_close(payload["audio"], torch.arange(4).repeat_interleave(2))
|
||||
|
||||
@@ -86,7 +86,7 @@ def test_self_merge_retains_snapshot_and_content(key):
|
||||
assert payload is not None
|
||||
payload = payload.merged_with(payload)
|
||||
assert int(payload[key]) == 24000
|
||||
payload.consolidate_tensors(TensorAccumulationStrategy.CONCAT_LAST)
|
||||
payload.consolidate_tensors(OutputModality.AUDIO)
|
||||
torch.testing.assert_close(payload["audio"], torch.tensor([1.0, 2.0, 1.0, 2.0]))
|
||||
|
||||
|
||||
@@ -138,6 +138,6 @@ def test_non_snapshot_tensors_still_accumulate_and_empty_merge_keeps_identity():
|
||||
assert first is not None and second is not None
|
||||
assert MultimodalPayload().merged_with(first) is first
|
||||
merged = first.merged_with(second)
|
||||
merged.consolidate_tensors(TensorAccumulationStrategy.CONCAT_LAST)
|
||||
merged.consolidate_tensors(OutputModality.AUDIO)
|
||||
torch.testing.assert_close(merged["latent"], torch.tensor([1.0, 3.0]))
|
||||
torch.testing.assert_close(merged["audio"], torch.tensor([2.0, 4.0]))
|
||||
@@ -389,7 +389,7 @@ class TestSingleStageModeDetection:
|
||||
return_value=None,
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.load_deploy_config",
|
||||
"vllm_omni.engine.omni_engine_base.load_deploy_config",
|
||||
return_value=SimpleNamespace(duplex_session=DuplexSessionRuntimeConfig()),
|
||||
)
|
||||
mocker.patch.object(
|
||||
@@ -446,11 +446,11 @@ class TestSingleStageModeDetection:
|
||||
def test_deploy_config_loads_duplex_runtime_config(self, mocker: MockerFixture):
|
||||
duplex_session = DuplexSessionRuntimeConfig(max_sessions=2)
|
||||
get_pipeline_config = mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.StageConfigFactory.get_pipeline_config",
|
||||
"vllm_omni.engine.omni_engine_base.StageConfigFactory.get_pipeline_config",
|
||||
return_value=None,
|
||||
)
|
||||
load_deploy_config = mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.load_deploy_config",
|
||||
"vllm_omni.engine.omni_engine_base.load_deploy_config",
|
||||
return_value=SimpleNamespace(duplex_session=duplex_session),
|
||||
)
|
||||
|
||||
@@ -467,7 +467,8 @@ class TestSingleStageModeDetection:
|
||||
deploy_config_path="/fake/duplex.yaml",
|
||||
)
|
||||
load_deploy_config.assert_called_once_with("/fake/duplex.yaml")
|
||||
assert engine.duplex_session_config is duplex_session
|
||||
# The turn-based engine only keeps the resolved deploy profile for introspection.
|
||||
assert engine.deploy_config.duplex_session is duplex_session
|
||||
|
||||
def test_auto_discovered_deploy_loads_duplex_runtime_config(
|
||||
self,
|
||||
@@ -476,11 +477,11 @@ class TestSingleStageModeDetection:
|
||||
deploy_path = "/resolved/qwen3_omni_moe.yaml"
|
||||
duplex_session = DuplexSessionRuntimeConfig(server_vad_model_path="/models/silero_vad.onnx")
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.StageConfigFactory.get_pipeline_config",
|
||||
"vllm_omni.engine.omni_engine_base.StageConfigFactory.get_pipeline_config",
|
||||
return_value=None,
|
||||
)
|
||||
load_deploy_config = mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.load_deploy_config",
|
||||
"vllm_omni.engine.omni_engine_base.load_deploy_config",
|
||||
return_value=SimpleNamespace(duplex_session=duplex_session),
|
||||
)
|
||||
|
||||
@@ -557,7 +558,7 @@ class TestEndpointRestrictionsTrustRemoteCode:
|
||||
|
||||
def _make_engine_no_thread(self, mocker: MockerFixture, **kwargs: Any) -> AsyncOmniEngine:
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.load_deploy_config",
|
||||
"vllm_omni.engine.omni_engine_base.load_deploy_config",
|
||||
return_value=SimpleNamespace(duplex_session=DuplexSessionRuntimeConfig()),
|
||||
)
|
||||
mocker.patch.object(
|
||||
|
||||
@@ -0,0 +1,609 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The thin websocket handler: handshake, command translation, event pump, resume/takeover."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import WebSocketDisconnect
|
||||
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex import commands
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities
|
||||
from vllm_omni.engine.duplex.events import AudioDelta, DuplexEvent, SessionClosed, SessionCreated
|
||||
from vllm_omni.engine.duplex.messages import DuplexSessionError
|
||||
from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
from vllm_omni.entrypoints.duplex.websocket import MAX_EVENT_BYTES
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_DISCONNECT = object()
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self, query: dict[str, str] | None = None) -> None:
|
||||
self.query_params = dict(query or {})
|
||||
self.sent: list[dict[str, Any]] = []
|
||||
self.accepted = False
|
||||
self.closed: list[tuple[int, str]] = []
|
||||
self._send_failure: str | None = None
|
||||
self._inbound: asyncio.Queue[Any] = asyncio.Queue()
|
||||
|
||||
async def accept(self) -> None:
|
||||
self.accepted = True
|
||||
|
||||
async def send_json(self, payload: dict[str, Any]) -> None:
|
||||
if self.closed or self._send_failure is not None:
|
||||
raise RuntimeError(
|
||||
self._send_failure or "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'."
|
||||
)
|
||||
self.sent.append(json.loads(json.dumps(payload)))
|
||||
|
||||
async def receive_text(self) -> str:
|
||||
item = await self._inbound.get()
|
||||
if item is _DISCONNECT:
|
||||
raise WebSocketDisconnect(code=1000)
|
||||
return item
|
||||
|
||||
async def close(self, code: int = 1000, reason: str = "") -> None:
|
||||
self.closed.append((code, reason))
|
||||
# A closed socket ends the reader exactly like a client disconnect.
|
||||
self._inbound.put_nowait(_DISCONNECT)
|
||||
|
||||
# ---- test helpers ----
|
||||
|
||||
def feed(self, payload: dict[str, Any] | str) -> None:
|
||||
self._inbound.put_nowait(payload if isinstance(payload, str) else json.dumps(payload))
|
||||
|
||||
def disconnect(self) -> None:
|
||||
self._inbound.put_nowait(_DISCONNECT)
|
||||
|
||||
def break_sends(self) -> None:
|
||||
"""Kill the write half only: the reader stays parked, as it does in practice."""
|
||||
self._send_failure = "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'."
|
||||
|
||||
def types(self) -> list[str]:
|
||||
return [payload["type"] for payload in self.sent]
|
||||
|
||||
async def wait_for(self, wire_type: str, *, timeout_s: float = 2.0) -> dict[str, Any]:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout_s
|
||||
while loop.time() < deadline:
|
||||
for payload in self.sent:
|
||||
if payload["type"] == wire_type:
|
||||
return payload
|
||||
await asyncio.sleep(0.005)
|
||||
raise AssertionError(f"no {wire_type!r} in {self.types()}")
|
||||
|
||||
|
||||
class FakeHandle:
|
||||
def __init__(self, session_id: str, capabilities: DuplexCapabilities, *, idle_timeout_s: float = 300) -> None:
|
||||
self.session_id = session_id
|
||||
self.capabilities = capabilities
|
||||
self.public_session: dict[str, Any] = {"id": session_id, "idle_timeout_s": idle_timeout_s}
|
||||
self.lease_generation = 0
|
||||
self.closed = False
|
||||
self.close_reasons: list[str] = []
|
||||
self.commands: list[commands.DuplexCommand] = []
|
||||
self._outbox: asyncio.Queue[DuplexEvent] = asyncio.Queue()
|
||||
|
||||
def deliver(self, event: DuplexEvent) -> None:
|
||||
self._outbox.put_nowait(event)
|
||||
|
||||
async def submit(self, command: commands.DuplexCommand) -> None:
|
||||
if self.closed:
|
||||
raise DuplexSessionError("closed", code="session_closed", session_id=self.session_id)
|
||||
self.commands.append(command)
|
||||
|
||||
async def events(self):
|
||||
while True:
|
||||
event = await self._outbox.get()
|
||||
yield event
|
||||
if event.is_terminal:
|
||||
return
|
||||
|
||||
async def close(self, *, reason: str = "client_close", timeout: float | None = None) -> None:
|
||||
self.close_reasons.append(reason)
|
||||
if self.closed:
|
||||
return
|
||||
self.closed = True
|
||||
self.deliver(SessionClosed(session_id=self.session_id, reason=reason))
|
||||
|
||||
|
||||
class FakeOmni:
|
||||
def __init__(
|
||||
self, *, resumable: bool = True, replay_max_bytes: int = 64 * 1024, idle_timeout_s: float = 300
|
||||
) -> None:
|
||||
self.duplex_session_config = DuplexSessionRuntimeConfig(
|
||||
resume_replay_ttl_s=60.0, resume_replay_max_bytes_per_session=replay_max_bytes
|
||||
)
|
||||
self.capabilities = DuplexCapabilities(supports_session_resume=resumable)
|
||||
self.idle_timeout_s = idle_timeout_s
|
||||
self.opened: list[dict[str, Any]] = []
|
||||
self.handles: dict[str, FakeHandle] = {}
|
||||
self.resumed: list[tuple[str, int]] = []
|
||||
self.detached: list[str] = []
|
||||
self.open_error: DuplexSessionError | None = None
|
||||
|
||||
async def open_session(self, config: Any) -> FakeHandle:
|
||||
self.opened.append(dict(config))
|
||||
if self.open_error is not None:
|
||||
raise self.open_error
|
||||
session_id = f"duplex-{len(self.handles) + 1:032x}"
|
||||
handle = FakeHandle(session_id, self.capabilities, idle_timeout_s=self.idle_timeout_s)
|
||||
self.handles[session_id] = handle
|
||||
handle.deliver(SessionCreated(session_id=session_id, session={"id": session_id, "model": config.get("model")}))
|
||||
return handle
|
||||
|
||||
def get_session(self, session_id: str) -> FakeHandle | None:
|
||||
return self.handles.get(session_id)
|
||||
|
||||
async def resume_session(self, session_id: str, *, expected_lease_generation: int) -> FakeHandle:
|
||||
self.resumed.append((session_id, expected_lease_generation))
|
||||
handle = self.handles[session_id]
|
||||
handle.lease_generation = expected_lease_generation + 1
|
||||
return handle
|
||||
|
||||
async def detach_session(self, session_id: str) -> None:
|
||||
self.detached.append(session_id)
|
||||
|
||||
|
||||
def _handler(omni: FakeOmni, **kwargs: Any) -> OmniDuplexSessionHandler:
|
||||
kwargs.setdefault("config_timeout_s", 1.0)
|
||||
kwargs.setdefault("idle_timeout_s", 5.0)
|
||||
return OmniDuplexSessionHandler(duplex_omni=omni, **kwargs)
|
||||
|
||||
|
||||
def _session_update(**session: Any) -> dict[str, Any]:
|
||||
return {"type": "session.update", "session": {"model": "test-model", "modalities": ["audio", "text"], **session}}
|
||||
|
||||
|
||||
async def _open(
|
||||
handler: OmniDuplexSessionHandler, omni: FakeOmni, **session: Any
|
||||
) -> tuple[FakeWebSocket, FakeHandle, asyncio.Task]:
|
||||
ws = FakeWebSocket({"duplex": "1", "autostart": "0"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed(_session_update(**session))
|
||||
created = await ws.wait_for("session.created")
|
||||
handle = omni.handles[created["session"]["id"]]
|
||||
return ws, handle, task
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Handshake and commands #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_update_opens_a_session_and_announces_the_server_id() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni, session_id="mine", instructions="hi")
|
||||
|
||||
assert ws.accepted
|
||||
created = ws.sent[0]
|
||||
assert created["type"] == "session.created"
|
||||
assert created["session"]["id"] == handle.session_id != "mine"
|
||||
assert created["attachment_generation"] == 1
|
||||
assert isinstance(created["resume_token"], str) and created["resume_token"]
|
||||
assert "incarnation" not in created
|
||||
assert "server_event_seq" not in created
|
||||
# The whole session object went to open_session; the id inside it is ignored there.
|
||||
assert omni.opened == [
|
||||
{"model": "test-model", "modalities": ["audio", "text"], "session_id": "mine", "instructions": "hi"}
|
||||
]
|
||||
|
||||
ws.feed({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 8).decode("ascii")})
|
||||
ws.feed({"type": "input_audio_buffer.commit", "event_id": "evt-c"})
|
||||
await asyncio.sleep(0.05)
|
||||
assert [type(command) for command in handle.commands] == [commands.AppendAudio, commands.Commit]
|
||||
assert handle.commands[1].event_id == "evt-c"
|
||||
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
# A resumable session is detached (engine-owned grace), not closed.
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert handle.close_reasons == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_events_are_journaled_and_sent_in_order() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
delta = await ws.wait_for("response.output_audio.delta")
|
||||
assert delta["delta"] == "aGk=" and delta["server_event_seq"] == 1
|
||||
|
||||
ws.feed({"type": "session.event_ack", "server_event_seq": 1})
|
||||
ws.feed({"type": "session.event_ack", "server_event_seq": 9, "event_id": "evt-ack"})
|
||||
error = await ws.wait_for("error")
|
||||
assert error["error"]["code"] == "invalid_event_ack" and error["error"]["event_id"] == "evt-ack"
|
||||
|
||||
await handle.close(reason="client_close")
|
||||
closed = await ws.wait_for("session.closed")
|
||||
assert closed["reason"] == "client_close"
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.closed[0][1] == "client_close"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_envelope_errors_are_answered_locally_without_touching_the_session() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
ws.feed("not json")
|
||||
ws.feed("x" * (MAX_EVENT_BYTES + 1))
|
||||
ws.feed({"type": "totally.unknown", "event_id": "evt-u"})
|
||||
ws.feed({"type": "session.resume", "event_id": "evt-r"})
|
||||
ws.feed({"type": "playback.ack", "event_id": "evt-p"}) # played_ms missing
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
codes = [payload["error"]["code"] for payload in ws.sent if payload["type"] == "error"]
|
||||
assert codes == ["invalid_json", "event_too_large", "unknown_event", "unsupported_session_resume", "bad_event"]
|
||||
errors = [payload for payload in ws.sent if payload["type"] == "error"]
|
||||
assert errors[2]["error"]["event_id"] == "evt-u"
|
||||
assert errors[4]["error"]["event_id"] == "evt-p"
|
||||
assert handle.commands == []
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_rejection_is_reported_and_the_socket_ends() -> None:
|
||||
omni = FakeOmni()
|
||||
omni.open_error = DuplexSessionError("no room", code="resource_exhausted", retryable=True)
|
||||
handler = _handler(omni)
|
||||
ws = FakeWebSocket({"duplex": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed(_session_update())
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.types() == ["error"]
|
||||
assert ws.sent[0]["error"]["code"] == "resource_exhausted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_config_timeout_and_first_message_validation() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni, config_timeout_s=0.05)
|
||||
ws = FakeWebSocket({"duplex": "1"})
|
||||
await asyncio.wait_for(handler.handle_realtime_session(ws), timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "config_timeout"
|
||||
|
||||
ws = FakeWebSocket({"duplex": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed("{not json")
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "invalid_json"
|
||||
assert omni.opened == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_timeout_detaches_a_resumable_session_like_a_disconnect() -> None:
|
||||
omni = FakeOmni(idle_timeout_s=0.05)
|
||||
handler = _handler(omni, idle_timeout_s=0.05)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
# Serving makes no session-lifetime decision: the engine lease decides
|
||||
# whether a silent, detached session expires.
|
||||
assert handle.close_reasons == []
|
||||
assert omni.detached == [handle.session_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_idle_timeout_closes_a_non_resumable_session() -> None:
|
||||
omni = FakeOmni(idle_timeout_s=0.05, resumable=False)
|
||||
handler = _handler(omni, idle_timeout_s=0.05)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert handle.close_reasons == ["disconnect"]
|
||||
assert omni.detached == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transport_send_failure_detaches_the_session_instead_of_closing_it() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
# Only the write half dies, so the pump's send is the one and only report
|
||||
# of the broken socket (the reader is still parked on receive).
|
||||
ws.break_sends()
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
# The engine session is alive and resumable, so this is a disconnect of
|
||||
# the current attachment, not a session close; the pump keeps journaling.
|
||||
assert handle.close_reasons == []
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert not handler._pumps[handle.session_id].done()
|
||||
|
||||
# The reader sees the same broken socket a moment later. The attachment is
|
||||
# already gone, so this must not detach a second time: another detach would
|
||||
# restart the engine's disconnect grace window.
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert handle.close_reasons == []
|
||||
|
||||
handle.deliver(SessionClosed(session_id=handle.session_id, reason="client_close"))
|
||||
await asyncio.sleep(0.05)
|
||||
assert handle.session_id not in handler._pumps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_resumable_session_is_closed_on_disconnect() -> None:
|
||||
omni = FakeOmni(resumable=False)
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
assert "resume_token" not in ws.sent[0]
|
||||
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert handle.close_reasons == ["disconnect"]
|
||||
assert omni.detached == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_client_close_still_delivers_session_closed_before_the_socket_goes() -> None:
|
||||
"""The read loop must not outrun the pump's terminal event.
|
||||
|
||||
``DuplexSessionHandle._deliver`` queues ``SessionClosed`` and marks the
|
||||
handle closed in one synchronous step, so ``handle.closed`` is already true
|
||||
while the event is still sitting in the outbox. ``_read_loop`` loops on
|
||||
exactly that flag: it returned, the endpoint returned, and the ASGI server
|
||||
tore the socket down with ``session.closed`` unsent -- the client saw an
|
||||
abrupt close (no close frame) instead of the terminal event it was waiting
|
||||
for.
|
||||
|
||||
The teardown is what makes this observable, so it is modelled here: a real
|
||||
connection stops accepting writes the moment the endpoint returns, which is
|
||||
why a pump that is merely *still scheduled* is already too late.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws = FakeWebSocket({"duplex": "1", "autostart": "0"})
|
||||
|
||||
async def serve() -> None:
|
||||
try:
|
||||
await handler.handle_realtime_session(ws)
|
||||
finally:
|
||||
# The ASGI server drops the connection when the endpoint returns;
|
||||
# anything the pump writes after this never reaches the client.
|
||||
ws.break_sends()
|
||||
|
||||
task = asyncio.create_task(serve())
|
||||
ws.feed(_session_update())
|
||||
created = await ws.wait_for("session.created")
|
||||
handle = omni.handles[created["session"]["id"]]
|
||||
submit = handle.submit
|
||||
|
||||
async def closing_submit(command: commands.DuplexCommand) -> None:
|
||||
await submit(command)
|
||||
if isinstance(command, commands.CloseSession):
|
||||
# Same order as the real handle: queue the event, then flip the flag.
|
||||
handle.deliver(SessionClosed(session_id=handle.session_id, reason="client_close"))
|
||||
handle.closed = True
|
||||
|
||||
handle.submit = closing_submit # type: ignore[method-assign]
|
||||
|
||||
ws.feed({"type": "session.close"})
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
assert ws.types()[-1] == "session.closed", f"terminal event never reached the wire: {ws.types()}"
|
||||
assert ws.closed and ws.closed[0][0] == 1000, "the socket must close with a normal close frame"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Resume and takeover #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def _resume(
|
||||
handler: OmniDuplexSessionHandler,
|
||||
session_id: str,
|
||||
token: str,
|
||||
*,
|
||||
last_seq: int = 0,
|
||||
) -> tuple[FakeWebSocket, asyncio.Task]:
|
||||
ws = FakeWebSocket({"duplex": "1", "resume": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": session_id,
|
||||
"resume_token": token,
|
||||
"last_received_server_event_seq": last_seq,
|
||||
}
|
||||
)
|
||||
return ws, task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_after_disconnect_replays_missed_events_and_rotates_the_token() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="one"))
|
||||
await ws.wait_for("response.output_audio.delta")
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
# Emitted while detached: journaled for replay only.
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="two"))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
ws2, task2 = await _resume(handler, handle.session_id, token, last_seq=1)
|
||||
resumed = await ws2.wait_for("session.resumed")
|
||||
assert resumed["session_id"] == handle.session_id
|
||||
assert resumed["attachment_generation"] == 2
|
||||
assert resumed["resume_token"] != token
|
||||
assert "incarnation" not in resumed
|
||||
replayed = await ws2.wait_for("response.output_audio.delta")
|
||||
assert replayed["delta"] == "two" and replayed["server_event_seq"] == 2
|
||||
assert omni.resumed == [(handle.session_id, 0)]
|
||||
|
||||
# The old token is revoked by the rotation.
|
||||
ws3, task3 = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(task3, timeout=2.0)
|
||||
assert ws3.sent[0]["error"]["code"] == "invalid_resume_token"
|
||||
|
||||
ws2.feed({"type": "session.close"})
|
||||
await asyncio.sleep(0.05)
|
||||
ws2.disconnect()
|
||||
await asyncio.wait_for(task2, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_takes_over_a_live_attachment() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
|
||||
ws2, task2 = await _resume(handler, handle.session_id, token)
|
||||
await ws2.wait_for("session.resumed")
|
||||
replaced = await ws.wait_for("session.replaced")
|
||||
assert replaced["attachment_generation"] == 1
|
||||
assert ws.closed and ws.closed[0][1] == "session_replaced"
|
||||
# The replaced socket's later input is ignored; the new one drives the session.
|
||||
ws.feed({"type": "input_audio_buffer.clear"})
|
||||
ws2.feed({"type": "input_audio_buffer.clear", "event_id": "evt-new"})
|
||||
await asyncio.sleep(0.05)
|
||||
assert [command.event_id for command in handle.commands] == ["evt-new"]
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
ws2.disconnect()
|
||||
await asyncio.wait_for(task2, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_validation_errors() -> None:
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws = FakeWebSocket({"duplex": "1", "resume": "1"})
|
||||
task = asyncio.create_task(handler.handle_realtime_session(ws))
|
||||
ws.feed({"type": "session.resume", "session_id": "duplex-x"})
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "invalid_session_resume"
|
||||
|
||||
ws, task = await _resume(handler, "duplex-unknown", "token")
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert ws.sent[0]["error"]["code"] == "session_resume_expired"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_journal_overflow_degrades_to_live_delivery_with_resync_required() -> None:
|
||||
omni = FakeOmni(replay_max_bytes=256)
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="x" * 400))
|
||||
resync = await ws.wait_for("session.resync_required")
|
||||
assert resync["reason"] == "journal_overflow"
|
||||
delta = await ws.wait_for("response.output_audio.delta")
|
||||
assert "server_event_seq" not in delta
|
||||
assert ws.types().index("session.resync_required") < ws.types().index("response.output_audio.delta")
|
||||
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Envelope helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_realtime_envelope_query_rules() -> None:
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m"})
|
||||
assert envelope.initial_open_payload() == {"model": "m"}
|
||||
assert envelope.initial_open_payload() is None # autostart happens once
|
||||
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
assert envelope.resume_only is True
|
||||
assert envelope.initial_open_payload() is None
|
||||
assert RealtimeEnvelope.from_query_params({"resume": "1"}).resume_only is True
|
||||
assert RealtimeEnvelope.from_query_params({"session_id": "mine"}).default_session_payload() == {"model": None}
|
||||
|
||||
|
||||
def test_realtime_envelope_first_message_classification() -> None:
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
resume = envelope.first_message({"type": "session.resume", "session_id": "s"})
|
||||
assert resume.kind == "resume" and resume.resume_payload["session_id"] == "s"
|
||||
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
opened = envelope.first_message({"type": "session.update", "session": {"model": "x", "instructions": "hi"}})
|
||||
assert opened.kind == "open" and opened.session_payload == {"model": "x", "instructions": "hi"}
|
||||
assert opened.pending_command_payload is None
|
||||
|
||||
envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
|
||||
autostarted = envelope.first_message({"type": "input_audio_buffer.commit"})
|
||||
assert autostarted.kind == "open" and autostarted.session_payload == {"model": "m"}
|
||||
assert autostarted.pending_command_payload == {"type": "input_audio_buffer.commit"}
|
||||
|
||||
|
||||
def test_parse_resume_request_requires_the_three_fields_only() -> None:
|
||||
request = parse_resume_request({"session_id": "s", "resume_token": "t", "last_received_server_event_seq": 3})
|
||||
assert request is not None
|
||||
assert (request.session_id, request.resume_token, request.last_received_server_event_seq) == ("s", "t", 3)
|
||||
assert parse_resume_request({"session_id": "s", "resume_token": "t"}).last_received_server_event_seq == 0
|
||||
assert parse_resume_request({"session_id": "s", "resume_token": "t", "incarnation": 1}) is not None
|
||||
assert parse_resume_request({"session_id": "s"}) is None
|
||||
assert parse_resume_request({"session_id": "s", "resume_token": "t", "last_received_server_event_seq": -1}) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_that_fails_to_activate_does_not_detach_the_live_attachment() -> None:
|
||||
"""A rejected resume must roll back only what it owns.
|
||||
|
||||
Two reconnects can both authenticate and both complete the engine resume;
|
||||
only one activates. The loser's activation raises, and the rollback used to
|
||||
detach the *session*, starting the engine's disconnect grace for the winner
|
||||
-- a grace ordinary heartbeats do not clear.
|
||||
|
||||
The activation failure is injected rather than raced for: what is under test
|
||||
is the rollback's precondition, not the window that produces it.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
try:
|
||||
assert await registry.has_attachment(handle.session_id), "the opener is attached"
|
||||
detached_before = list(omni.detached)
|
||||
|
||||
async def failing_resume(*args, **kwargs):
|
||||
raise RuntimeError("transport activation lost the race")
|
||||
|
||||
original_resume = registry.resume
|
||||
registry.resume = failing_resume # type: ignore[method-assign]
|
||||
try:
|
||||
ws_lose, task_lose = await _resume(handler, handle.session_id, token)
|
||||
error = await ws_lose.wait_for("error")
|
||||
finally:
|
||||
registry.resume = original_resume # type: ignore[method-assign]
|
||||
|
||||
assert error["error"]["code"] == "session_resume_conflict"
|
||||
assert omni.detached == detached_before, "the loser must not detach the winner"
|
||||
assert await registry.has_attachment(handle.session_id), "the winner is still attached"
|
||||
finally:
|
||||
for pending in (task, locals().get("task_lose")):
|
||||
if pending is not None:
|
||||
pending.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await pending
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.runtime_adapter import (
|
||||
validate_serving_runtime_adapter,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"module_name",
|
||||
[
|
||||
"vllm_omni.entrypoints.duplex.session_runner",
|
||||
"vllm_omni.entrypoints.duplex.runtime_bridge",
|
||||
"vllm_omni.entrypoints.duplex.serving",
|
||||
],
|
||||
)
|
||||
def test_generic_openai_runtime_import_does_not_load_model_adapters(module_name: str) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = f"""
|
||||
import sys
|
||||
|
||||
import {module_name}
|
||||
|
||||
model_duplex_prefixes = (
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex",
|
||||
"vllm_omni.model_executor.models.nemotron_voicechat.duplex",
|
||||
"vllm_omni.model_executor.models.personaplex.duplex",
|
||||
)
|
||||
loaded = sorted(
|
||||
name
|
||||
for name in sys.modules
|
||||
if any(
|
||||
name == prefix or name.startswith(prefix + ".")
|
||||
for prefix in model_duplex_prefixes
|
||||
)
|
||||
)
|
||||
if loaded:
|
||||
raise SystemExit("generic OpenAI runtime loaded model duplex modules: " + ", ".join(loaded))
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_generic_handler_supports_adapterless_chat_fallback() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
script = """
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
|
||||
chat_service = SimpleNamespace(
|
||||
engine_client=SimpleNamespace(),
|
||||
model_config=SimpleNamespace(model="test-model"),
|
||||
)
|
||||
handler = OmniDuplexSessionHandler(chat_service=chat_service)
|
||||
if handler._serving_runtime_adapter is not None:
|
||||
raise SystemExit("generic handler silently selected a model serving adapter")
|
||||
"""
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def _valid_runtime_adapter() -> SimpleNamespace:
|
||||
data_plane = SimpleNamespace(
|
||||
begin_request=lambda request_id: None,
|
||||
is_terminal=lambda request_id: False,
|
||||
mark_terminal=lambda request_id: None,
|
||||
close_stream=lambda request_id: None,
|
||||
close_session=lambda session_id, **kwargs: None,
|
||||
project=lambda result, **kwargs: (),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
adapter_id="test-adapter",
|
||||
session_states={},
|
||||
data_plane=data_plane,
|
||||
clean_response_done_prefix="clean",
|
||||
interrupted_tts_prefix="interrupted",
|
||||
private_runtime_config_keys=frozenset({"private_key"}),
|
||||
create_session_state=lambda: object(),
|
||||
session_state=lambda session_id: object(),
|
||||
remove_session_state=lambda session_id: None,
|
||||
is_enabled=lambda config: True,
|
||||
capabilities=lambda **kwargs: object(),
|
||||
validate_client_extra_body=lambda extra_body: None,
|
||||
prepare_runtime_config=lambda config, **kwargs: {},
|
||||
runtime_config_for_update=lambda config, current: {},
|
||||
data_plane_context=lambda **kwargs: object(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "invalid_value"),
|
||||
[
|
||||
("session_states", None),
|
||||
("clean_response_done_prefix", None),
|
||||
("interrupted_tts_prefix", None),
|
||||
("private_runtime_config_keys", {"private_key"}),
|
||||
],
|
||||
)
|
||||
def test_runtime_adapter_validator_rejects_invalid_protocol_attributes(
|
||||
attribute: str,
|
||||
invalid_value: object,
|
||||
) -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
setattr(adapter, attribute, invalid_value)
|
||||
|
||||
with pytest.raises(TypeError, match=attribute):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
|
||||
|
||||
def test_runtime_adapter_validator_rejects_incomplete_data_plane() -> None:
|
||||
adapter = _valid_runtime_adapter()
|
||||
adapter.data_plane.close_session = None
|
||||
|
||||
with pytest.raises(TypeError, match="data_plane.*close_session"):
|
||||
validate_serving_runtime_adapter(adapter)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Regression tests for duplex ``runtime_control`` payload redaction.
|
||||
|
||||
The redacted result is emitted on the WebSocket (for example
|
||||
``session.created.runtime_control``), so it must always survive
|
||||
``json.dumps``. Control results carry typed values straight from the engine --
|
||||
``fence`` and ``accepted_fence`` are ``DuplexFence`` instances added by
|
||||
``DuplexControlClient.execute`` -- which previously reached
|
||||
``WebSocket.send_json`` unconverted and killed the session at handshake.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.entrypoints.duplex.runtime_bridge import NativeRuntimeBridgeMixin
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
_redact = NativeRuntimeBridgeMixin._redact_runtime_control_result
|
||||
|
||||
|
||||
def _control_result() -> dict[str, object]:
|
||||
"""The shape ``DuplexControlClient.execute`` returns for an open."""
|
||||
return {
|
||||
"ok": True,
|
||||
"operation": "open",
|
||||
"session_id": "sess-1",
|
||||
"fence": DuplexFence("sess-1", epoch=0, turn_id=0, response_seq=0, incarnation=0),
|
||||
"accepted_fence": DuplexFence("sess-1", epoch=1, turn_id=2, response_seq=3, incarnation=4),
|
||||
"stage_results": [{"stage_id": -1, "replica_id": -1, "result": {"supported": True}}],
|
||||
}
|
||||
|
||||
|
||||
def test_redacted_control_result_is_json_serializable() -> None:
|
||||
"""The whole payload must serialize the way send_json does."""
|
||||
redacted = _redact(_control_result())
|
||||
# Must not raise.
|
||||
json.dumps({"type": "session.created", "runtime_control": redacted})
|
||||
|
||||
|
||||
def test_fences_are_converted_to_plain_dicts() -> None:
|
||||
redacted = _redact(_control_result())
|
||||
assert redacted["fence"] == {
|
||||
"session_id": "sess-1",
|
||||
"epoch": 0,
|
||||
"turn_id": 0,
|
||||
"response_seq": 0,
|
||||
"incarnation": 0,
|
||||
}
|
||||
# Field values are preserved, not just the type.
|
||||
assert redacted["accepted_fence"]["epoch"] == 1
|
||||
assert redacted["accepted_fence"]["turn_id"] == 2
|
||||
assert redacted["accepted_fence"]["response_seq"] == 3
|
||||
assert redacted["accepted_fence"]["incarnation"] == 4
|
||||
|
||||
|
||||
def test_nested_dataclasses_are_converted() -> None:
|
||||
"""Fences can appear inside stage results, not only at the top level."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"stage_results": [
|
||||
{"stage_id": 0, "result": {"fence": DuplexFence("sess-2", epoch=7)}},
|
||||
]
|
||||
}
|
||||
)
|
||||
json.dumps(redacted)
|
||||
assert redacted["stage_results"][0]["result"]["fence"]["epoch"] == 7
|
||||
|
||||
|
||||
def test_enums_are_reduced_to_values() -> None:
|
||||
class _Mode(str, Enum):
|
||||
APPEND = "append_audio_chunk"
|
||||
|
||||
redacted = _redact({"mode": _Mode.APPEND})
|
||||
json.dumps(redacted)
|
||||
assert redacted["mode"] == "append_audio_chunk"
|
||||
|
||||
|
||||
def test_redaction_still_drops_heavy_payload_keys() -> None:
|
||||
"""The pre-existing contract must be preserved."""
|
||||
redacted = _redact(
|
||||
{
|
||||
"ok": True,
|
||||
"stage_handoff": object(),
|
||||
"tts_handoff": object(),
|
||||
"omni_payload": object(),
|
||||
"tts_hidden_states": object(),
|
||||
"tts_token_ids": object(),
|
||||
"traceback": "long trace",
|
||||
}
|
||||
)
|
||||
assert redacted == {"ok": True}
|
||||
|
||||
|
||||
def test_plain_values_pass_through_unchanged() -> None:
|
||||
payload = {"ok": True, "count": 3, "name": "x", "missing": None, "items": [1, "two"]}
|
||||
assert _redact(payload) == payload
|
||||
|
||||
|
||||
def test_dataclass_types_are_not_treated_as_instances() -> None:
|
||||
"""A class object is not a dataclass instance and must not be asdict'd."""
|
||||
|
||||
@dataclass
|
||||
class _Shape:
|
||||
x: int = 1
|
||||
|
||||
assert _redact(_Shape) is _Shape
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.websocket import (
|
||||
DuplexWebSocketActor,
|
||||
normalize_duplex_input_event,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self) -> None:
|
||||
self.sent: list[dict[str, object]] = []
|
||||
|
||||
async def send_json(self, payload: dict[str, object]) -> None:
|
||||
self.sent.append(dict(payload))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("event", "expected"),
|
||||
[
|
||||
({"type": "signal_turn", "event": "barge_in"}, {"type": "turn.signal", "event": "barge_in"}),
|
||||
({"type": "close_session"}, {"type": "session.close"}),
|
||||
({"type": "audio.playback_ack", "played_ms": 1}, {"type": "playback.ack", "played_ms": 1}),
|
||||
({"type": "input_text.append", "text": "a"}, {"type": "input.text.append", "text": "a"}),
|
||||
({"type": "push_text", "text": "b"}, {"type": "input.text.append", "text": "b"}),
|
||||
(
|
||||
{"type": "input.audio.append", "audio": "wav"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "wav"},
|
||||
),
|
||||
(
|
||||
{"type": "push_chunk", "audio": "wav", "format": "pcm_f32le"},
|
||||
{"type": "input_audio_buffer.append", "audio": "wav", "format": "pcm_f32le"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_input_aliases_normalize_once_at_mailbox_boundary(event, expected):
|
||||
assert normalize_duplex_input_event(event) == expected
|
||||
|
||||
|
||||
def test_actor_uses_one_fifo_mailbox_for_inbound_events():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
|
||||
assert isinstance(actor.mailbox, asyncio.Queue)
|
||||
assert not isinstance(actor.mailbox, asyncio.PriorityQueue)
|
||||
assert not hasattr(actor, "control_queue")
|
||||
assert not hasattr(actor, "input_queue")
|
||||
assert not hasattr(actor, "event_queue")
|
||||
assert not hasattr(actor, "lifecycle_state")
|
||||
assert not hasattr(actor, "last_response_id")
|
||||
assert not hasattr(actor, "overlap_speech_ms")
|
||||
assert not hasattr(actor, "transition")
|
||||
assert not hasattr(actor, "output_generation_in_flight")
|
||||
assert not hasattr(actor, "session")
|
||||
assert not hasattr(actor, "runtime_opened")
|
||||
assert not hasattr(actor, "runtime_closed")
|
||||
assert not hasattr(actor, "drain_input_queue")
|
||||
assert not hasattr(actor, "control_events_seen")
|
||||
assert not hasattr(actor, "input_events_seen")
|
||||
assert not hasattr(actor, "cancel_count")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_event_does_not_overtake_earlier_audio_input():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": "response.cancel"})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": "response.cancel"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("event_type", ["input.cancel", "session.close", "close_session"])
|
||||
async def test_terminal_control_preserves_wire_order(event_type: str):
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
await actor.enqueue_event({"type": "input_audio_buffer.append", "audio": "pcm"})
|
||||
await actor.enqueue_event({"type": event_type})
|
||||
|
||||
assert await actor.next_event() == {"type": "input_audio_buffer.append", "audio": "pcm"}
|
||||
assert await actor.next_event() == {"type": event_type}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mailbox_delivers_every_enqueued_event_exactly_once():
|
||||
actor = DuplexWebSocketActor(FakeWebSocket())
|
||||
events = [
|
||||
{"type": "input_audio_buffer.append", "audio": str(index)}
|
||||
if index % 2
|
||||
else {"type": "response.created", "response_id": f"resp-{index}"}
|
||||
for index in range(20)
|
||||
]
|
||||
|
||||
await asyncio.gather(*(actor.enqueue_event(dict(event)) for event in events))
|
||||
received = [await actor.next_event() for _ in events]
|
||||
|
||||
assert sorted(received, key=repr) == sorted(events, key=repr)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_is_single_owner_of_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "one"})
|
||||
await actor.send_json({"type": "two"})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "one"}, {"type": "two"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_fence_payload_is_dropped_before_websocket_send():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 2)
|
||||
writer = asyncio.create_task(actor.writer_loop())
|
||||
|
||||
await actor.send_json({"type": "response.output_audio.delta", "epoch": 1})
|
||||
await actor.send_json({"type": "response.output_audio.delta", "epoch": 2})
|
||||
await actor.close_writer()
|
||||
await writer
|
||||
|
||||
assert websocket.sent == [{"type": "response.output_audio.delta", "epoch": 2}]
|
||||
assert actor.stale_output_dropped == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writer_does_not_revoke_accepted_terminal_after_close_starts():
|
||||
websocket = FakeWebSocket()
|
||||
actor = DuplexWebSocketActor(websocket, current_epoch=lambda: 1)
|
||||
await actor.send_json({"type": "response.done", "epoch": 1, "response_id": "resp-1"})
|
||||
actor.closing = True
|
||||
await actor.close_writer()
|
||||
|
||||
await actor.writer_loop()
|
||||
|
||||
assert websocket.sent == [{"type": "response.done", "epoch": 1, "response_id": "resp-1"}]
|
||||
assert actor.stale_output_dropped == 0
|
||||
@@ -1,601 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
DuplexSessionRegistry,
|
||||
DuplexTurnController,
|
||||
DuplexTurnEventType,
|
||||
DuplexTurnState,
|
||||
ResponseCreateOptions,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.realtime_session import (
|
||||
NativeRealtimeSessionProtocol,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.realtime_state import RealtimeStateOwner
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.capabilities import (
|
||||
minicpmo45_native_capabilities,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_realtime_state_owner_uses_explicit_bindings_not_dynamic_attribute_proxy():
|
||||
assert "__getattr__" not in RealtimeStateOwner.__dict__
|
||||
assert "__setattr__" not in RealtimeStateOwner.__dict__
|
||||
|
||||
|
||||
class _ProtocolWebSocket:
|
||||
def __init__(self, *events: dict[str, object]) -> None:
|
||||
self._events = asyncio.Queue()
|
||||
for event in events:
|
||||
self._events.put_nowait(json.dumps(event))
|
||||
|
||||
async def receive_text(self) -> str:
|
||||
return await self._events.get()
|
||||
|
||||
|
||||
def test_duplex_session_commits_text_and_audio_as_one_turn():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create(DuplexSessionConfig(model="test-model"))
|
||||
|
||||
session.append_text("hello")
|
||||
session.append_audio("YWJj", fmt="wav", sample_rate_hz=16000)
|
||||
committed = session.commit_user_input()
|
||||
|
||||
assert committed is not None
|
||||
assert committed.turn_id == 0
|
||||
assert committed.input_commit_seq == 1
|
||||
assert committed.epoch == 0
|
||||
assert len(session.history) == 1
|
||||
content = session.history[0]["content"]
|
||||
assert isinstance(content, list)
|
||||
assert content[0] == {"type": "text", "text": "hello"}
|
||||
assert content[1]["type"] == "audio_url"
|
||||
assert content[1]["audio_url"]["url"] == "data:audio/wav;base64,YWJj"
|
||||
assert session.turn_state == DuplexTurnState.USER_COMMITTED
|
||||
|
||||
|
||||
def test_response_options_apply_to_one_response_without_mutating_session_defaults():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-options",
|
||||
config=DuplexSessionConfig(instructions="base", voice="base-voice", max_tokens=64),
|
||||
)
|
||||
session.reserve_response_options(
|
||||
ResponseCreateOptions(
|
||||
instructions="one response",
|
||||
voice="override-voice",
|
||||
max_tokens=8,
|
||||
)
|
||||
)
|
||||
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
session.begin_response()
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.response_config.instructions == "one response"
|
||||
assert session.response_config.voice == "override-voice"
|
||||
assert session.response_config.max_tokens == 8
|
||||
|
||||
session.end_response()
|
||||
assert session.config.instructions == "base"
|
||||
assert session.config.voice == "base-voice"
|
||||
assert session.config.max_tokens == 64
|
||||
|
||||
|
||||
def test_response_options_cannot_overwrite_an_unconsumed_reservation():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-options-pending",
|
||||
config=DuplexSessionConfig(instructions="base"),
|
||||
)
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="first"))
|
||||
|
||||
with pytest.raises(RuntimeError, match="already reserved"):
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="second"))
|
||||
|
||||
session.begin_response()
|
||||
assert session.response_config.instructions == "first"
|
||||
|
||||
|
||||
def test_response_options_cannot_be_reserved_while_response_is_active():
|
||||
session = DuplexSession(
|
||||
session_id="sid-response-options-active",
|
||||
config=DuplexSessionConfig(instructions="base"),
|
||||
)
|
||||
session.begin_response()
|
||||
|
||||
with pytest.raises(RuntimeError, match="active"):
|
||||
session.reserve_response_options(ResponseCreateOptions(instructions="too late"))
|
||||
|
||||
|
||||
def test_realtime_model_name_does_not_implicitly_enable_native_duplex():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime({"model": "openbmb/MiniCPM-o-4_5"})
|
||||
|
||||
assert "native_duplex" not in event["session"]["extra_body"]
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_explicit_native_duplex_flag_is_preserved():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_body": {"native_duplex": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
|
||||
|
||||
def test_realtime_legacy_native_duplex_alias_is_normalized():
|
||||
# The deprecated model-prefixed spelling still opts in, but everything
|
||||
# downstream (and every session echo) sees only the canonical key.
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
event = protocol._session_create_from_realtime(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
}
|
||||
)
|
||||
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
def test_realtime_explicit_query_native_duplex_flag_is_available_before_autostart():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
|
||||
|
||||
assert event["type"] == "session.create"
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
|
||||
|
||||
def test_realtime_legacy_query_native_duplex_alias_seeds_canonical_key():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"minicpmo45_native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
event = json.loads(asyncio.run(protocol.receive_internal_event_text(None)))
|
||||
|
||||
assert event["type"] == "session.create"
|
||||
assert event["session"]["extra_body"]["native_duplex"] is True
|
||||
assert "minicpmo45_native_duplex" not in event["session"]["extra_body"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_resume_heartbeat_and_event_ack_translate_without_session_autostart():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
}
|
||||
)
|
||||
|
||||
resume = await protocol._to_duplex_event(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
)
|
||||
heartbeat = await protocol._to_duplex_event(
|
||||
{
|
||||
"type": "session.heartbeat",
|
||||
"event_id": "event-heartbeat",
|
||||
}
|
||||
)
|
||||
event_ack = await protocol._to_duplex_event(
|
||||
{
|
||||
"type": "session.event_ack",
|
||||
"server_event_seq": 19,
|
||||
}
|
||||
)
|
||||
|
||||
assert resume == {
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
assert heartbeat == {
|
||||
"type": "session.heartbeat",
|
||||
"event_id": "event-heartbeat",
|
||||
}
|
||||
assert event_ack == {
|
||||
"type": "session.event_ack",
|
||||
"server_event_seq": 19,
|
||||
}
|
||||
assert protocol._opened is False
|
||||
|
||||
wire_protocol = NativeRealtimeSessionProtocol({})
|
||||
first_wire_event = json.loads(
|
||||
await wire_protocol.receive_internal_event_text(
|
||||
_ProtocolWebSocket(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert first_wire_event["type"] == "session.resume"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_realtime_resume_query_suppresses_model_autostart_on_same_url():
|
||||
protocol = NativeRealtimeSessionProtocol(
|
||||
{
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"native_duplex": "1",
|
||||
"resume": "1",
|
||||
}
|
||||
)
|
||||
resume = {
|
||||
"type": "session.resume",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"resume_token": "opaque-token",
|
||||
"last_received_server_event_seq": 17,
|
||||
}
|
||||
|
||||
first_wire_event = json.loads(await protocol.receive_internal_event_text(_ProtocolWebSocket(resume)))
|
||||
|
||||
assert first_wire_event == resume
|
||||
assert protocol._autostarted_default_session is False
|
||||
|
||||
|
||||
def test_realtime_projects_resume_lifecycle_events_without_duplex_prefix():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
|
||||
created = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.created",
|
||||
"session": {"id": "sid-resume"},
|
||||
"incarnation": 3,
|
||||
"attachment_generation": 1,
|
||||
"resume_token": "first-token",
|
||||
}
|
||||
)[0]
|
||||
resumed = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.resumed",
|
||||
"session_id": "sid-resume",
|
||||
"incarnation": 3,
|
||||
"attachment_generation": 2,
|
||||
"resume_token": "second-token",
|
||||
}
|
||||
)[0]
|
||||
heartbeat = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.heartbeat_ack",
|
||||
"session_id": "sid-resume",
|
||||
}
|
||||
)[0]
|
||||
replaced = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.replaced",
|
||||
"session_id": "sid-resume",
|
||||
"attachment_generation": 1,
|
||||
}
|
||||
)[0]
|
||||
resync = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "session.resync_required",
|
||||
"session_id": "sid-resume",
|
||||
"reason": "journal_gap",
|
||||
}
|
||||
)[0]
|
||||
|
||||
assert created["type"] == "session.created"
|
||||
assert created["incarnation"] == 3
|
||||
assert created["attachment_generation"] == 1
|
||||
assert created["resume_token"] == "first-token"
|
||||
assert resumed["type"] == "session.resumed"
|
||||
assert resumed["resume_token"] == "second-token"
|
||||
assert heartbeat["type"] == "session.heartbeat_ack"
|
||||
assert replaced["type"] == "session.replaced"
|
||||
assert resync == {
|
||||
"type": "session.resync_required",
|
||||
"session_id": "sid-resume",
|
||||
"reason": "journal_gap",
|
||||
"event_id": resync["event_id"],
|
||||
}
|
||||
|
||||
|
||||
def test_realtime_projects_listen_response_identity():
|
||||
protocol = NativeRealtimeSessionProtocol({})
|
||||
protocol.encode_outbound_event({"type": "session.created", "session": {"id": "sid-listen"}})
|
||||
|
||||
bound = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sid-listen",
|
||||
"epoch": 0,
|
||||
"reason": "model_listen",
|
||||
"model_listen": True,
|
||||
"response_id": "resp_1",
|
||||
}
|
||||
)[0]
|
||||
# A listen that terminates a precreated response keeps the response
|
||||
# identity on the Realtime dialect; clients demultiplex on response_id.
|
||||
assert bound["type"] == "response.listen"
|
||||
assert bound["response_id"] == "resp_1"
|
||||
assert bound["response"]["id"] == "resp_1"
|
||||
assert bound["response"]["status"] == "listening"
|
||||
|
||||
anonymous = protocol.encode_outbound_event(
|
||||
{
|
||||
"type": "response.listen",
|
||||
"session_id": "sid-listen",
|
||||
"epoch": 0,
|
||||
"reason": "silence_or_noise",
|
||||
}
|
||||
)[0]
|
||||
assert "response_id" not in anonymous
|
||||
assert "id" not in anonymous["response"]
|
||||
|
||||
|
||||
def test_duplex_session_registry_advances_incarnation_when_id_is_reused():
|
||||
registry = DuplexSessionRegistry()
|
||||
|
||||
first = registry.create(session_id="reused-session")
|
||||
registry.close(first.session_id)
|
||||
second = registry.create(session_id="reused-session")
|
||||
|
||||
assert first.incarnation == 0
|
||||
assert second.incarnation == 1
|
||||
|
||||
|
||||
def test_native_input_commit_does_not_advance_model_turn_identity():
|
||||
session = DuplexSession(
|
||||
session_id="sid-model-owned-turn",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
|
||||
first = session.commit_native_audio_input(transcript="first chunk")
|
||||
second = session.commit_native_audio_input(transcript="second chunk")
|
||||
|
||||
assert session.input_commit_seq == 2
|
||||
assert first.input_commit_seq == 1
|
||||
assert second.input_commit_seq == 2
|
||||
assert first.turn_id == second.turn_id == session.turn_id == 0
|
||||
|
||||
|
||||
def test_duplex_session_owns_response_and_overlap_identity():
|
||||
session = DuplexSession(session_id="sid-state-owner", config=DuplexSessionConfig())
|
||||
|
||||
response_id = session.begin_response()
|
||||
session.accumulate_overlap_speech(320)
|
||||
session.accumulate_overlap_speech(180)
|
||||
session.end_response()
|
||||
|
||||
assert session.active_response_id is None
|
||||
assert session.last_response_id == response_id
|
||||
assert session.overlap_speech_ms == 500
|
||||
assert session.reset_overlap_speech() == 500
|
||||
assert session.overlap_speech_ms == 0
|
||||
|
||||
|
||||
def test_duplex_session_composes_single_owner_ledgers():
|
||||
session = DuplexSession(session_id="sid-ledgers", config=DuplexSessionConfig())
|
||||
|
||||
session.append_text("hello")
|
||||
session.bind_request("req-1")
|
||||
response_id = session.begin_response(turn_id=3)
|
||||
session.mark_audio_sent(duration_ms=240)
|
||||
|
||||
assert not hasattr(session, "input")
|
||||
assert not hasattr(session, "response")
|
||||
assert not hasattr(session, "playback_ledger")
|
||||
assert not hasattr(session, "conversation")
|
||||
assert session.pending_text == ("hello",)
|
||||
assert session.active_request_id == "req-1"
|
||||
assert session.active_response_id == response_id
|
||||
assert session.active_response_turn_id == 3
|
||||
assert session.playback.sent_ms == 240
|
||||
|
||||
try:
|
||||
session.playback.sent_ms = 480
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("playback view must be immutable")
|
||||
|
||||
|
||||
def test_duplex_barge_in_advances_epoch_and_drops_uncommitted_assistant_text():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
response_id = session.begin_response()
|
||||
session.bind_request("chatcmpl-duplex-test")
|
||||
session.append_assistant_text("unplayed answer")
|
||||
|
||||
new_epoch = session.barge_in()
|
||||
|
||||
assert response_id is not None
|
||||
assert new_epoch == 1
|
||||
assert session.epoch == 1
|
||||
assert session.active_request_id is None
|
||||
assert session.active_response_id is None
|
||||
assert session.assistant_text_buffer == ()
|
||||
assert session.history == ()
|
||||
assert session.turn_state == DuplexTurnState.BARGE_IN
|
||||
|
||||
|
||||
def test_duplex_playback_ack_tracks_committed_cursor_separately():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
session.mark_audio_sent(duration_ms=10_000)
|
||||
|
||||
session.acknowledge_playback(played_ms=2_000)
|
||||
|
||||
assert session.playback.generated_ms == 10_000
|
||||
assert session.playback.sent_ms == 10_000
|
||||
assert session.playback.played_ms == 2_000
|
||||
assert session.playback.committed_ms == 2_000
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sample_rate_hz,frame_count", [(22_050, 23), (24_000, 24)])
|
||||
def test_duplex_generated_audio_rounds_only_cumulative_frames(sample_rate_hz: int, frame_count: int):
|
||||
session = DuplexSessionRegistry().create()
|
||||
response_id = session.begin_response()
|
||||
|
||||
durations = [
|
||||
session.record_generated_audio(response_id, frame_count=1, sample_rate_hz=sample_rate_hz)
|
||||
for _ in range(frame_count)
|
||||
]
|
||||
|
||||
assert durations[:-1] == [0] * (frame_count - 1)
|
||||
assert durations[-1] == 1
|
||||
assert session.playback.generated_ms == 1
|
||||
assert session.playback.sent_ms == 0
|
||||
assert session.playback.played_ms == 0
|
||||
|
||||
|
||||
def test_duplex_generated_audio_rejects_rate_change_without_advancing():
|
||||
session = DuplexSessionRegistry().create()
|
||||
response_id = session.begin_response()
|
||||
session.record_generated_audio(response_id, frame_count=2205, sample_rate_hz=22_050)
|
||||
|
||||
with pytest.raises(ValueError, match="sample rate changed"):
|
||||
session.record_generated_audio(response_id, frame_count=2400, sample_rate_hz=24_000)
|
||||
|
||||
assert session.playback.generated_ms == 100
|
||||
assert session.playback.sent_ms == 0
|
||||
|
||||
|
||||
def test_duplex_late_audio_accounting_cannot_change_new_response():
|
||||
session = DuplexSessionRegistry().create()
|
||||
old_response = session.begin_response()
|
||||
session.record_generated_audio(old_response, frame_count=2205, sample_rate_hz=22_050)
|
||||
session.barge_in()
|
||||
new_response = session.begin_response()
|
||||
session.record_generated_audio(new_response, frame_count=2400, sample_rate_hz=48_000)
|
||||
current_state = session.turn_state
|
||||
|
||||
assert session.record_generated_audio(old_response, frame_count=2205, sample_rate_hz=22_050) is None
|
||||
session.mark_audio_sent(100, response_id=old_response, text_chars=99)
|
||||
|
||||
assert session.playback_for_response(old_response).sent_ms == 100
|
||||
assert session.playback.generated_ms == 50
|
||||
assert session.playback.sent_ms == 0
|
||||
assert session.assistant_audio_text_marks == ()
|
||||
assert session.turn_state == current_state
|
||||
|
||||
|
||||
def test_duplex_history_commit_uses_audio_text_alignment_marks():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
session.begin_response()
|
||||
|
||||
session.append_assistant_text("hello ")
|
||||
session.mark_audio_sent(duration_ms=1_000, text_chars=6)
|
||||
session.append_assistant_text("world")
|
||||
session.mark_audio_sent(duration_ms=2_000, text_chars=11)
|
||||
session.acknowledge_playback(played_ms=1_200, committed_ms=1_200)
|
||||
|
||||
committed = session.end_response(commit_text=True)
|
||||
|
||||
assert committed == {"role": "assistant", "content": "hello w"}
|
||||
|
||||
|
||||
def test_turn_controller_accepts_external_signal_source():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
controller = DuplexTurnController()
|
||||
|
||||
event = controller.signal(session, DuplexTurnEventType.USER_STARTED.value)
|
||||
|
||||
assert event["type"] == "turn.event"
|
||||
assert event["event"] == "user_started"
|
||||
assert event["turn_state"] == "user_speaking"
|
||||
assert session.turn_state == DuplexTurnState.USER_SPEAKING
|
||||
|
||||
|
||||
def test_duplex_capabilities_do_not_claim_core_kv_or_input_append():
|
||||
registry = DuplexSessionRegistry()
|
||||
session = registry.create()
|
||||
caps = session.capabilities.as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "serving_session_adapter"
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_input_append"] is False
|
||||
assert caps["supports_reencode_context"] is True
|
||||
assert caps["adapter_patterns"] == ["chunk_group_append"]
|
||||
assert "turn_commit_only" in caps["input_modes"]
|
||||
assert "client_event" in caps["signal_sources"]
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_separate_model_state_from_core_kv_lease():
|
||||
caps = minicpmo45_native_capabilities(max_sessions=2).as_dict()
|
||||
|
||||
assert caps["implementation_level"] == "model_native_duplex"
|
||||
assert caps["supports_input_append"] is True
|
||||
assert caps["input_modes"] == ["append_audio_chunk"]
|
||||
assert caps["adapter_patterns"] == ["scheduler_data_plane"]
|
||||
assert caps["supports_model_internal_state"] is True
|
||||
assert caps["requires_model_runner_kv"] is True
|
||||
assert caps["requires_native_stage_role"] is True
|
||||
assert caps["supports_kv_lease"] is False
|
||||
assert caps["supports_core_kv_lease"] is False
|
||||
assert caps["supports_stage_resumption"] is True
|
||||
assert caps["supports_scheduler_native_append"] is False
|
||||
assert caps["supports_core_resumable_request"] is True
|
||||
assert caps["supports_stage_connector_handoff"] is True
|
||||
assert caps["supports_audio_truncate"] is True
|
||||
assert caps["supports_barge_in"] is True
|
||||
assert caps["target_barge_in_latency_ms"] is None
|
||||
assert caps["supports_multi_session"] is True
|
||||
assert caps["supports_multi_session_same_replica"] is True
|
||||
assert caps["supports_session_lease"] is True
|
||||
assert caps["supports_session_resume"] is True
|
||||
assert caps["session_admission_mode"] == "engine_managed"
|
||||
assert caps["stage_handoff_transport"] == "scheduler_data_plane"
|
||||
|
||||
|
||||
def test_duplex_overlap_policy_auto_falls_back_to_listen_only():
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexOverlapPolicy,
|
||||
DuplexSessionConfig,
|
||||
)
|
||||
|
||||
assert DuplexSessionConfig._normalize_overlap_policy("auto") == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
|
||||
|
||||
def test_minicpmo_native_capabilities_do_not_overclaim_single_session_deployment():
|
||||
caps = minicpmo45_native_capabilities(max_sessions=1).as_dict()
|
||||
|
||||
assert caps["supports_multi_session"] is False
|
||||
assert caps["supports_multi_session_same_replica"] is False
|
||||
|
||||
|
||||
def test_duplex_overlap_policy_defaults_and_invalid_values_to_listen_only():
|
||||
assert DuplexSessionConfig().overlap_policy == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
assert DuplexSessionConfig._normalize_overlap_policy("not-a-policy") == DuplexOverlapPolicy.LISTEN_ONLY.value
|
||||
@@ -133,7 +133,7 @@ async def test_registry_failed_send_is_accepted_only_when_journaled(mocker, jour
|
||||
assert accepted.call_args_list == expected_calls
|
||||
raise RuntimeError("transport lost")
|
||||
|
||||
await registry.create("sid-send-failure", incarnation=0, send=failing_send, close=mocker.AsyncMock())
|
||||
await registry.create("sid-send-failure", send=failing_send, close=mocker.AsyncMock())
|
||||
|
||||
with pytest.raises(RuntimeError, match="transport lost"):
|
||||
await registry.send_event(
|
||||
@@ -152,7 +152,7 @@ async def test_registry_detached_event_is_accepted_only_when_journaled(mocker, j
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
send = mocker.AsyncMock()
|
||||
accepted = mocker.Mock()
|
||||
created = await registry.create("sid-detached-acceptance", incarnation=0, send=send, close=mocker.AsyncMock())
|
||||
created = await registry.create("sid-detached-acceptance", send=send, close=mocker.AsyncMock())
|
||||
await registry.detach("sid-detached-acceptance", attachment_generation=created.attachment_generation)
|
||||
|
||||
entry = await registry.send_event(
|
||||
@@ -175,7 +175,7 @@ async def test_registry_unjournaled_event_is_accepted_after_successful_send(mock
|
||||
async def send(payload):
|
||||
accepted.assert_not_called()
|
||||
|
||||
await registry.create("sid-live-acceptance", incarnation=0, send=send, close=mocker.AsyncMock())
|
||||
await registry.create("sid-live-acceptance", send=send, close=mocker.AsyncMock())
|
||||
|
||||
entry = await registry.send_event(
|
||||
"sid-live-acceptance",
|
||||
@@ -193,7 +193,7 @@ async def test_registry_overflow_does_not_accept_or_send_event(mocker) -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=100)
|
||||
accepted = mocker.Mock()
|
||||
send = mocker.AsyncMock()
|
||||
await registry.create("sid-overflow-acceptance", incarnation=0, send=send, close=mocker.AsyncMock())
|
||||
await registry.create("sid-overflow-acceptance", send=send, close=mocker.AsyncMock())
|
||||
|
||||
with pytest.raises(DuplexJournalOverflowError, match="byte limit"):
|
||||
await registry.send_event(
|
||||
@@ -231,13 +231,12 @@ async def test_registry_resume_rotates_token_replays_and_atomically_replaces_att
|
||||
async def close_b(reason):
|
||||
closes_b.append(reason)
|
||||
|
||||
created = await registry.create("sid", incarnation=2, send=send_a, close=close_a)
|
||||
created = await registry.create("sid", send=send_a, close=close_a)
|
||||
await registry.send_event("sid", {"type": "event-1"})
|
||||
await registry.send_event("sid", {"type": "event-2"})
|
||||
|
||||
resumed = await registry.resume(
|
||||
"sid",
|
||||
incarnation=2,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=1,
|
||||
send=send_b,
|
||||
@@ -260,6 +259,34 @@ async def test_registry_resume_rotates_token_replays_and_atomically_replaces_att
|
||||
assert closes_a == [] and sends_b == [] and closes_b == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_detach_without_a_generation_targets_the_current_attachment_once() -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
sends: list[dict] = []
|
||||
|
||||
async def send(payload):
|
||||
sends.append(payload)
|
||||
|
||||
async def close(reason):
|
||||
return None
|
||||
|
||||
await registry.create("sid-current", send=send, close=close)
|
||||
|
||||
# The outbound pump outlives any one connection and only knows the session:
|
||||
# ``None`` means "whichever socket is attached right now".
|
||||
assert await registry.detach("sid-current", attachment_generation=None) is True
|
||||
# A second report of the same dead socket is not a new detach, so it cannot
|
||||
# restart the engine's disconnect grace window.
|
||||
assert await registry.detach("sid-current", attachment_generation=None) is False
|
||||
assert await registry.detach("sid-current", attachment_generation=1) is False
|
||||
assert await registry.is_current_attachment("sid-current", 1) is False
|
||||
|
||||
# A detached session still journals for a later resume.
|
||||
await registry.send_event("sid-current", {"type": "event-1"})
|
||||
assert sends == []
|
||||
assert await registry.detach("missing-session", attachment_generation=None) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_resume_sends_activation_then_replay_before_new_live_events() -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(
|
||||
@@ -282,13 +309,12 @@ async def test_registry_resume_sends_activation_then_replay_before_new_live_even
|
||||
activation_started.set()
|
||||
await release_activation.wait()
|
||||
|
||||
created = await registry.create("sid-order", incarnation=0, send=old_send, close=close)
|
||||
created = await registry.create("sid-order", send=old_send, close=close)
|
||||
await registry.detach("sid-order", attachment_generation=1)
|
||||
await registry.send_event("sid-order", {"type": "replayed"})
|
||||
resume_task = asyncio.create_task(
|
||||
registry.resume(
|
||||
"sid-order",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=new_send,
|
||||
@@ -322,13 +348,12 @@ async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recover
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-recovery", incarnation=0, send=send, close=close)
|
||||
created = await registry.create("sid-recovery", send=send, close=close)
|
||||
await registry.detach("sid-recovery", attachment_generation=1)
|
||||
|
||||
with pytest.raises(RuntimeError, match="transport lost"):
|
||||
await registry.resume(
|
||||
"sid-recovery",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=failing_send,
|
||||
@@ -342,7 +367,6 @@ async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recover
|
||||
|
||||
recovered = await registry.resume(
|
||||
"sid-recovery",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -353,7 +377,6 @@ async def test_registry_resume_delivery_failure_keeps_one_shot_old_token_recover
|
||||
with pytest.raises(InvalidResumeTokenError):
|
||||
await registry.resume(
|
||||
"sid-recovery",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -374,13 +397,12 @@ async def test_registry_concurrent_resume_allows_exactly_one_rotated_token_winne
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-race", incarnation=0, send=send, close=close)
|
||||
created = await registry.create("sid-race", send=send, close=close)
|
||||
|
||||
async def attempt():
|
||||
try:
|
||||
return await registry.resume(
|
||||
"sid-race",
|
||||
incarnation=0,
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -396,7 +418,7 @@ async def test_registry_concurrent_resume_allows_exactly_one_rotated_token_winne
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> None:
|
||||
async def test_registry_keeps_sessions_journals_and_tokens_isolated() -> None:
|
||||
registry = DuplexSessionAttachmentRegistry(
|
||||
replay_ttl_s=60.0,
|
||||
replay_max_bytes_per_session=4096,
|
||||
@@ -408,8 +430,8 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created_a = await registry.create("sid-a", incarnation=1, send=send, close=close)
|
||||
created_b = await registry.create("sid-b", incarnation=4, send=send, close=close)
|
||||
created_a = await registry.create("sid-a", send=send, close=close)
|
||||
created_b = await registry.create("sid-b", send=send, close=close)
|
||||
await registry.detach("sid-a", attachment_generation=1)
|
||||
await registry.detach("sid-b", attachment_generation=1)
|
||||
await registry.send_event("sid-a", {"type": "a-only"})
|
||||
@@ -417,7 +439,6 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
|
||||
resumed_a = await registry.resume(
|
||||
"sid-a",
|
||||
incarnation=1,
|
||||
resume_token=created_a.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -425,7 +446,6 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
)
|
||||
resumed_b = await registry.resume(
|
||||
"sid-b",
|
||||
incarnation=4,
|
||||
resume_token=created_b.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
@@ -434,11 +454,10 @@ async def test_registry_keeps_sessions_journals_and_incarnations_isolated() -> N
|
||||
|
||||
assert [entry.payload["type"] for entry in resumed_a.replay_entries] == ["a-only"]
|
||||
assert [entry.payload["type"] for entry in resumed_b.replay_entries] == ["b-only"]
|
||||
with pytest.raises(ValueError, match="incarnation mismatch"):
|
||||
with pytest.raises(InvalidResumeTokenError):
|
||||
await registry.resume(
|
||||
"sid-a",
|
||||
incarnation=2,
|
||||
resume_token=resumed_a.resume_token.plaintext,
|
||||
resume_token=resumed_b.resume_token.plaintext,
|
||||
last_received_server_event_seq=1,
|
||||
send=send,
|
||||
close=close,
|
||||
@@ -458,7 +477,7 @@ async def test_registry_repr_never_contains_plaintext_tokens() -> None:
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-repr", incarnation=0, send=send, close=close)
|
||||
created = await registry.create("sid-repr", send=send, close=close)
|
||||
await registry.detach("sid-repr", attachment_generation=1)
|
||||
entry = await registry.send_event(
|
||||
"sid-repr",
|
||||
@@ -469,3 +488,67 @@ async def test_registry_repr_never_contains_plaintext_tokens() -> None:
|
||||
assert created.resume_token.plaintext not in repr(created)
|
||||
assert created.resume_token.plaintext not in repr(entry)
|
||||
assert created.resume_token.plaintext not in repr(registry)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("cancel_on", ["activation", "replay"])
|
||||
async def test_registry_resume_cancelled_mid_delivery_rolls_back_like_a_failure(cancel_on: str) -> None:
|
||||
"""Cancellation must take the same rollback as an ordinary delivery failure.
|
||||
|
||||
``CancelledError`` derives from ``BaseException``, so an ``except
|
||||
Exception`` around the activation and replay sends let a cancelled resume
|
||||
keep its attachment as the current generation while the token it rotated
|
||||
past was already gone — the session left attached to an unusable transport
|
||||
with no way back. Both send points are covered because either can be the
|
||||
one that gets cancelled.
|
||||
"""
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
|
||||
async def send(payload):
|
||||
del payload
|
||||
|
||||
async def close(reason):
|
||||
del reason
|
||||
|
||||
created = await registry.create("sid-cancel", send=send, close=close)
|
||||
# One journaled event, so the replay loop has something to send.
|
||||
await registry.send_event("sid-cancel", {"type": "response.output_text.delta", "delta": "a"})
|
||||
await registry.detach("sid-cancel", attachment_generation=1)
|
||||
|
||||
sends = 0
|
||||
cancel_at = 1 if cancel_on == "activation" else 2
|
||||
|
||||
async def cancelling_send(payload):
|
||||
del payload
|
||||
nonlocal sends
|
||||
sends += 1
|
||||
if sends == cancel_at:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await registry.resume(
|
||||
"sid-cancel",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=cancelling_send,
|
||||
close=close,
|
||||
activation_payload_factory=lambda token, generation: {
|
||||
"type": "session.resumed",
|
||||
"resume_token": token.plaintext,
|
||||
"attachment_generation": generation,
|
||||
},
|
||||
)
|
||||
|
||||
assert sends == cancel_at, "the send that was cancelled is the one under test"
|
||||
# The cancelled attachment must not remain current...
|
||||
assert not await registry.has_attachment("sid-cancel")
|
||||
# ...and the token the caller presented has to work again, or the session
|
||||
# is unreachable: resume() had already rotated past it.
|
||||
recovered = await registry.resume(
|
||||
"sid-cancel",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
close=close,
|
||||
)
|
||||
assert recovered.attachment_generation == 3
|
||||
@@ -920,7 +920,6 @@ async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
|
||||
monkeypatch.setattr(api_server, "create_streaming_video_handler", lambda **_k: _marker("streaming_video"))
|
||||
monkeypatch.setattr(api_server, "OpenAIServingRealtime", _FakeCtor)
|
||||
monkeypatch.setattr(api_server, "OmniOpenAIServingVideo", _FakeCtor)
|
||||
monkeypatch.setattr(api_server, "should_enable_duplex_endpoint", lambda *_a, **_k: False)
|
||||
|
||||
state = State()
|
||||
await api_server.omni_init_app_state(engine, state, _minimal_args())
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Guards for the duplex-only API server (unified full-duplex framework).
|
||||
|
||||
A model whose pipeline declares ``duplex_plugin`` is served through
|
||||
``DuplexOmni`` and nothing else: sessions run over ``/v1/realtime?duplex=1``
|
||||
(alias ``/v1/duplex``) and every turn-based route reports "not available".
|
||||
These tests lock that shape, which ``test_api_server_guards.py`` covers for the
|
||||
diffusion and multi-stage servers:
|
||||
|
||||
* the pipeline probe that decides duplex vs turn-based at startup,
|
||||
* the ``app.state`` snapshot of a duplex server (what is wired, what stays
|
||||
``None``),
|
||||
* both duplex websocket routes reaching the session handler, and
|
||||
``/v1/realtime`` without the flag still reporting the turn-based Realtime API
|
||||
as unavailable,
|
||||
* the startup-warmup gate that holds real clients while the warmup session runs.
|
||||
|
||||
CPU-only: no engine is started; ``DuplexOmni`` is subclassed with just the
|
||||
accessors the app-state wiring reads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.datastructures import State
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
|
||||
from vllm_omni.config.config_factory import StageConfigFactory
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
from vllm_omni.entrypoints.openai import api_server
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# ``app.state`` of a duplex-only server. Presence is not enough: a route whose
|
||||
# handler is missing answers "not available", so "not wired" is written as None.
|
||||
_DUPLEX_APP_STATE_KEYS = {
|
||||
"engine_client",
|
||||
"log_stats",
|
||||
"args",
|
||||
"sleeping_stages",
|
||||
"stage_configs",
|
||||
"vllm_config",
|
||||
"diffusion_engine",
|
||||
"openai_serving_models",
|
||||
"serving_tokenization",
|
||||
"serving_tokens",
|
||||
"online_renderer",
|
||||
"openai_serving_chat",
|
||||
"openai_serving_chat_batch",
|
||||
"openai_serving_completion",
|
||||
"openai_serving_responses",
|
||||
"openai_serving_embedding",
|
||||
"openai_serving_pooling",
|
||||
"openai_serving_classification",
|
||||
"openai_serving_scores",
|
||||
"openai_serving_transcription",
|
||||
"openai_serving_translation",
|
||||
"openai_serving_speech",
|
||||
"openai_serving_audio_generate",
|
||||
"openai_serving_video",
|
||||
"openai_streaming_speech",
|
||||
"openai_streaming_video",
|
||||
"openai_streaming_video_output",
|
||||
"openai_serving_realtime",
|
||||
"openai_serving_realtime_robot",
|
||||
"anthropic_serving_messages",
|
||||
"openai_serving_duplex",
|
||||
"enable_server_load_tracking",
|
||||
"server_load_metrics",
|
||||
}
|
||||
#: Every turn-based service, plus the Realtime route that is not the duplex one.
|
||||
#: ``openai_serving_chat`` is absent: a duplex engine also serves chat, through
|
||||
#: the ordinary turn-based service, because DuplexOmni extends AsyncOmni.
|
||||
#: ``online_renderer`` is absent for the same reason -- the chat service needs it.
|
||||
_DUPLEX_MUST_BE_NONE = _DUPLEX_APP_STATE_KEYS - {
|
||||
"engine_client",
|
||||
"log_stats",
|
||||
"args",
|
||||
"sleeping_stages",
|
||||
"stage_configs",
|
||||
"vllm_config",
|
||||
"openai_serving_models",
|
||||
"openai_serving_duplex",
|
||||
"openai_serving_chat",
|
||||
"online_renderer",
|
||||
"enable_server_load_tracking",
|
||||
"server_load_metrics",
|
||||
}
|
||||
_DUPLEX_MUST_BE_WIRED = {
|
||||
"engine_client",
|
||||
"args",
|
||||
"stage_configs",
|
||||
"vllm_config",
|
||||
"openai_serving_models",
|
||||
"openai_serving_duplex",
|
||||
"openai_serving_chat",
|
||||
}
|
||||
|
||||
|
||||
class _FakeDuplexOmni(DuplexOmni):
|
||||
"""A ``DuplexOmni`` without an engine: only what the app-state wiring reads."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.model = "demo-duplex-model"
|
||||
self._stage_configs = [object(), object(), object()]
|
||||
self._vllm_config = SimpleNamespace(
|
||||
lora_config=None,
|
||||
model_config=SimpleNamespace(),
|
||||
parallel_config=SimpleNamespace(_api_process_rank=0),
|
||||
)
|
||||
self._duplex_session_config = DuplexSessionRuntimeConfig()
|
||||
self._capabilities = DuplexCapabilities(supports_chat_completions=True)
|
||||
|
||||
async def get_vllm_config(self):
|
||||
return self._vllm_config
|
||||
|
||||
async def get_supported_tasks(self) -> tuple[str, ...]:
|
||||
return ("generate",)
|
||||
|
||||
async def get_tokenizer(self):
|
||||
return SimpleNamespace(chat_template="{{ messages }}")
|
||||
|
||||
@property
|
||||
def stage_configs(self) -> list[object]:
|
||||
return self._stage_configs
|
||||
|
||||
@property
|
||||
def duplex_session_config(self) -> DuplexSessionRuntimeConfig:
|
||||
return self._duplex_session_config
|
||||
|
||||
@property
|
||||
def duplex_capabilities(self) -> DuplexCapabilities:
|
||||
return self._capabilities
|
||||
|
||||
# Properties on AsyncOmni, so the fake overrides rather than assigns them.
|
||||
@property
|
||||
def model_config(self) -> SimpleNamespace:
|
||||
return SimpleNamespace()
|
||||
|
||||
@property
|
||||
def renderer(self) -> SimpleNamespace:
|
||||
return SimpleNamespace()
|
||||
|
||||
|
||||
class _FakeModels:
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
self.base_model_paths = kwargs.get("base_model_paths") or []
|
||||
|
||||
|
||||
def _minimal_args(**overrides) -> SimpleNamespace:
|
||||
args = SimpleNamespace(
|
||||
model="demo-duplex-model",
|
||||
served_model_name=None,
|
||||
disable_log_stats=True,
|
||||
enable_log_requests=False,
|
||||
max_log_len=None,
|
||||
enable_server_load_tracking=False,
|
||||
trust_remote_code=True,
|
||||
deploy_config=None,
|
||||
# What the chat service reads when a duplex model also serves chat.
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
trust_request_chat_template=False,
|
||||
default_chat_template_kwargs=None,
|
||||
response_role="assistant",
|
||||
return_tokens_as_token_ids=False,
|
||||
enable_auto_tool_choice=False,
|
||||
exclude_tools_when_tool_choice_none=False,
|
||||
tool_call_parser=None,
|
||||
structured_outputs_config=SimpleNamespace(reasoning_parser=None),
|
||||
enable_prompt_tokens_details=False,
|
||||
enable_force_include_usage=False,
|
||||
enable_log_outputs=False,
|
||||
enable_log_deltas=False,
|
||||
)
|
||||
for key, value in overrides.items():
|
||||
setattr(args, key, value)
|
||||
return args
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Startup: which entrypoint class serves the model #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pipeline_config", "expected"),
|
||||
[
|
||||
(SimpleNamespace(duplex_plugin="pkg.mod.Plugin"), True),
|
||||
(SimpleNamespace(duplex_plugin=None), False),
|
||||
(SimpleNamespace(duplex_plugin=""), False),
|
||||
(SimpleNamespace(), False),
|
||||
(None, False),
|
||||
],
|
||||
)
|
||||
def test_duplex_model_probe_follows_the_pipeline_plugin(monkeypatch, pipeline_config, expected: bool) -> None:
|
||||
"""``duplex_plugin`` alone decides whether ``serve`` builds ``DuplexOmni``."""
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
def fake_get_pipeline_config(*, model, trust_remote_code, deploy_config_path=None, **_kwargs):
|
||||
seen.update(model=model, trust_remote_code=trust_remote_code, deploy_config_path=deploy_config_path)
|
||||
return pipeline_config
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", fake_get_pipeline_config)
|
||||
|
||||
kwargs = {"trust_remote_code": True, "deploy_config": "deploy.yaml"}
|
||||
assert api_server._is_duplex_model("demo-duplex-model", kwargs) is expected
|
||||
assert seen == {
|
||||
"model": "demo-duplex-model",
|
||||
"trust_remote_code": True,
|
||||
"deploy_config_path": "deploy.yaml",
|
||||
}
|
||||
|
||||
|
||||
def test_duplex_model_probe_propagates_resolution_errors(monkeypatch) -> None:
|
||||
"""A pipeline that cannot be resolved must fail startup, not fall back to turn-based."""
|
||||
|
||||
def fake_get_pipeline_config(**_kwargs):
|
||||
raise ValueError("needs trust_remote_code=True")
|
||||
|
||||
monkeypatch.setattr(StageConfigFactory, "get_pipeline_config", fake_get_pipeline_config)
|
||||
|
||||
with pytest.raises(ValueError, match="trust_remote_code"):
|
||||
api_server._is_duplex_model("demo-duplex-model", {})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# app.state of a duplex-only server #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_app_state_wires_only_the_session_surfaces(monkeypatch) -> None:
|
||||
"""Lock the duplex ``app.state``: the two session-backed surfaces are live, every turn-based service is None.
|
||||
|
||||
Fails if a turn-based service is wired into a duplex server (its route
|
||||
would answer instead of reporting "not available"), or if a key the routes
|
||||
read disappears. ``openai_serving_chat`` is the one shared name: the route
|
||||
is served, but by the adapter that runs it on a duplex session, never by
|
||||
the turn-based chat service.
|
||||
"""
|
||||
monkeypatch.setattr(api_server, "OpenAIServingModels", _FakeModels)
|
||||
monkeypatch.setattr(api_server, "OnlineRenderer", lambda **kwargs: SimpleNamespace(**kwargs))
|
||||
monkeypatch.setattr(api_server, "OmniOpenAIServingChat", lambda **kwargs: SimpleNamespace(**kwargs))
|
||||
engine = _FakeDuplexOmni()
|
||||
state = State()
|
||||
|
||||
await api_server.omni_init_app_state(engine, state, _minimal_args())
|
||||
|
||||
present = {key for key in _DUPLEX_APP_STATE_KEYS if hasattr(state, key)}
|
||||
assert present == _DUPLEX_APP_STATE_KEYS
|
||||
not_wired = sorted(key for key in _DUPLEX_MUST_BE_WIRED if getattr(state, key) is None)
|
||||
assert not not_wired, f"duplex app.state keys registered but not wired: {not_wired}"
|
||||
unexpectedly_set = sorted(key for key in _DUPLEX_MUST_BE_NONE if getattr(state, key) is not None)
|
||||
assert not unexpectedly_set, f"turn-based services wired into a duplex server: {unexpectedly_set}"
|
||||
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
|
||||
assert state.openai_serving_chat is not None
|
||||
assert state.engine_client is engine
|
||||
assert state.vllm_config is engine._vllm_config
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Routes #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _RecordingHandler:
|
||||
"""Stands in for ``OmniDuplexSessionHandler`` on the app state."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.queries: list[dict[str, str]] = []
|
||||
|
||||
async def handle_realtime_session(self, websocket) -> None:
|
||||
await websocket.accept()
|
||||
self.queries.append(dict(websocket.query_params))
|
||||
await websocket.send_json({"type": "session.created", "session": {"id": "duplex-test"}})
|
||||
await websocket.close()
|
||||
|
||||
|
||||
def _duplex_app(handler: object) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.include_router(api_server.router)
|
||||
app.state.openai_serving_duplex = handler
|
||||
# A duplex server serves no turn-based Realtime route.
|
||||
app.state.openai_serving_realtime = None
|
||||
return app
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/duplex", "/v1/realtime?duplex=1"])
|
||||
def test_both_duplex_routes_reach_the_session_handler(path: str) -> None:
|
||||
"""``/v1/duplex`` is an alias of ``/v1/realtime?duplex=1``: same handler, same protocol."""
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(path) as websocket:
|
||||
assert websocket.receive_json()["type"] == "session.created"
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert len(handler.queries) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["1", "true", "on"])
|
||||
def test_realtime_route_switches_to_duplex_only_for_the_documented_flag_values(flag: str) -> None:
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(f"/v1/realtime?duplex={flag}") as websocket:
|
||||
assert websocket.receive_json()["type"] == "session.created"
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert handler.queries == [{"duplex": flag}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query", ["", "?model=openbmb%2FMiniCPM-o-4_5"])
|
||||
def test_realtime_without_a_duplex_flag_reaches_the_session_handler(query: str) -> None:
|
||||
"""A bare connection to a duplex deployment is a duplex session.
|
||||
|
||||
``?duplex=1`` is the spelling the server advertises, but it is not a
|
||||
requirement: a deployment that declares a duplex plugin answers plain
|
||||
``/v1/realtime`` too, so a stock Realtime client needs no vendor query
|
||||
parameter.
|
||||
"""
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(f"/v1/realtime{query}") as websocket:
|
||||
assert websocket.receive_json()["type"] == "session.created"
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert len(handler.queries) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query", ["?duplex=0", "?duplex=false", "?duplex=off"])
|
||||
def test_realtime_opted_out_of_duplex_falls_through_to_the_turn_based_route(query: str) -> None:
|
||||
"""An explicit opt-out selects the legacy handler, which a duplex server does not mount."""
|
||||
handler = _RecordingHandler()
|
||||
|
||||
with TestClient(_duplex_app(handler)) as client:
|
||||
with client.websocket_connect(f"/v1/realtime{query}") as websocket:
|
||||
assert websocket.receive_json() == {
|
||||
"type": "error",
|
||||
"error": "Realtime API is not available",
|
||||
"code": "unsupported",
|
||||
}
|
||||
with pytest.raises(WebSocketDisconnect):
|
||||
websocket.receive_text()
|
||||
|
||||
assert handler.queries == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Startup warmup gate #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _warmup_websocket(warmup_done: asyncio.Event | None, query: dict[str, str]) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
app=SimpleNamespace(state=SimpleNamespace(duplex_warmup_done=warmup_done)),
|
||||
query_params=query,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_warmup_gate_holds_clients_until_the_warmup_session_finishes() -> None:
|
||||
warmup_done = asyncio.Event()
|
||||
websocket = _warmup_websocket(warmup_done, {})
|
||||
|
||||
waiting = asyncio.create_task(api_server._wait_for_duplex_warmup(websocket))
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(asyncio.shield(waiting), timeout=0.05)
|
||||
|
||||
warmup_done.set()
|
||||
await asyncio.wait_for(waiting, timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("warmup_done", "query"),
|
||||
[
|
||||
(None, {}),
|
||||
(asyncio.Event(), {"vllm_omni_warmup": "1"}),
|
||||
],
|
||||
ids=["no-warmup-configured", "the-warmup-connection-itself"],
|
||||
)
|
||||
async def test_warmup_gate_lets_the_warmup_connection_and_plain_servers_through(warmup_done, query) -> None:
|
||||
websocket = _warmup_websocket(warmup_done, query)
|
||||
|
||||
await asyncio.wait_for(api_server._wait_for_duplex_warmup(websocket), timeout=1.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_model_that_does_not_declare_chat_completions_does_not_get_the_route(monkeypatch) -> None:
|
||||
"""The decision stays the model's: no capability, no chat service, and the route says so."""
|
||||
monkeypatch.setattr(api_server, "OpenAIServingModels", _FakeModels)
|
||||
engine = _FakeDuplexOmni()
|
||||
engine._capabilities = DuplexCapabilities(supports_chat_completions=False)
|
||||
state = State()
|
||||
|
||||
await api_server.omni_init_app_state(engine, state, _minimal_args())
|
||||
|
||||
assert state.openai_serving_chat is None
|
||||
assert isinstance(state.openai_serving_duplex, OmniDuplexSessionHandler)
|
||||
File diff suppressed because it is too large.
Load diff
@@ -32,7 +32,7 @@ from tests.helpers.media import (
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.config.stage_config import load_deploy_config
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
from vllm_omni.engine.duplex.vad import (
|
||||
SILERO_VAD_FILENAME,
|
||||
SILERO_VAD_REPO_ID,
|
||||
SILERO_VAD_REVISION,
|
||||
|
||||
@@ -1,429 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
import threading
|
||||
import wave
|
||||
import weakref
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.protocol import (
|
||||
DuplexSession,
|
||||
DuplexSessionConfig,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.server_vad import (
|
||||
ServerVADConfig,
|
||||
ServerVADFrame,
|
||||
ServerVADPipeline,
|
||||
SileroVADBackend,
|
||||
SileroVADBackendProvider,
|
||||
SpeechEndpointDecision,
|
||||
ThresholdEndpointPolicy,
|
||||
parse_session_turn_detection,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class SequenceDetector:
|
||||
frame_samples = 160
|
||||
|
||||
def __init__(self, probabilities: list[float]) -> None:
|
||||
self.probabilities = probabilities
|
||||
|
||||
def new_state(self) -> int:
|
||||
return 0
|
||||
|
||||
def infer(self, frame: np.ndarray, state: object) -> tuple[float, object]:
|
||||
index = int(state)
|
||||
probability = self.probabilities[index] if index < len(self.probabilities) else 0.0
|
||||
return probability, index + 1
|
||||
|
||||
|
||||
def test_server_vad_config_defaults_and_validation():
|
||||
config = ServerVADConfig.from_value({"type": "server_vad"})
|
||||
|
||||
assert config.type == "server_vad"
|
||||
assert config.threshold == 0.5
|
||||
assert config.prefix_padding_ms == 300
|
||||
assert config.silence_duration_ms == 500
|
||||
assert config.create_response is True
|
||||
assert config.interrupt_response is None
|
||||
assert "interrupt_response" not in config.as_dict()
|
||||
assert config.min_speech_duration_ms is None
|
||||
assert config.as_dict()["type"] == config.type
|
||||
|
||||
interrupting = ServerVADConfig.from_value(
|
||||
{
|
||||
"type": "server_vad",
|
||||
"interrupt_response": True,
|
||||
"min_speech_duration_ms": 96,
|
||||
}
|
||||
)
|
||||
assert interrupting.interrupt_response is True
|
||||
assert interrupting.min_speech_duration_ms == 96
|
||||
for value in (None, 0, "true"):
|
||||
with pytest.raises(ValueError, match="interrupt_response must be a boolean"):
|
||||
ServerVADConfig.from_value({"type": "server_vad", "interrupt_response": value})
|
||||
for enabled in (False, True):
|
||||
_, aliased = parse_session_turn_detection(
|
||||
{
|
||||
"turn_detection": {"type": "server_vad"},
|
||||
"audio": {"input": {"turn_detection": {"type": "server_vad", "interrupt_response": enabled}}},
|
||||
}
|
||||
)
|
||||
assert aliased is not None and aliased.interrupt_response is enabled
|
||||
with pytest.raises(ValueError, match="Unknown"):
|
||||
ServerVADConfig.from_value({"type": "server_vad", "semantic_eagerness": "high"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prefix_padding_ms", [0, 10, 30])
|
||||
async def test_server_vad_pipeline_handles_arbitrary_chunk_boundaries(prefix_padding_ms: int):
|
||||
detector = SequenceDetector([0.0, 0.9, 0.8, 0.0, 0.0])
|
||||
pipeline = ServerVADPipeline(
|
||||
detector,
|
||||
ServerVADConfig(prefix_padding_ms=prefix_padding_ms, silence_duration_ms=20),
|
||||
)
|
||||
samples = np.zeros(detector.frame_samples * 5, dtype=np.float32)
|
||||
|
||||
batches = [
|
||||
await pipeline.push(samples[:73]),
|
||||
await pipeline.push(samples[73:431]),
|
||||
await pipeline.push(samples[431:]),
|
||||
]
|
||||
frames = [frame for batch in batches for frame in batch.frames]
|
||||
|
||||
assert len(frames) == 5
|
||||
assert [index for index, frame in enumerate(frames) if frame.decision.speech_started] == [1]
|
||||
assert [index for index, frame in enumerate(frames) if frame.decision.speech_stopped] == [4]
|
||||
assert frames[1].decision.audio_start_ms == max(0, 10 - prefix_padding_ms)
|
||||
assert frames[4].decision.audio_end_ms == 50
|
||||
|
||||
pipeline.reset()
|
||||
reset_batch = await pipeline.push(samples)
|
||||
assert [index for index, frame in enumerate(reset_batch.frames) if frame.decision.speech_started] == [1]
|
||||
assert [index for index, frame in enumerate(reset_batch.frames) if frame.decision.speech_stopped] == [4]
|
||||
# Keep session-relative timestamps, but never reference prefix audio
|
||||
# discarded by reset at 50 ms.
|
||||
assert reset_batch.frames[1].decision.audio_start_ms == max(50, 60 - prefix_padding_ms)
|
||||
assert reset_batch.frames[4].decision.audio_end_ms == 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_retained_audio_does_not_keep_input_chunk():
|
||||
pipeline = ServerVADPipeline(SequenceDetector([]), ServerVADConfig())
|
||||
session = DuplexSession("server-vad-memory", DuplexSessionConfig())
|
||||
samples = np.zeros(160 * 100 + 1, dtype=np.float32)
|
||||
source_ref = weakref.ref(samples)
|
||||
batch = await pipeline.push(samples)
|
||||
session.append_server_vad_frame(
|
||||
batch.frames[-1].samples,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=80,
|
||||
)
|
||||
session.append_server_vad_frame(
|
||||
batch.frames[-1].samples,
|
||||
speech_started=True,
|
||||
speech_stopped=False,
|
||||
prefix_samples=80,
|
||||
)
|
||||
del batch, samples
|
||||
# Allow the completed to_thread callback to release its result, too.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert source_ref() is None
|
||||
assert pipeline.scratch_bytes == 4
|
||||
assert session.server_vad_utterance_bytes == (80 + 160) * 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_pipeline_pcm16_resampling_is_split_invariant():
|
||||
sample_index = np.arange(2_400, dtype=np.int32)
|
||||
source = ((sample_index * 7919) % 65_536 - 32_768).astype("<i2")
|
||||
# Supply ample silence after the signal so the streaming resampler can
|
||||
# emit a complete set of VAD frames without depending on its FIR width.
|
||||
right_context = np.zeros(240, dtype="<i2")
|
||||
continuous_source = np.concatenate((source, right_context))
|
||||
|
||||
async def process(chunk_sizes: list[int]) -> tuple[np.ndarray, list[SpeechEndpointDecision]]:
|
||||
pipeline = ServerVADPipeline(
|
||||
SequenceDetector([0.0] * 10),
|
||||
ServerVADConfig(),
|
||||
)
|
||||
frames: list[ServerVADFrame] = []
|
||||
offset = 0
|
||||
for chunk_size in chunk_sizes:
|
||||
chunk = continuous_source[offset : offset + chunk_size]
|
||||
batch = await pipeline.push_pcm16(chunk.tobytes(), source_sample_rate_hz=24_000)
|
||||
frames.extend(batch.frames)
|
||||
offset += chunk_size
|
||||
assert offset == continuous_source.size
|
||||
return np.concatenate([frame.samples for frame in frames]), [frame.decision for frame in frames]
|
||||
|
||||
whole_samples, whole_decisions = await process([continuous_source.size])
|
||||
split_samples, split_decisions = await process([7, 13, 511, 512, 513, continuous_source.size - 1_556])
|
||||
|
||||
np.testing.assert_array_equal(split_samples, whole_samples)
|
||||
assert whole_samples.size == 1_600
|
||||
assert split_decisions == whole_decisions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_vad_pipeline_reset_clears_resampler_and_source_rate_lock():
|
||||
pipeline = ServerVADPipeline(
|
||||
SequenceDetector([0.0]),
|
||||
ServerVADConfig(),
|
||||
)
|
||||
|
||||
batch = await pipeline.push_pcm16(
|
||||
np.asarray([30_000, -30_000], dtype="<i2").tobytes(),
|
||||
source_sample_rate_hz=24_000,
|
||||
)
|
||||
assert not batch.frames
|
||||
|
||||
pipeline.reset()
|
||||
|
||||
source = np.zeros(160, dtype="<i2")
|
||||
source[:3] = [-32768, 0, 32767]
|
||||
batch = await pipeline.push_pcm16(source.tobytes(), source_sample_rate_hz=16_000)
|
||||
|
||||
assert len(batch.frames) == 1
|
||||
np.testing.assert_array_equal(
|
||||
batch.frames[0].samples[:3],
|
||||
np.asarray([-1.0, 0.0, 32767 / 32768], dtype=np.float32),
|
||||
)
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_uses_silero_v62_exit_hysteresis():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(threshold=0.5, silence_duration_ms=20),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
started = policy.update(0.8, frame_start_sample=0, frame_samples=frame_samples)
|
||||
assert started.speech_started is True
|
||||
|
||||
# Values below the activation threshold but above threshold - 0.15 must
|
||||
# not begin a silence candidate or close an active speech segment.
|
||||
for frame_index in range(1, 5):
|
||||
decision = policy.update(
|
||||
0.4,
|
||||
frame_start_sample=frame_index * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
assert decision.speech_stopped is False
|
||||
|
||||
first_silence = policy.update(
|
||||
0.2,
|
||||
frame_start_sample=5 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
stopped = policy.update(
|
||||
0.2,
|
||||
frame_start_sample=6 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
|
||||
assert first_silence.speech_stopped is False
|
||||
assert stopped.speech_stopped is True
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_honors_minimum_speech_duration():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(prefix_padding_ms=0, min_speech_duration_ms=25),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
assert not policy.update(0.9, frame_start_sample=0, frame_samples=frame_samples).speech_started
|
||||
assert not policy.update(0.9, frame_start_sample=160, frame_samples=frame_samples).speech_started
|
||||
started = policy.update(0.9, frame_start_sample=320, frame_samples=frame_samples)
|
||||
|
||||
assert started.speech_started is True
|
||||
assert started.audio_start_ms == 0
|
||||
|
||||
|
||||
def test_threshold_endpoint_policy_clamps_silero_exit_threshold():
|
||||
policy = ThresholdEndpointPolicy(
|
||||
ServerVADConfig(threshold=0.1, silence_duration_ms=20),
|
||||
sample_rate_hz=16_000,
|
||||
)
|
||||
frame_samples = 160
|
||||
|
||||
started = policy.update(0.9, frame_start_sample=0, frame_samples=frame_samples)
|
||||
assert started.speech_started is True
|
||||
|
||||
first_silence = policy.update(
|
||||
0.0,
|
||||
frame_start_sample=frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
stopped = policy.update(
|
||||
0.0,
|
||||
frame_start_sample=2 * frame_samples,
|
||||
frame_samples=frame_samples,
|
||||
)
|
||||
|
||||
assert first_silence.speech_stopped is False
|
||||
assert stopped.speech_stopped is True
|
||||
# The committed audio ends at 30 ms: 10 ms of speech followed by the
|
||||
# 20 ms of silence required to detect the endpoint.
|
||||
assert stopped.audio_end_ms == 30
|
||||
assert stopped.endpoint_delay_ms == 20
|
||||
|
||||
|
||||
def test_silero_backend_matches_upstream_v62_streaming_contract(monkeypatch, tmp_path):
|
||||
class FakeSessionOptions:
|
||||
inter_op_num_threads = 0
|
||||
intra_op_num_threads = 0
|
||||
|
||||
class FakeInferenceSession:
|
||||
def __init__(self, path: str, *, providers: list[str], sess_options: object) -> None:
|
||||
self.providers = providers
|
||||
self.sess_options = sess_options
|
||||
self.calls: list[dict[str, np.ndarray]] = []
|
||||
self.run_barrier: threading.Barrier | None = None
|
||||
sessions.append(self)
|
||||
|
||||
def get_inputs(self) -> list[SimpleNamespace]:
|
||||
return [SimpleNamespace(name=name) for name in ("input", "state", "sr")]
|
||||
|
||||
def run(self, _outputs: object, inputs: dict[str, np.ndarray]) -> list[np.ndarray]:
|
||||
self.calls.append({name: np.array(value, copy=True) for name, value in inputs.items()})
|
||||
if self.run_barrier is not None:
|
||||
self.run_barrier.wait(timeout=5)
|
||||
return [
|
||||
np.asarray([[0.75]], dtype=np.float32),
|
||||
np.full((2, 1, 128), len(self.calls), dtype=np.float32),
|
||||
]
|
||||
|
||||
sessions: list[FakeInferenceSession] = []
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"onnxruntime",
|
||||
SimpleNamespace(
|
||||
SessionOptions=FakeSessionOptions,
|
||||
InferenceSession=FakeInferenceSession,
|
||||
),
|
||||
)
|
||||
|
||||
model_path = tmp_path / "silero_vad.onnx"
|
||||
model_path.write_bytes(b"fake-model")
|
||||
backend = SileroVADBackend(model_path)
|
||||
session = sessions[0]
|
||||
|
||||
assert session.providers == ["CPUExecutionProvider"]
|
||||
assert session.sess_options.inter_op_num_threads == 1
|
||||
assert session.sess_options.intra_op_num_threads == 1
|
||||
assert session.calls[0]["input"].shape == (1, 576)
|
||||
assert session.calls[0]["state"].shape == (2, 1, 128)
|
||||
np.testing.assert_array_equal(session.calls[0]["state"], np.zeros((2, 1, 128), dtype=np.float32))
|
||||
assert session.calls[0]["sr"].shape == ()
|
||||
assert session.calls[0]["sr"].item() == 16_000
|
||||
|
||||
state = backend.new_state()
|
||||
first_frame = np.arange(backend.frame_samples, dtype=np.float32)
|
||||
probability, state = backend.infer(first_frame, state)
|
||||
first_call = session.calls[1]
|
||||
|
||||
assert probability == pytest.approx(0.75)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, : backend.context_samples],
|
||||
np.zeros((1, backend.context_samples), dtype=np.float32),
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
first_call["input"][:, backend.context_samples :],
|
||||
first_frame[None, :],
|
||||
)
|
||||
np.testing.assert_array_equal(first_call["state"], np.zeros((2, 1, 128), dtype=np.float32))
|
||||
|
||||
second_frame = -first_frame
|
||||
backend.infer(second_frame, state)
|
||||
second_call = session.calls[2]
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, : backend.context_samples],
|
||||
first_frame[None, -backend.context_samples :],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
second_call["input"][:, backend.context_samples :],
|
||||
second_frame[None, :],
|
||||
)
|
||||
np.testing.assert_array_equal(
|
||||
second_call["state"],
|
||||
np.full((2, 1, 128), 2, dtype=np.float32),
|
||||
)
|
||||
|
||||
# Independent pipeline states may enter the shared ORT session concurrently.
|
||||
session.run_barrier = threading.Barrier(2)
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
futures = [
|
||||
executor.submit(backend.infer, first_frame, backend.new_state()),
|
||||
executor.submit(backend.infer, second_frame, backend.new_state()),
|
||||
]
|
||||
assert [future.result()[0] for future in futures] == pytest.approx([0.75, 0.75])
|
||||
|
||||
|
||||
def test_duplex_session_owns_server_vad_prefix_and_utterance_audio():
|
||||
session = DuplexSession("server-vad-session", DuplexSessionConfig())
|
||||
frame = np.zeros(160, dtype=np.float32)
|
||||
|
||||
session.reserve_input_bytes(frame.nbytes * 3, limit=frame.nbytes * 4)
|
||||
released = session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
assert released == 0
|
||||
released = session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=False,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
assert released == frame.nbytes
|
||||
session.release_input_bytes(released)
|
||||
session.append_server_vad_frame(
|
||||
frame,
|
||||
speech_started=True,
|
||||
speech_stopped=False,
|
||||
prefix_samples=160,
|
||||
)
|
||||
|
||||
assert session.server_vad_utterance_bytes == frame.nbytes * 2
|
||||
assert session.stage_server_vad_audio_for_commit() is True
|
||||
committed = session.commit_user_input()
|
||||
assert committed is not None
|
||||
assert committed.message["role"] == "user"
|
||||
audio_url = committed.message["content"][0]["audio_url"]["url"]
|
||||
assert audio_url.startswith("data:audio/wav;base64,")
|
||||
wav_bytes = base64.b64decode(audio_url.partition(",")[2])
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file:
|
||||
assert wav_file.getnchannels() == 1
|
||||
assert wav_file.getsampwidth() == 2
|
||||
assert wav_file.getframerate() == 16_000
|
||||
assert wav_file.getnframes() == frame.size * 2
|
||||
|
||||
|
||||
def test_silero_provider_rejects_missing_or_invalid_local_artifact(tmp_path):
|
||||
missing = SileroVADBackendProvider(model_path=str(tmp_path / "missing.onnx"))
|
||||
with pytest.raises(RuntimeError, match="does not exist"):
|
||||
missing.get()
|
||||
|
||||
invalid_path = tmp_path / "silero_vad.onnx"
|
||||
invalid_path.write_bytes(b"not-the-pinned-model")
|
||||
invalid = SileroVADBackendProvider(model_path=str(invalid_path))
|
||||
with pytest.raises(RuntimeError, match="checksum mismatch"):
|
||||
invalid.get()
|
||||
@@ -674,7 +674,7 @@ def test_resolve_stage_configs_delegates_overrides_to_resolver(mocker):
|
||||
engine_args=SimpleNamespace(additional_config=additional_config),
|
||||
)
|
||||
resolve_config = mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.resolve_omni_config",
|
||||
"vllm_omni.engine.omni_engine_base.resolve_omni_config",
|
||||
return_value=OmniConfigResolution(
|
||||
config_path="dummy.yaml",
|
||||
stage_configs=(fake_diffusion_stage,),
|
||||
|
||||
@@ -1,601 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.duplex.lease import DuplexLeaseActivity
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexOutputAction,
|
||||
DuplexOutputDecision,
|
||||
duplex_resource_request_id,
|
||||
)
|
||||
from vllm_omni.engine.messages import OutputMessage
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.client_request_state import ClientRequestState
|
||||
from vllm_omni.entrypoints.duplex_request_client import DuplexRequestClient
|
||||
from vllm_omni.metrics.stats import OrchestratorAggregator, StageRequestStats, StageStats
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.outputs.duplex import attach_duplex_output_decision
|
||||
|
||||
|
||||
def test_async_omni_uses_extracted_duplex_request_client():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = SimpleNamespace(num_stages=2)
|
||||
app.request_states = {}
|
||||
app._duplex_request_client = None
|
||||
|
||||
client = app._get_duplex_request_client()
|
||||
|
||||
assert isinstance(client, DuplexRequestClient)
|
||||
assert not hasattr(client, "_async_omni")
|
||||
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_open_duplex_forwards_session_config_and_timeout():
|
||||
calls = []
|
||||
output_handler_starts = 0
|
||||
|
||||
async def open_duplex_session_async(session_id, **kwargs):
|
||||
calls.append((session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = SimpleNamespace(open_duplex_session_async=open_duplex_session_async)
|
||||
|
||||
def start_output_handler():
|
||||
nonlocal output_handler_starts
|
||||
output_handler_starts += 1
|
||||
|
||||
app._final_output_handler = start_output_handler
|
||||
|
||||
result = await app.open_duplex_session_async(
|
||||
"sid",
|
||||
fence=DuplexFence("sid"),
|
||||
capabilities={"implementation_level": "model_native_duplex"},
|
||||
session_config={"instructions": "Be brief.", "idle_timeout_s": 30},
|
||||
timeout=7.5,
|
||||
)
|
||||
|
||||
assert result == {"ok": True}
|
||||
assert output_handler_starts == 1
|
||||
assert calls == [
|
||||
(
|
||||
"sid",
|
||||
{
|
||||
"session_mode": "duplex",
|
||||
"capabilities": {"implementation_level": "model_native_duplex"},
|
||||
"session_config": {"instructions": "Be brief.", "idle_timeout_s": 30},
|
||||
"fence": DuplexFence("sid"),
|
||||
"timeout": 7.5,
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_runtime_controls_forward_timeout():
|
||||
calls = []
|
||||
cancelled_fence = DuplexFence("sid", epoch=2, turn_id=3)
|
||||
next_fence = DuplexFence("sid", epoch=3, turn_id=3)
|
||||
|
||||
async def append_duplex_input_async(session_id, **kwargs):
|
||||
calls.append(("append", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def signal_duplex_turn_async(session_id, **kwargs):
|
||||
calls.append(("signal", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def close_duplex_session_async(session_id, **kwargs):
|
||||
calls.append(("close", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def touch_duplex_session_async(session_id, **kwargs):
|
||||
calls.append(("touch", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
async def resume_duplex_session_async(session_id, **kwargs):
|
||||
calls.append(("resume", session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
app.engine = SimpleNamespace(
|
||||
append_duplex_input_async=append_duplex_input_async,
|
||||
signal_duplex_turn_async=signal_duplex_turn_async,
|
||||
close_duplex_session_async=close_duplex_session_async,
|
||||
touch_duplex_session_async=touch_duplex_session_async,
|
||||
resume_duplex_session_async=resume_duplex_session_async,
|
||||
)
|
||||
|
||||
await app.append_duplex_input_async(
|
||||
"sid",
|
||||
mode="append_audio_chunk",
|
||||
payload={},
|
||||
fence=cancelled_fence,
|
||||
timeout=12.5,
|
||||
)
|
||||
await app.signal_duplex_turn_async(
|
||||
"sid",
|
||||
event="barge_in",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
timeout=13.5,
|
||||
)
|
||||
await app.close_duplex_session_async("sid", reason="done", fence=next_fence, timeout=14.5)
|
||||
await app.touch_duplex_session_async(
|
||||
"sid",
|
||||
fence=next_fence,
|
||||
activity=DuplexLeaseActivity.HEARTBEAT,
|
||||
timeout=15.5,
|
||||
)
|
||||
await app.resume_duplex_session_async(
|
||||
"sid",
|
||||
fence=next_fence,
|
||||
expected_lease_generation=3,
|
||||
timeout=16.5,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(
|
||||
"append",
|
||||
"sid",
|
||||
{
|
||||
"mode": "append_audio_chunk",
|
||||
"payload": {},
|
||||
"final": False,
|
||||
"expected_epoch": None,
|
||||
"fence": cancelled_fence,
|
||||
"timeout": 12.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"signal",
|
||||
"sid",
|
||||
{
|
||||
"event": "barge_in",
|
||||
"fence": cancelled_fence,
|
||||
"next_fence": next_fence,
|
||||
"timeout": 13.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"close",
|
||||
"sid",
|
||||
{
|
||||
"reason": "done",
|
||||
"fence": next_fence,
|
||||
"timeout": 14.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"touch",
|
||||
"sid",
|
||||
{
|
||||
"fence": next_fence,
|
||||
"activity": DuplexLeaseActivity.HEARTBEAT,
|
||||
"timeout": 15.5,
|
||||
},
|
||||
),
|
||||
(
|
||||
"resume",
|
||||
"sid",
|
||||
{
|
||||
"fence": next_fence,
|
||||
"expected_lease_generation": 3,
|
||||
"timeout": 16.5,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_append_can_defer_data_plane_output_collection():
|
||||
app = object.__new__(AsyncOmni)
|
||||
request_id = duplex_resource_request_id(DuplexFence("sid"), "stage0")
|
||||
|
||||
async def append_duplex_input_async(session_id, **kwargs):
|
||||
del session_id, kwargs
|
||||
request_state = app.request_states.get(request_id)
|
||||
assert request_state is not None
|
||||
assert request_state.metrics is not None
|
||||
return {
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": request_id,
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
app.engine = SimpleNamespace(append_duplex_input_async=append_duplex_input_async)
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
|
||||
result = await app.append_duplex_input_async(
|
||||
"sid",
|
||||
mode="append_audio_chunk",
|
||||
payload={},
|
||||
fence=DuplexFence("sid"),
|
||||
collect_outputs=False,
|
||||
)
|
||||
|
||||
assert "data_plane_outputs" not in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_append_forwards_fence():
|
||||
calls = []
|
||||
|
||||
async def append_duplex_input_async(session_id, **kwargs):
|
||||
calls.append((session_id, kwargs))
|
||||
return {"ok": True}
|
||||
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
app.engine = SimpleNamespace(append_duplex_input_async=append_duplex_input_async)
|
||||
fence = DuplexFence("sid", epoch=1, turn_id=2)
|
||||
|
||||
await app.append_duplex_input_async(
|
||||
"sid",
|
||||
mode="append_audio_chunk",
|
||||
payload={},
|
||||
fence=fence,
|
||||
collect_outputs=False,
|
||||
)
|
||||
|
||||
assert calls[0][1]["fence"] is fence
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_collect_waits_for_response_stage():
|
||||
app = object.__new__(AsyncOmni)
|
||||
req_state = ClientRequestState("duplex-sid-e0-stage0")
|
||||
stage0_output = OmniRequestOutput(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=0,
|
||||
finished=False,
|
||||
)
|
||||
stage1_output = OmniRequestOutput(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=1,
|
||||
final_output_type="audio",
|
||||
finished=False,
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=0,
|
||||
engine_outputs=stage0_output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=1,
|
||||
engine_outputs=stage1_output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
outputs = await app._collect_duplex_data_plane_outputs(
|
||||
"duplex-sid-e0-stage0",
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert outputs == [stage1_output]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_request_client_retains_output_route_at_segment_terminal():
|
||||
request_id = "duplex-sid-e0-stage0"
|
||||
request_state = ClientRequestState(request_id)
|
||||
request_states = {request_id: request_state}
|
||||
output = OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=1,
|
||||
final_output_type="audio",
|
||||
finished=True,
|
||||
)
|
||||
await request_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=1,
|
||||
engine_outputs=output,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
client = DuplexRequestClient(
|
||||
SimpleNamespace(),
|
||||
SimpleNamespace(
|
||||
request_states=request_states,
|
||||
num_stages=2,
|
||||
log_stats=False,
|
||||
),
|
||||
)
|
||||
|
||||
outputs = await client.collect_outputs(
|
||||
request_id,
|
||||
request_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert outputs == [output]
|
||||
assert request_states[request_id] is request_state
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_direct_output_keeps_stage_metrics():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app._duplex_request_client = None
|
||||
app._final_output_handler = lambda: None
|
||||
request_id = "duplex-sid-e0-stage0"
|
||||
req_state = ClientRequestState(request_id)
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={
|
||||
"duplex_direct_response": True,
|
||||
"duplex_native_decision": "speak",
|
||||
},
|
||||
)
|
||||
direct_output = attach_duplex_output_decision(
|
||||
OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
final_output_type="text",
|
||||
finished=True,
|
||||
),
|
||||
decision,
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
engine_outputs=direct_output,
|
||||
metrics=StageRequestStats(
|
||||
batch_id=1,
|
||||
batch_size=1,
|
||||
num_tokens_in=160,
|
||||
num_tokens_out=8,
|
||||
stage_gen_time_ms=80.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(total_token=8, total_gen_time_ms=80.0),
|
||||
vllm_ttft_ms=12.5,
|
||||
vllm_tpot_ms=9.0,
|
||||
vllm_itl_ms=8.75,
|
||||
vllm_itls_ms=[8.0, 9.5],
|
||||
),
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
|
||||
outputs = await app._collect_duplex_data_plane_outputs(
|
||||
request_id,
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert outputs == [direct_output]
|
||||
assert direct_output.metrics["stage_metrics"]["0"]["vllm_ttft_ms"] == 12.5
|
||||
assert direct_output.metrics["stage_metrics"]["0"]["vllm_tpot_ms"] == 9.0
|
||||
assert direct_output.metrics["stage_metrics"]["0"]["vllm_itls_ms"] == [8.0, 9.5]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_metrics_cursor_isolates_resumable_collect_windows():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app._duplex_request_client = None
|
||||
app._final_output_handler = lambda: None
|
||||
request_id = "duplex-sid-e0-stage0"
|
||||
req_state = ClientRequestState(request_id)
|
||||
req_state.metrics = OrchestratorAggregator(
|
||||
num_stages=2,
|
||||
log_stats=False,
|
||||
wall_start_ts=0.0,
|
||||
final_stage_id_for_e2e=1,
|
||||
)
|
||||
|
||||
def stage_message(*, batch_id: int, tokens: int, ttft_ms: float, itls: list[float]) -> OutputMessage:
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={"duplex_direct_response": True, "duplex_native_decision": "speak"},
|
||||
)
|
||||
output = attach_duplex_output_decision(
|
||||
OmniRequestOutput(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
final_output_type="text",
|
||||
finished=True,
|
||||
),
|
||||
decision,
|
||||
)
|
||||
return OutputMessage(
|
||||
request_id=request_id,
|
||||
stage_id=0,
|
||||
engine_outputs=output,
|
||||
metrics=StageRequestStats(
|
||||
batch_id=batch_id,
|
||||
batch_size=1,
|
||||
num_tokens_in=160,
|
||||
num_tokens_out=tokens,
|
||||
stage_gen_time_ms=80.0,
|
||||
rx_transfer_bytes=0,
|
||||
rx_decode_time_ms=0.0,
|
||||
rx_in_flight_time_ms=0.0,
|
||||
stage_stats=StageStats(total_token=tokens, total_gen_time_ms=80.0),
|
||||
vllm_ttft_ms=ttft_ms,
|
||||
vllm_tpot_ms=9.0,
|
||||
vllm_itl_ms=sum(itls) / len(itls),
|
||||
vllm_itls_ms=itls,
|
||||
),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
await req_state.queue.put(stage_message(batch_id=1, tokens=71, ttft_ms=62.5, itls=[10.0, 11.0]))
|
||||
first = await app._collect_duplex_data_plane_outputs(
|
||||
request_id,
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
await req_state.queue.put(stage_message(batch_id=2, tokens=155, ttft_ms=81.0, itls=[12.0, 13.0, 14.0]))
|
||||
second = await app._collect_duplex_data_plane_outputs(
|
||||
request_id,
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert first[0].metrics["stage_metrics"]["0"]["num_tokens_out"] == 71
|
||||
second_stage0 = second[0].metrics["stage_metrics"]["0"]
|
||||
assert second_stage0["num_tokens_out"] == 155
|
||||
assert second_stage0["vllm_ttft_ms"] == 81.0
|
||||
assert second_stage0["vllm_itls_ms"] == [12.0, 13.0, 14.0]
|
||||
|
||||
|
||||
def test_async_omni_duplex_direct_output_prefers_outer_control_metadata():
|
||||
decision = DuplexOutputDecision(
|
||||
action=DuplexOutputAction.DIRECT_RESPONSE,
|
||||
metadata={
|
||||
"duplex_direct_response": True,
|
||||
"duplex_native_decision": "listen",
|
||||
},
|
||||
)
|
||||
output = attach_duplex_output_decision(
|
||||
OmniRequestOutput(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=0,
|
||||
outputs=[],
|
||||
_multimodal_output={"kind": "processed-payload"},
|
||||
_custom_output={"inner": "completion"},
|
||||
),
|
||||
decision,
|
||||
)
|
||||
|
||||
assert AsyncOmni._is_direct_duplex_data_plane_response(output)
|
||||
|
||||
|
||||
def test_async_omni_drops_unregistered_duplex_prefixed_output():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.request_states = {}
|
||||
output = OmniRequestOutput(request_id="duplex-unregistered-e0-stage0", stage_id=0)
|
||||
|
||||
keep_processing, request_id, stage_id, request_state = app._handle_output_message(
|
||||
OutputMessage(
|
||||
request_id=output.request_id,
|
||||
stage_id=0,
|
||||
engine_outputs=output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert keep_processing is True
|
||||
assert request_id is stage_id is request_state is None
|
||||
assert app.request_states == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_duplex_collect_wraps_raw_response_stage_output():
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.log_stats = False
|
||||
app._enable_ar_profiler = False
|
||||
app.request_states = {}
|
||||
app.prom_metrics = SimpleNamespace(
|
||||
set_running=lambda value: None,
|
||||
set_waiting=lambda value: None,
|
||||
)
|
||||
app.engine = SimpleNamespace(
|
||||
num_stages=2,
|
||||
get_stage_metadata=lambda stage_id: SimpleNamespace(
|
||||
final_output=stage_id == 1,
|
||||
final_output_type="audio",
|
||||
),
|
||||
)
|
||||
req_state = ClientRequestState("duplex-sid-e0-stage0")
|
||||
raw_stage1_output = SimpleNamespace(
|
||||
finished=False,
|
||||
outputs=[],
|
||||
stage_durations={},
|
||||
peak_memory_mb=0.0,
|
||||
final_output_type="audio",
|
||||
)
|
||||
await req_state.queue.put(
|
||||
OutputMessage(
|
||||
request_id="duplex-sid-e0-stage0",
|
||||
stage_id=1,
|
||||
engine_outputs=raw_stage1_output,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
outputs = await app._collect_duplex_data_plane_outputs(
|
||||
"duplex-sid-e0-stage0",
|
||||
req_state,
|
||||
response_stage_id=1,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].request_id == "duplex-sid-e0-stage0"
|
||||
assert outputs[0].finished is False
|
||||
assert outputs[0].stage_id == 1
|
||||
assert outputs[0].final_output_type == "audio"
|
||||
assert outputs[0].peak_memory_mb == raw_stage1_output.peak_memory_mb
|
||||
|
||||
|
||||
def test_async_omni_duplex_request_info_includes_response_stage():
|
||||
request_id, response_stage_id = AsyncOmni._duplex_data_plane_request_info(
|
||||
{
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": "duplex-sid-e0-stage0",
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
assert request_id == "duplex-sid-e0-stage0"
|
||||
assert response_stage_id == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_request_client_retains_output_route_when_close_fails():
|
||||
fence = DuplexFence("sid-close-route")
|
||||
request_id = duplex_resource_request_id(fence, "stage0")
|
||||
request_states = {request_id: ClientRequestState(request_id)}
|
||||
|
||||
async def close_duplex_session_async(session_id, **kwargs):
|
||||
del session_id, kwargs
|
||||
raise RuntimeError("close failed")
|
||||
|
||||
client = DuplexRequestClient(
|
||||
SimpleNamespace(close_duplex_session_async=close_duplex_session_async),
|
||||
SimpleNamespace(request_states=request_states),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="close failed"):
|
||||
await client.close(fence.session_id, reason="test", fence=fence, timeout=1.0)
|
||||
|
||||
assert request_id in request_states
|
||||
@@ -53,7 +53,7 @@ def test_async_engine_consumes_disable_log_stats_before_resolution(monkeypatch:
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.engine.async_omni_engine.load_and_resolve_stage_configs",
|
||||
"vllm_omni.engine.omni_engine_base.load_and_resolve_stage_configs",
|
||||
fake_resolve,
|
||||
)
|
||||
engine = SimpleNamespace(_apply_strategy_lb_policy=lambda *args, **kwargs: None)
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.openai_api.test_duplex_handler import FakeChatService, TimedWebSocket
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.duplex.protocol import DuplexSession, DuplexSessionConfig
|
||||
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FenceRecordingEngine:
|
||||
output_modalities = ["text", "audio"]
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[str, DuplexFence | None]] = []
|
||||
self.signal_next_fences: list[DuplexFence | None] = []
|
||||
|
||||
async def open_duplex_session_async(self, session_id, *, fence=None, **kwargs):
|
||||
del session_id, kwargs
|
||||
self.calls.append(("open", fence))
|
||||
return {"ok": True}
|
||||
|
||||
async def append_duplex_input_async(self, session_id, *, fence=None, **kwargs):
|
||||
del session_id, kwargs
|
||||
self.calls.append(("append", fence))
|
||||
return {"ok": True}
|
||||
|
||||
async def signal_duplex_turn_async(
|
||||
self,
|
||||
session_id,
|
||||
*,
|
||||
event,
|
||||
fence=None,
|
||||
next_fence=None,
|
||||
timeout=None,
|
||||
):
|
||||
del session_id, event, timeout
|
||||
self.calls.append(("signal", fence))
|
||||
self.signal_next_fences.append(next_fence)
|
||||
return {"ok": True}
|
||||
|
||||
async def close_duplex_session_async(self, session_id, *, fence=None, **kwargs):
|
||||
del session_id, kwargs
|
||||
self.calls.append(("close", fence))
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_omni_forwards_fence_for_all_runtime_operations():
|
||||
engine = FenceRecordingEngine()
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = engine
|
||||
app.request_states = {}
|
||||
app._final_output_handler = lambda: None
|
||||
fence = DuplexFence("sid", epoch=1, turn_id=2)
|
||||
|
||||
await app.open_duplex_session_async("sid", fence=fence)
|
||||
await app.append_duplex_input_async(
|
||||
"sid", mode="append_audio_chunk", payload={}, fence=fence, collect_outputs=False
|
||||
)
|
||||
await app.signal_duplex_turn_async("sid", event="turn.end", fence=fence)
|
||||
await app.close_duplex_session_async("sid", reason="done", fence=fence)
|
||||
|
||||
assert engine.calls == [(operation, fence) for operation in ("open", "append", "signal", "close")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_handler_passes_current_session_fence_for_runtime_controls():
|
||||
engine = FenceRecordingEngine()
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
|
||||
session = DuplexSession(
|
||||
session_id="sid",
|
||||
config=DuplexSessionConfig(extra_body={"native_duplex": True}),
|
||||
epoch=1,
|
||||
turn_id=2,
|
||||
)
|
||||
ws = TimedWebSocket()
|
||||
fence = DuplexFence("sid", epoch=1, turn_id=2)
|
||||
|
||||
await handler._open_runtime_session(session, ws.send_json)
|
||||
await handler._signal_runtime_session(session, "turn.end", send_json=ws.send_json)
|
||||
await handler._close_runtime_session(session, reason="done", send_json=ws.send_json)
|
||||
|
||||
assert engine.calls == [(operation, fence) for operation in ("open", "signal", "close")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_signal_uses_fence_captured_before_session_turn_advances():
|
||||
engine = FenceRecordingEngine()
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(engine))
|
||||
session = DuplexSession(session_id="sid", config=DuplexSessionConfig(extra_body={"native_duplex": True}))
|
||||
captured_fence = DuplexFence("sid", epoch=0, turn_id=0)
|
||||
session.turn_id = 1
|
||||
|
||||
await handler._signal_runtime_session(
|
||||
session,
|
||||
"input.commit",
|
||||
fence=captured_fence,
|
||||
)
|
||||
|
||||
assert engine.calls == [("signal", captured_fence)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_handler_forwards_cancel_fence_through_async_omni_facade():
|
||||
engine = FenceRecordingEngine()
|
||||
app = object.__new__(AsyncOmni)
|
||||
app.engine = engine
|
||||
handler = OmniDuplexSessionHandler(chat_service=FakeChatService(app))
|
||||
session = DuplexSession(session_id="sid-cancel", config=DuplexSessionConfig(extra_body={"native_duplex": True}))
|
||||
cancelled_fence = DuplexFence("sid-cancel", epoch=2, turn_id=3)
|
||||
next_fence = DuplexFence("sid-cancel", epoch=3, turn_id=3)
|
||||
|
||||
assert await handler._signal_runtime_session(
|
||||
session,
|
||||
"input.cancel",
|
||||
fence=cancelled_fence,
|
||||
next_fence=next_fence,
|
||||
)
|
||||
|
||||
assert engine.calls == [("signal", cancelled_fence)]
|
||||
assert engine.signal_next_fences == [next_fence]
|
||||
@@ -0,0 +1,326 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""``DuplexOmni`` / ``DuplexSessionHandle``: the thin Python API over a fake engine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.entrypoints.test_omni_entrypoints import FakeAsyncOmniEngine
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.duplex import commands
|
||||
from vllm_omni.engine.duplex.config import DuplexCapabilities, DuplexSessionConfig
|
||||
from vllm_omni.engine.duplex.events import (
|
||||
AudioDelta,
|
||||
DuplexEvent,
|
||||
SessionClosed,
|
||||
SessionCreated,
|
||||
SessionExpired,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import (
|
||||
DuplexControlResultMessage,
|
||||
DuplexSessionError,
|
||||
DuplexSessionEventMessage,
|
||||
)
|
||||
from vllm_omni.engine.messages import ErrorMessage
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
class FakeDuplexEngine(FakeAsyncOmniEngine):
|
||||
"""The engine surface ``DuplexOmni`` uses; every call is recorded."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.duplex_session_config = DuplexSessionRuntimeConfig(max_sessions=2)
|
||||
self.duplex_capabilities = DuplexCapabilities(supports_session_resume=True)
|
||||
self.opened: list[tuple[str, DuplexSessionConfig]] = []
|
||||
self.closed: list[tuple[str, str]] = []
|
||||
self.resumed: list[tuple[str, int]] = []
|
||||
self.touched: list[tuple[str, str]] = []
|
||||
self.commands: list[tuple[str, commands.DuplexCommand]] = []
|
||||
self.open_error: DuplexSessionError | None = None
|
||||
self.close_error: DuplexSessionError | None = None
|
||||
|
||||
def _result(self, operation: str, session_id: str, **fields: Any) -> DuplexControlResultMessage:
|
||||
return DuplexControlResultMessage(control_id="c", operation=operation, session_id=session_id, ok=True, **fields)
|
||||
|
||||
async def open_session_async(self, session_id, session_config, *, timeout=None):
|
||||
self.opened.append((session_id, session_config))
|
||||
if self.open_error is not None:
|
||||
raise self.open_error
|
||||
return self._result(
|
||||
"open",
|
||||
session_id,
|
||||
capabilities=self.duplex_capabilities,
|
||||
public_session={"id": session_id, "idle_timeout_s": 7},
|
||||
lease_generation=0,
|
||||
)
|
||||
|
||||
async def close_session_async(self, session_id, *, reason="client_close", timeout=None):
|
||||
self.closed.append((session_id, reason))
|
||||
if self.close_error is not None:
|
||||
raise self.close_error
|
||||
# The runner emits session.closed before answering the RPC.
|
||||
self.emit(session_id, SessionClosed(reason=reason))
|
||||
return self._result("close", session_id)
|
||||
|
||||
async def resume_session_async(self, session_id, *, expected_lease_generation, timeout=None):
|
||||
self.resumed.append((session_id, expected_lease_generation))
|
||||
return self._result("resume", session_id, lease_generation=expected_lease_generation + 1)
|
||||
|
||||
async def touch_session_async(self, session_id, *, activity, timeout=None):
|
||||
self.touched.append((session_id, activity))
|
||||
return self._result("touch", session_id)
|
||||
|
||||
async def submit_command_async(self, session_id, command):
|
||||
self.commands.append((session_id, command))
|
||||
|
||||
def emit(self, session_id: str, event: DuplexEvent) -> None:
|
||||
# The session manager binds the session identity to every event it emits.
|
||||
event = replace(event, session_id=session_id)
|
||||
self.output_q.put(DuplexSessionEventMessage(session_id=session_id, event=event))
|
||||
|
||||
|
||||
def _make_omni(monkeypatch: pytest.MonkeyPatch) -> tuple[DuplexOmni, FakeDuplexEngine]:
|
||||
engine = FakeDuplexEngine()
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.duplex_omni.DuplexOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
|
||||
return DuplexOmni("dummy-model"), engine
|
||||
|
||||
|
||||
async def _collect(handle: DuplexSessionHandle, count: int, *, timeout_s: float = 2.0) -> list[DuplexEvent]:
|
||||
events: list[DuplexEvent] = []
|
||||
|
||||
async def consume() -> None:
|
||||
async for event in handle.events():
|
||||
events.append(event)
|
||||
if len(events) >= count:
|
||||
break
|
||||
|
||||
await asyncio.wait_for(consume(), timeout=timeout_s)
|
||||
return events
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_session_allocates_the_id_and_ignores_client_ids(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session({"session_id": "mine", "id": "mine-too", "instructions": "hi"})
|
||||
|
||||
assert handle.session_id.startswith("duplex-") and len(handle.session_id) == len("duplex-") + 32
|
||||
assert handle.session_id != "mine"
|
||||
((session_id, config),) = engine.opened
|
||||
assert session_id == handle.session_id
|
||||
assert isinstance(config, DuplexSessionConfig)
|
||||
assert config.model == "dummy-model" and config.instructions == "hi"
|
||||
assert handle.capabilities is engine.duplex_capabilities
|
||||
assert handle.public_session["id"] == handle.session_id
|
||||
assert omni.sessions == {handle.session_id: handle}
|
||||
assert omni.get_session(handle.session_id) is handle
|
||||
assert omni.active_session_count() == 1
|
||||
|
||||
second = await omni.open_session(DuplexSessionConfig(instructions="typed"))
|
||||
assert second.session_id != handle.session_id
|
||||
assert engine.opened[1][1].model == "dummy-model"
|
||||
assert omni.duplex_session_config.max_sessions == 2
|
||||
assert omni.duplex_capabilities.supports_session_resume is True
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_open_failure_drops_the_pending_handle_and_raises_the_engine_code(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
engine.open_error = DuplexSessionError("full", code="resource_exhausted", retryable=True)
|
||||
try:
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await omni.open_session()
|
||||
assert excinfo.value.code == "resource_exhausted"
|
||||
assert omni.sessions == {}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_are_routed_to_their_handle_and_end_on_session_closed(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.emit("duplex-unknown", AudioDelta(delta="zzz")) # dropped
|
||||
engine.emit(handle.session_id, SessionCreated(session={"id": handle.session_id}))
|
||||
engine.emit(handle.session_id, AudioDelta(response_id="r1", delta="abc"))
|
||||
engine.emit(handle.session_id, SessionClosed(reason="client_close"))
|
||||
|
||||
events = await _collect(handle, 10)
|
||||
|
||||
assert [event.type for event in events] == ["session.created", "response.output_audio.delta", "session.closed"]
|
||||
assert events[0].session_id == handle.session_id
|
||||
assert handle.closed and handle.close_reason == "client_close"
|
||||
assert await handle.wait_closed() == "client_close"
|
||||
assert handle.session_id not in omni.sessions
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await handle.submit(commands.Heartbeat())
|
||||
assert excinfo.value.code == "session_closed"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_is_single_consumer_but_may_be_reentered(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.emit(handle.session_id, AudioDelta(delta="one"))
|
||||
first = handle.events()
|
||||
assert (await first.__anext__()).type == "response.output_audio.delta"
|
||||
with pytest.raises(RuntimeError, match="active events"):
|
||||
await handle.events().__anext__()
|
||||
await first.aclose()
|
||||
|
||||
engine.emit(handle.session_id, SessionExpired(reason="idle_ttl_expired"))
|
||||
events = await _collect(handle, 1)
|
||||
assert events[0].type == "session.expired" and events[0].is_terminal
|
||||
assert handle.closed and handle.close_reason == "idle_ttl_expired"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_session_waits_for_the_typed_session_closed_event(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
consumer = asyncio.create_task(_collect(handle, 1))
|
||||
await handle.close(reason="done")
|
||||
|
||||
assert engine.closed == [(handle.session_id, "done")]
|
||||
assert handle.closed and handle.close_reason == "done"
|
||||
assert (await consumer)[0].type == "session.closed"
|
||||
# Idempotent: a second close is a no-op.
|
||||
await omni.close_session(handle.session_id)
|
||||
assert len(engine.closed) == 1
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_failure_marks_the_handle_closed_and_propagates(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
engine.close_error = DuplexSessionError("boom", code="engine_error")
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
with pytest.raises(DuplexSessionError, match="boom"):
|
||||
await omni.close_session(handle.session_id)
|
||||
assert handle.closed
|
||||
# A failed close must not leave a dead handle shadowing the id.
|
||||
assert omni.sessions == {}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_engine_death_ends_every_session_handle(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.output_q.put(ErrorMessage(error="orchestrator died", fatal=True))
|
||||
|
||||
await asyncio.wait_for(handle.wait_closed(), timeout=2.0)
|
||||
assert handle.closed
|
||||
assert "engine_dead" in (handle.close_reason or "")
|
||||
assert omni.sessions == {}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_detach_and_touch_forward_to_the_engine(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
resumed = await omni.resume_session(handle.session_id, expected_lease_generation=0)
|
||||
assert resumed is handle and handle.lease_generation == 1
|
||||
assert engine.resumed == [(handle.session_id, 0)]
|
||||
|
||||
await omni.detach_session(handle.session_id)
|
||||
await omni.touch_session(handle.session_id, activity="heartbeat")
|
||||
assert engine.touched == [(handle.session_id, "detach"), (handle.session_id, "heartbeat")]
|
||||
|
||||
for call in (
|
||||
omni.resume_session("duplex-nope", expected_lease_generation=0),
|
||||
omni.detach_session("duplex-nope"),
|
||||
):
|
||||
with pytest.raises(DuplexSessionError) as excinfo:
|
||||
await call
|
||||
assert excinfo.value.code == "unknown_session"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_wrappers_build_typed_commands_in_caller_order(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
await handle.append_audio(b"\x00\x01", sample_rate_hz=16000, is_speech=True)
|
||||
await handle.commit(create_response=True)
|
||||
await handle.create_response({"instructions": "x"})
|
||||
await handle.cancel_response("resp-1")
|
||||
await handle.barge_in()
|
||||
await handle.ack_playback(120.7, response_id="resp-1", committed_ms=100)
|
||||
await handle.update({"temperature": 0.2})
|
||||
await handle.create_item({"id": "item_1", "type": "message"}, previous_item_id="root")
|
||||
await handle.delete_item("item_1")
|
||||
await handle.truncate_item("item_1", audio_end_ms=50)
|
||||
await handle.signal_turn("user_started", {"x": 1})
|
||||
await handle.heartbeat()
|
||||
|
||||
submitted = [command for session_id, command in engine.commands if session_id == handle.session_id]
|
||||
assert [type(command) for command in submitted] == [
|
||||
commands.AppendAudio,
|
||||
commands.Commit,
|
||||
commands.CreateResponse,
|
||||
commands.CancelResponse,
|
||||
commands.BargeIn,
|
||||
commands.AckPlayback,
|
||||
commands.UpdateSession,
|
||||
commands.CreateItem,
|
||||
commands.DeleteItem,
|
||||
commands.TruncateItem,
|
||||
commands.SignalTurn,
|
||||
commands.Heartbeat,
|
||||
]
|
||||
append, commit, create, cancel, _, ack, update, item, delete, truncate, signal, _ = submitted
|
||||
assert append.audio == b"\x00\x01" and append.sample_rate_hz == 16000 and append.is_speech is True
|
||||
assert commit.create_response is True and commit.final is True
|
||||
assert create.options == {"instructions": "x"}
|
||||
assert cancel.response_id == "resp-1"
|
||||
assert ack.played_ms == 120 and ack.committed_ms == 100 and ack.response_id == "resp-1"
|
||||
assert update.patch == {"temperature": 0.2}
|
||||
assert item.previous_item_id == "root" and item.item["id"] == "item_1"
|
||||
assert delete.item_id == "item_1"
|
||||
assert truncate.audio_end_ms == 50
|
||||
assert signal.event == "user_started" and signal.signal_payload == {"x": 1}
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_closes_every_handle(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
first = await omni.open_session()
|
||||
second = await omni.open_session()
|
||||
|
||||
omni.shutdown()
|
||||
|
||||
assert first.closed and second.closed
|
||||
assert first.close_reason == "shutdown"
|
||||
assert omni.sessions == {}
|
||||
assert engine.shutdown_called
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Unit tests for OmniBase and AsyncOmni profiler methods."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
@@ -28,10 +31,14 @@ class TestOmniBaseProfiler:
|
||||
@pytest.fixture
|
||||
def omni_base_instance(self, mock_engine, mocker: MockerFixture):
|
||||
"""Create an OmniBase instance with mocked dependencies."""
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", return_value=mock_engine)
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
# ``OmniBase`` builds its engine through the ``_create_engine`` seam that
|
||||
# ``AsyncOmni`` / ``DuplexOmni`` implement; there is no module-level
|
||||
# engine class to patch any more.
|
||||
mocker.patch.object(OmniBase, "_create_engine", return_value=mock_engine)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", side_effect=lambda x: x)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.weakref.finalize")
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
instance = OmniBase(model="test-model")
|
||||
return instance
|
||||
@@ -48,10 +55,14 @@ class TestOmniBaseProfiler:
|
||||
|
||||
def test_removed_diffusion_batch_size_fails_loudly(self, mock_engine, mocker: MockerFixture):
|
||||
"""Legacy diffusion_batch_size callers receive an actionable error."""
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", return_value=mock_engine)
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
# ``OmniBase`` builds its engine through the ``_create_engine`` seam that
|
||||
# ``AsyncOmni`` / ``DuplexOmni`` implement; there is no module-level
|
||||
# engine class to patch any more.
|
||||
mocker.patch.object(OmniBase, "_create_engine", return_value=mock_engine)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.omni_snapshot_download", side_effect=lambda x: x)
|
||||
mocker.patch("vllm_omni.entrypoints.omni_base.weakref.finalize")
|
||||
from vllm_omni.entrypoints.omni_base import OmniBase
|
||||
|
||||
with pytest.raises(TypeError, match="diffusion_batch_size.*max_num_seqs"):
|
||||
OmniBase(model="test-model", diffusion_batch_size=8)
|
||||
|
||||
@@ -118,8 +118,6 @@ class FakeAsyncOmniEngine:
|
||||
self.output_processors = [SimpleNamespace(tokenizer=None) for _ in range(self.num_stages)]
|
||||
self.input_processor = None
|
||||
self.endpoint_restrictions = ()
|
||||
self.duplex_session_config = None
|
||||
self.duplex_serving_adapter_path = None
|
||||
|
||||
self.output_q: queue.Queue[Any] = queue.Queue()
|
||||
self.submitted: list[dict[str, Any]] = []
|
||||
@@ -183,7 +181,8 @@ class FakeAsyncOmniEngine:
|
||||
|
||||
|
||||
def _patch_engine(monkeypatch: pytest.MonkeyPatch, engine: FakeAsyncOmniEngine) -> None:
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.AsyncOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni.AsyncOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.async_omni.AsyncOmniEngine", lambda *args, **kwargs: engine)
|
||||
monkeypatch.setattr("vllm_omni.entrypoints.omni_base.omni_snapshot_download", lambda model: model)
|
||||
# Don't add random UUIDs to requests calling .generate since we usually
|
||||
# just want to check for present requests anyway, and would need to just
|
||||
|
||||
@@ -36,7 +36,7 @@ def test_client_uses_proxy_relative_realtime_url_and_model_policy_session():
|
||||
assert "new URL(config.realtimePath, window.location.href)" in source
|
||||
assert "url.protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'" in source
|
||||
assert "url.searchParams.set('autostart', '0')" in source
|
||||
assert "url.searchParams.set('native_duplex', '1')" in source
|
||||
assert "native_duplex" not in source
|
||||
assert "auto_response: true" in source
|
||||
assert "input_audio_buffer.append" in source
|
||||
assert "input_audio_buffer.commit" not in source
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.duplex.commit_policy import (
|
||||
from vllm_omni.engine.duplex.session.commit_policy import (
|
||||
CommitAction,
|
||||
CommitSnapshot,
|
||||
decide_commit_action,
|
||||
@@ -24,7 +24,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=False,
|
||||
response_in_progress=False,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.START_AUTO_RESPONSE,
|
||||
@@ -36,7 +36,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=True,
|
||||
response_in_progress=True,
|
||||
playback_active=True,
|
||||
),
|
||||
CommitAction.START_AUTO_RESPONSE,
|
||||
@@ -48,7 +48,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
speech_since_commit=False,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=True,
|
||||
response_in_progress=True,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.COMMIT_ONLY,
|
||||
@@ -60,7 +60,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=True,
|
||||
response_in_progress=True,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.DEFER_ACTIVE_RESPONSE,
|
||||
@@ -72,7 +72,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=800,
|
||||
native_response_in_progress=False,
|
||||
response_in_progress=False,
|
||||
playback_active=True,
|
||||
),
|
||||
CommitAction.DEFER_ACTIVE_RESPONSE,
|
||||
@@ -84,7 +84,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
speech_since_commit=True,
|
||||
active_response_id=None,
|
||||
overlap_speech_ms=0,
|
||||
native_response_in_progress=False,
|
||||
response_in_progress=False,
|
||||
playback_active=False,
|
||||
),
|
||||
CommitAction.COMMIT_ONLY,
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""The cumulative Stage1 audio cursor.
|
||||
|
||||
Stage1 hands the data plane a snapshot of every sample it has generated for the
|
||||
request so far, and only the new tail may reach the client. The cursor that
|
||||
makes that possible is the one piece of per-request state whose loss is silent
|
||||
and expensive: rewinding it re-sends the whole session as a single delta.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.duplex.data_plane import (
|
||||
MiniCPMO45DataPlaneSession,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
REQUEST_ID = "duplex-s.c2Vzc2lvbg==.e.0.r.stage0"
|
||||
|
||||
|
||||
def _session() -> MiniCPMO45DataPlaneSession:
|
||||
return MiniCPMO45DataPlaneSession(lambda *args, **kwargs: "")
|
||||
|
||||
|
||||
def _snapshot(num_samples: int) -> np.ndarray:
|
||||
return np.arange(num_samples, dtype=np.float32)
|
||||
|
||||
|
||||
def _sliced_len(session: MiniCPMO45DataPlaneSession, num_samples: int) -> int:
|
||||
sliced = session.slice_cumulative_audio(REQUEST_ID, _snapshot(num_samples))
|
||||
if sliced is None:
|
||||
return 0
|
||||
return int(np.asarray(sliced).reshape(-1).size)
|
||||
|
||||
|
||||
def test_a_growing_snapshot_yields_only_the_new_tail() -> None:
|
||||
session = _session()
|
||||
assert _sliced_len(session, 24000) == 24000
|
||||
assert _sliced_len(session, 48000) == 24000
|
||||
assert _sliced_len(session, 48000) == 0 # unchanged snapshot emits nothing
|
||||
|
||||
|
||||
def test_a_placeholder_unit_does_not_rewind_the_cursor() -> None:
|
||||
"""The failure behind a 300s benchmark case producing 1373s of audio.
|
||||
|
||||
A continuation unit arrives as a one-sample snapshot. Taking that as "the
|
||||
producer restarted" drops the cursor to 1, so the next real snapshot is
|
||||
sliced from the start and the client is handed the entire session again.
|
||||
"""
|
||||
session = _session()
|
||||
assert _sliced_len(session, 637440) == 637440
|
||||
assert _sliced_len(session, 1) == 1
|
||||
assert _sliced_len(session, 1) == 1
|
||||
assert _sliced_len(session, 1) == 1
|
||||
|
||||
# 649920 - 637440, not 649920 - 1.
|
||||
assert _sliced_len(session, 649920) == 12480
|
||||
|
||||
|
||||
def test_a_genuinely_restarted_producer_is_still_picked_up() -> None:
|
||||
"""Two consecutive below-cursor snapshots that grow are a new buffer."""
|
||||
session = _session()
|
||||
assert _sliced_len(session, 48000) == 48000
|
||||
assert _sliced_len(session, 1000) == 1000 # head of the new buffer
|
||||
assert _sliced_len(session, 3000) == 2000 # its tail, not 3000 - 48000
|
||||
assert _sliced_len(session, 5000) == 2000
|
||||
|
||||
|
||||
def test_close_stream_releases_the_cursor_for_the_next_stream() -> None:
|
||||
session = _session()
|
||||
assert _sliced_len(session, 48000) == 48000
|
||||
session.close_stream(REQUEST_ID)
|
||||
assert _sliced_len(session, 24000) == 24000
|
||||
@@ -55,20 +55,18 @@ class TestRegistryDeclaration:
|
||||
assert pipeline.model_type == _PIPELINE_KEY
|
||||
assert pipeline.model_arch == "MiniCPMO45OmniForConditionalGeneration"
|
||||
|
||||
def test_native_duplex_control_is_explicit_without_a_fixed_session_cap(self) -> None:
|
||||
def test_duplex_plugin_is_declared_without_a_fixed_session_cap(self) -> None:
|
||||
pipeline = OMNI_PIPELINES[_PIPELINE_KEY]
|
||||
assert pipeline.duplex_control_enabled is True
|
||||
assert pipeline.duplex_serving_adapter == (
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.serving_adapter.MiniCPMO45ServingRuntimeAdapter"
|
||||
assert pipeline.duplex_plugin == (
|
||||
"vllm_omni.model_executor.models.minicpmo_4_5.duplex.plugin.MiniCPMO45DuplexPlugin"
|
||||
)
|
||||
assert not hasattr(pipeline, "max_native_duplex_sessions")
|
||||
|
||||
def test_ordinary_pipeline_defaults_to_no_duplex_control(self) -> None:
|
||||
def test_ordinary_pipeline_declares_no_duplex_plugin(self) -> None:
|
||||
pipeline = PipelineConfig(
|
||||
model_type="ordinary", stages=(StagePipelineConfig(stage_id=0, model_stage="a", final_output=True),)
|
||||
)
|
||||
assert pipeline.duplex_control_enabled is False
|
||||
assert pipeline.duplex_serving_adapter is None
|
||||
assert pipeline.duplex_plugin is None
|
||||
assert not hasattr(pipeline, "max_native_duplex_sessions")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Nemotron VoiceChat duplex is not on the duplex plugin framework yet.
|
||||
|
||||
RFC vllm-omni#7181 splits the port: this PR lands the unified framework with
|
||||
MiniCPM-o 4.5, and the follow-up PR moves Nemotron VoiceChat onto
|
||||
``DuplexModelPlugin``. Until then the modules under test still import the
|
||||
pre-framework duplex runtime / serving adapter that this PR removed, so
|
||||
collecting them would fail at import time and abort the whole pytest session.
|
||||
The tests themselves are unchanged and come back with the port.
|
||||
|
||||
Listed per file rather than globbed, so the coverage loss is visible and shrinks
|
||||
file by file as the port lands. ``test_input.py`` imports nothing that was
|
||||
removed and keeps running.
|
||||
"""
|
||||
|
||||
#: Both import ``DuplexFence`` from ``engine.duplex.messages``, which no longer
|
||||
#: exports it; ``test_data_plane`` also imports ``engine.duplex.runtime``.
|
||||
collect_ignore = ["test_data_plane.py", "test_runtime_contract.py"]
|
||||
@@ -7,9 +7,9 @@ from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm_omni.engine.duplex.runtime import duplex_resource_request_id
|
||||
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import duplex_resource_request_id
|
||||
from vllm_omni.model_executor.models.nemotron_voicechat.duplex.data_plane import (
|
||||
NemotronVoiceChatDataPlaneContext,
|
||||
NemotronVoiceChatDataPlaneSession,
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""PersonaPlex duplex is not on the duplex plugin framework yet.
|
||||
|
||||
RFC vllm-omni#7181 splits the port: this PR lands the unified framework with
|
||||
MiniCPM-o 4.5, and the follow-up PR moves PersonaPlex onto ``DuplexModelPlugin``.
|
||||
Until then the modules under test still import the pre-framework duplex
|
||||
runtime / serving adapter that this PR removed, so collecting them would fail
|
||||
at import time and abort the whole pytest session. The tests themselves are
|
||||
unchanged and come back with the port.
|
||||
|
||||
Listed per file rather than globbed, so the coverage loss is visible and shrinks
|
||||
file by file as the port lands.
|
||||
"""
|
||||
|
||||
#: ``test_unified_runtime`` imports ``engine.duplex.runtime`` and
|
||||
#: ``entrypoints.duplex.runtime_adapter`` directly. ``test_stage0_runtime`` fails
|
||||
#: transitively: ``personaplex.duplex.runtime_extension`` imports ``DuplexFence``
|
||||
#: from ``engine.duplex.messages``, which no longer exports it.
|
||||
collect_ignore = ["test_stage0_runtime.py", "test_unified_runtime.py"]
|
||||
@@ -8,6 +8,12 @@ from types import SimpleNamespace
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexInputMode,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.runtime_adapter import (
|
||||
ServingRuntimeConfigError,
|
||||
)
|
||||
|
||||
from tests.e2e.online_serving import personaplex_realtime_duplex as e2e_driver
|
||||
from vllm_omni.config.stage_config import (
|
||||
@@ -15,12 +21,6 @@ from vllm_omni.config.stage_config import (
|
||||
merge_pipeline_deploy,
|
||||
)
|
||||
from vllm_omni.engine.duplex.messages import DuplexFence
|
||||
from vllm_omni.engine.duplex.runtime import (
|
||||
DuplexInputMode,
|
||||
)
|
||||
from vllm_omni.entrypoints.duplex.runtime_adapter import (
|
||||
ServingRuntimeConfigError,
|
||||
)
|
||||
from vllm_omni.model_executor.models.personaplex.duplex.config import DEFAULT_PERSONA
|
||||
from vllm_omni.model_executor.models.personaplex.duplex.data_plane import (
|
||||
PersonaPlexDataPlaneContext,
|
||||
|
||||
@@ -23,7 +23,7 @@ def _minicpmo_duplex_policy_case(
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
)
|
||||
|
||||
session_key = ("sid-policy", 1)
|
||||
session_key = "sid-policy"
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
model.model_stage = "llm"
|
||||
model._minicpmo45_native_duplex_token_ids_cache = {
|
||||
@@ -35,8 +35,7 @@ def _minicpmo_duplex_policy_case(
|
||||
row = DuplexSamplingRow(
|
||||
row_idx=0,
|
||||
request_id="req-policy",
|
||||
session_id=session_key[0],
|
||||
incarnation=session_key[1],
|
||||
session_id=session_key,
|
||||
seq=3,
|
||||
payload=payload,
|
||||
max_tokens=20,
|
||||
@@ -63,13 +62,12 @@ def test_minicpmo_model_hook_owns_duplex_sampling_rows_and_force_listen():
|
||||
"tts_bos_token_id": 8,
|
||||
"turn_eos_token_id": 9,
|
||||
}
|
||||
model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={("sid-hook", 2): state})
|
||||
model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={"sid-hook": state})
|
||||
logits = torch.zeros((1, 16), dtype=torch.float32)
|
||||
row = DuplexSamplingRow(
|
||||
row_idx=0,
|
||||
request_id="req-hook",
|
||||
session_id="sid-hook",
|
||||
incarnation=2,
|
||||
seq=3,
|
||||
payload={"force_listen": True, "is_speech": True},
|
||||
max_tokens=20,
|
||||
@@ -78,7 +76,7 @@ def test_minicpmo_model_hook_owns_duplex_sampling_rows_and_force_listen():
|
||||
model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,))
|
||||
|
||||
assert model._minicpmo45_active_duplex_rows == [0]
|
||||
assert model._minicpmo45_duplex_row_sessions == {0: ("sid-hook", 2)}
|
||||
assert model._minicpmo45_duplex_row_sessions == {0: "sid-hook"}
|
||||
assert model._minicpmo45_duplex_row_payloads == {0: row.payload}
|
||||
assert model._minicpmo45_duplex_row_max_tokens == {0: 20}
|
||||
assert logits[0, listen_id].item() == 0.0
|
||||
@@ -152,7 +150,7 @@ def test_minicpmo_model_hook_old_response_output_does_not_clear_pending_speech_c
|
||||
pending_speech_context=True,
|
||||
)
|
||||
model, row = _minicpmo_duplex_policy_case(state, {"is_speech": True})
|
||||
model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)}
|
||||
model._minicpmo45_duplex_row_sessions = {0: row.session_id}
|
||||
model._minicpmo45_duplex_row_payloads = {0: row.payload}
|
||||
|
||||
model._record_minicpmo45_duplex_terminator(
|
||||
@@ -173,7 +171,7 @@ def test_minicpmo_model_hook_new_response_output_clears_pending_speech_context()
|
||||
pending_speech_context=True,
|
||||
)
|
||||
model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False})
|
||||
model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)}
|
||||
model._minicpmo45_duplex_row_sessions = {0: row.session_id}
|
||||
model._minicpmo45_duplex_row_payloads = {0: row.payload}
|
||||
|
||||
model._record_minicpmo45_duplex_terminator(
|
||||
@@ -195,7 +193,7 @@ def test_minicpmo_model_hook_empty_speak_envelope_preserves_pending_speech_conte
|
||||
pending_speech_response_open=False,
|
||||
)
|
||||
model, row = _minicpmo_duplex_policy_case(state, {"is_speech": False})
|
||||
model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)}
|
||||
model._minicpmo45_duplex_row_sessions = {0: row.session_id}
|
||||
model._minicpmo45_duplex_row_payloads = {0: row.payload}
|
||||
token_ids = {
|
||||
"listen_token_id": 7,
|
||||
@@ -236,7 +234,7 @@ def test_minicpmo_model_hook_second_new_response_step_is_not_forced_to_listen():
|
||||
original_logits = logits.clone()
|
||||
|
||||
model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,))
|
||||
model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)}
|
||||
model._minicpmo45_duplex_row_sessions = {0: row.session_id}
|
||||
model._minicpmo45_duplex_row_payloads = {0: row.payload}
|
||||
model._record_minicpmo45_duplex_terminator(
|
||||
0,
|
||||
@@ -269,7 +267,7 @@ def test_minicpmo_model_hook_same_speech_row_second_step_does_not_rearm_pending_
|
||||
logits[0, 10] = 20.0
|
||||
|
||||
model.prepare_duplex_sampling(logits, SimpleNamespace(), (row,))
|
||||
model._minicpmo45_duplex_row_sessions = {0: (row.session_id, row.incarnation)}
|
||||
model._minicpmo45_duplex_row_sessions = {0: row.session_id}
|
||||
model._minicpmo45_duplex_row_payloads = {0: row.payload}
|
||||
model._record_minicpmo45_duplex_terminator(
|
||||
0,
|
||||
@@ -339,7 +337,6 @@ def test_generic_ar_runner_builds_typed_duplex_sampling_rows():
|
||||
"duplex": {
|
||||
"data_plane": True,
|
||||
"session_id": "sid-runner-hook",
|
||||
"incarnation": 4,
|
||||
"seq": 4,
|
||||
"payload": {"is_speech": True},
|
||||
}
|
||||
@@ -359,7 +356,6 @@ def test_generic_ar_runner_builds_typed_duplex_sampling_rows():
|
||||
assert rows[0].row_idx == 0
|
||||
assert rows[0].request_id == "req-duplex"
|
||||
assert rows[0].session_id == "sid-runner-hook"
|
||||
assert rows[0].incarnation == 4
|
||||
assert rows[0].seq == 4
|
||||
assert rows[0].payload == {"is_speech": True}
|
||||
assert rows[0].max_tokens == 32
|
||||
@@ -458,13 +454,13 @@ def test_generic_ar_runner_has_no_minicpmo_sampler_state_or_typeerror_probe():
|
||||
assert 'if "duplex_rows" not in str(exc)' not in source
|
||||
|
||||
|
||||
def test_minicpmo_model_cleans_incarnation_state_when_request_finishes():
|
||||
def test_minicpmo_model_cleans_session_state_when_request_finishes():
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
)
|
||||
|
||||
request_id = "duplex-sid-cleanup-i3-e0-stage0"
|
||||
session_key = ("sid-cleanup", 3)
|
||||
session_key = "sid-cleanup"
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
model.model = SimpleNamespace()
|
||||
model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: object()})
|
||||
@@ -1731,8 +1727,8 @@ def test_minicpmo_stage0_native_sampler_penalizes_text_from_prior_chunk():
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
model.model_stage = "llm"
|
||||
model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer())
|
||||
session_key = ("sid-cross-chunk-repetition", 0)
|
||||
state = _MiniCPMO45Stage0SessionState(session_id=session_key[0])
|
||||
session_key = "sid-cross-chunk-repetition"
|
||||
state = _MiniCPMO45Stage0SessionState(session_id=session_key)
|
||||
state.generated_tokens = [198] * 8
|
||||
model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state})
|
||||
model._minicpmo45_duplex_row_sessions = {0: session_key}
|
||||
@@ -1791,8 +1787,8 @@ def test_minicpmo_stage0_native_sampler_matches_official_negative_logit_penalty(
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
model.model_stage = "llm"
|
||||
model.thinker = SimpleNamespace(get_tokenizer=lambda: _Tokenizer())
|
||||
session_key = ("sid-negative-logit-repetition", 0)
|
||||
state = _MiniCPMO45Stage0SessionState(session_id=session_key[0])
|
||||
session_key = "sid-negative-logit-repetition"
|
||||
state = _MiniCPMO45Stage0SessionState(session_id=session_key)
|
||||
repeated = 198
|
||||
alternative = 1234
|
||||
state.generated_tokens = [repeated]
|
||||
@@ -1827,8 +1823,8 @@ def test_minicpmo_stage0_records_bounded_model_policy_history():
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
)
|
||||
|
||||
session_key = ("sid-text-history", 0)
|
||||
state = _MiniCPMO45Stage0SessionState(session_id=session_key[0])
|
||||
session_key = "sid-text-history"
|
||||
state = _MiniCPMO45Stage0SessionState(session_id=session_key)
|
||||
model = MiniCPMO45OmniForConditionalGeneration.__new__(MiniCPMO45OmniForConditionalGeneration)
|
||||
model._minicpmo45_duplex_data_plane_helper = SimpleNamespace(sessions={session_key: state})
|
||||
model._minicpmo45_duplex_row_sessions = {0: session_key}
|
||||
|
||||
@@ -79,16 +79,15 @@ CHECK_IMPORTS = {
|
||||
"vllm_omni/entrypoints/openai/video_stream_base.py",
|
||||
"vllm_omni/entrypoints/openai/video_stream_session.py",
|
||||
"vllm_omni/experimental/fullduplex/client.py",
|
||||
"vllm_omni/engine/duplex/audio.py",
|
||||
"vllm_omni/engine/duplex/commands.py",
|
||||
"vllm_omni/engine/duplex/contracts.py",
|
||||
"vllm_omni/engine/duplex/events.py",
|
||||
"vllm_omni/engine/duplex/realtime_commands.py",
|
||||
"vllm_omni/experimental/fullduplex/joyvl/memory/memory.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/adapter.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/input.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/runtime.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/plugin.py",
|
||||
"vllm_omni/model_executor/models/minicpmo_4_5/duplex/stage0.py",
|
||||
"vllm_omni/entrypoints/duplex/audio.py",
|
||||
"vllm_omni/entrypoints/duplex/realtime_input.py",
|
||||
"vllm_omni/entrypoints/duplex/runtime_bridge.py",
|
||||
"vllm_omni/entrypoints/duplex/serving.py",
|
||||
"vllm_omni/model_executor/models/personaplex/duplex/input.py",
|
||||
"vllm_omni/model_executor/models/personaplex/duplex/runtime_extension.py",
|
||||
"vllm_omni/model_executor/models/personaplex/duplex/stage0.py",
|
||||
|
||||
@@ -417,6 +417,13 @@ def _has_post_commit_decision(
|
||||
# refusal is harmless — record it instead of failing the case.
|
||||
_TOLERATED_ERROR_CODES = frozenset({"playback_ack_too_late"})
|
||||
|
||||
# ``session.close`` carries no reason of its own, so the server stamps the
|
||||
# terminal event with the reason it inferred for it. Anything else on a
|
||||
# ``session.closed`` after our own close request means the session ended for a
|
||||
# reason we did not ask for (a disconnect, a lease timeout, a shutdown), which
|
||||
# is what this guard is looking for.
|
||||
_CLIENT_CLOSE_REASONS = frozenset({"client_close"})
|
||||
|
||||
|
||||
def _error_code(event: dict[str, object]) -> str | None:
|
||||
error = event.get("error")
|
||||
@@ -452,7 +459,7 @@ def _raise_if_session_terminated(
|
||||
event_type == "session.closed"
|
||||
and explicit_close_from is not None
|
||||
and index >= explicit_close_from
|
||||
and reason is None
|
||||
and (reason is None or reason in _CLIENT_CLOSE_REASONS)
|
||||
)
|
||||
if expected:
|
||||
continue
|
||||
@@ -826,7 +833,7 @@ def write_batch_artifacts(
|
||||
)
|
||||
|
||||
|
||||
def _websocket_url(config: OmniInteractBenchmarkConfig, session_id: str) -> str:
|
||||
def _websocket_url(config: OmniInteractBenchmarkConfig) -> str:
|
||||
endpoint = (
|
||||
config.endpoint
|
||||
if urlsplit(config.endpoint).scheme
|
||||
@@ -834,13 +841,7 @@ def _websocket_url(config: OmniInteractBenchmarkConfig, session_id: str) -> str:
|
||||
)
|
||||
from vllm_omni.clients.duplex import build_realtime_url
|
||||
|
||||
return build_realtime_url(
|
||||
endpoint,
|
||||
config.model,
|
||||
autostart=False,
|
||||
native_duplex=True,
|
||||
session_id=session_id,
|
||||
)
|
||||
return build_realtime_url(endpoint, config.model, autostart=False)
|
||||
|
||||
|
||||
class _RealtimeSession:
|
||||
@@ -855,7 +856,7 @@ class _RealtimeSession:
|
||||
|
||||
_MAX_FRAME_BYTES = 64 * 1024 * 1024
|
||||
|
||||
def __init__(self, config: OmniInteractBenchmarkConfig, session_id: str, reference_audio: str) -> None:
|
||||
def __init__(self, config: OmniInteractBenchmarkConfig, reference_audio: str) -> None:
|
||||
from vllm_omni.clients.duplex import DuplexClient, EventCollector, SessionConfig
|
||||
|
||||
self.session_config = SessionConfig(
|
||||
@@ -864,12 +865,11 @@ class _RealtimeSession:
|
||||
playback_commit_policy="ack_only",
|
||||
idle_timeout_s=float(config.timeout_s),
|
||||
extra_body={
|
||||
"native_duplex": True,
|
||||
"force_listen_count": 0,
|
||||
**(config.extra_body or {}),
|
||||
},
|
||||
)
|
||||
self.url = _websocket_url(config, session_id)
|
||||
self.url = _websocket_url(config)
|
||||
headers = dict(config.extra_headers or {})
|
||||
|
||||
async def connect(url: str) -> WebSocketTransport:
|
||||
@@ -886,7 +886,6 @@ class _RealtimeSession:
|
||||
self.url,
|
||||
model=config.model,
|
||||
config=self.session_config,
|
||||
session_id=session_id,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=min(config.timeout_s, 20.0),
|
||||
@@ -1013,7 +1012,7 @@ async def run_omniinteract_case(
|
||||
frames = prepared_input.video_frames
|
||||
if not any(frames):
|
||||
raise ValueError(f"No video frames were decoded from {case.video_path}")
|
||||
async with _RealtimeSession(config, session_id, reference_audio) as client:
|
||||
async with _RealtimeSession(config, reference_audio) as client:
|
||||
session_from = 0 # the collector holds only this session's events
|
||||
pcm = _ensure_final_commit_tail(pcm, client.events.events)
|
||||
playback = _Playback()
|
||||
|
||||
@@ -14,7 +14,7 @@ import time
|
||||
import traceback
|
||||
import uuid
|
||||
import wave
|
||||
from collections.abc import Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -2153,10 +2153,68 @@ async def async_request_openai_audio_speech(
|
||||
return output
|
||||
|
||||
|
||||
#: Silence budget per Seed-TTS turn: a model-native duplex session generates
|
||||
#: per audio unit, and the target text rides the session context, so the
|
||||
#: silence only advances the clock. It stops at the turn's response.done --
|
||||
#: a native model that keeps hearing silence after its turn may decide to
|
||||
#: speak again, and the benchmark measures one response per utterance -- so
|
||||
#: the budget is only spent on a turn the model is slow to take: it may
|
||||
#: choose to listen on a few units first, and a turn that has not settled
|
||||
#: when the budget runs out is reported with what the model did.
|
||||
_SEED_TTS_SILENCE_SECONDS = 30.0
|
||||
#: A native model normally answers the seeded text within this much silence;
|
||||
#: a turn that needs more is logged so a slow-to-speak model shows in the run.
|
||||
_SEED_TTS_PROMPT_RESPONSE_S = 12.0
|
||||
#: MiniCPM-o emits 24 kHz mono; used to report audio_frames after the session closed.
|
||||
_SEED_TTS_OUTPUT_SAMPLE_RATE_HZ = 24_000
|
||||
|
||||
|
||||
def _seed_tts_turn_stall_report(events: object, response_offset: int, request_index: int, silence_s: float) -> str:
|
||||
"""Explain a Seed-TTS turn that never settled: what the model did with the silence."""
|
||||
response_ids = list(getattr(events, "response_ids")[response_offset:])
|
||||
if not response_ids:
|
||||
return (
|
||||
f"Seed-TTS Realtime TTS turn {request_index} never started a response: the model listened "
|
||||
f"through {silence_s:.1f}s of silence and the wait that followed"
|
||||
)
|
||||
audio_bytes = getattr(events, "audio_bytes")
|
||||
response_text = getattr(events, "response_text")
|
||||
started = ", ".join(
|
||||
f"{response_id} ({len(audio_bytes(response_id))} audio bytes, text {response_text(response_id)!r})"
|
||||
for response_id in response_ids
|
||||
)
|
||||
return (
|
||||
f"Seed-TTS Realtime TTS turn {request_index} started {len(response_ids)} response(s) after "
|
||||
f"{silence_s:.1f}s of silence but none reached response.done: {started}"
|
||||
)
|
||||
|
||||
|
||||
def _seed_tts_turn_response_id(events: object, response_offset: int, request_index: int) -> str:
|
||||
"""The response id the Seed-TTS turn is measured on: the first one with audio.
|
||||
|
||||
A model-native session answers the seeded text once, but nothing in the
|
||||
protocol stops it from speaking again on silence it hears afterwards, so
|
||||
a later audio response is the model's own and not a failed turn.
|
||||
"""
|
||||
response_ids = getattr(events, "response_ids")
|
||||
audio_bytes = getattr(events, "audio_bytes")
|
||||
audio_response_ids = [response_id for response_id in response_ids[response_offset:] if audio_bytes(response_id)]
|
||||
if not audio_response_ids:
|
||||
raise RuntimeError(f"Seed-TTS Realtime TTS turn {request_index} produced no audio response")
|
||||
if len(audio_response_ids) > 1:
|
||||
logger.warning(
|
||||
"Seed-TTS Realtime TTS turn %d: model spoke again after its response (%d audio responses); "
|
||||
"measuring the first",
|
||||
request_index,
|
||||
len(audio_response_ids),
|
||||
)
|
||||
return audio_response_ids[0]
|
||||
|
||||
|
||||
def _realtime_websocket_url(api_url: str) -> str:
|
||||
from vllm_omni.clients.duplex import build_realtime_url
|
||||
|
||||
return build_realtime_url(api_url, None, native_duplex=None)
|
||||
return build_realtime_url(api_url, None)
|
||||
|
||||
|
||||
def _nonnegative_number(value: object) -> bool:
|
||||
@@ -2321,6 +2379,13 @@ class _RealtimeTTSProbe:
|
||||
request time.
|
||||
"""
|
||||
|
||||
#: How long ``configure`` waits for a free duplex session. Every Seed-TTS
|
||||
#: utterance is one session and the deploy config admits ``max_sessions``
|
||||
#: of them, so a benchmark run above that concurrency queues for a slot
|
||||
#: rather than counting the server's (retryable) refusal as a failed
|
||||
#: request.
|
||||
_SESSION_SLOT_WAIT_S = 120.0
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
from vllm_omni.clients.duplex import EventCollector
|
||||
|
||||
@@ -2347,16 +2412,13 @@ class _RealtimeTTSProbe:
|
||||
*,
|
||||
output_audio_format: str = "pcm16",
|
||||
instructions: str | None = None,
|
||||
native_duplex: bool = False,
|
||||
auto_response: bool = False,
|
||||
extra_body: dict[str, object] | None = None,
|
||||
session_id: str | None = None,
|
||||
timeout_s: float = 120.0,
|
||||
) -> None:
|
||||
from vllm_omni.clients.duplex import AudioFormat, DuplexClient, SessionConfig
|
||||
from vllm_omni.clients.duplex import AudioFormat, DuplexClient, DuplexProtocolError, SessionConfig
|
||||
|
||||
session_extra_body: dict[str, object] = dict(extra_body or {})
|
||||
session_extra_body["native_duplex"] = bool(native_duplex)
|
||||
config = SessionConfig(
|
||||
output_audio=AudioFormat(output_audio_format, 24_000),
|
||||
instructions=instructions,
|
||||
@@ -2365,16 +2427,33 @@ class _RealtimeTTSProbe:
|
||||
playback_commit_policy="ack_only",
|
||||
extra_body=session_extra_body,
|
||||
)
|
||||
self._client = DuplexClient(
|
||||
deadline = time.monotonic() + self._SESSION_SLOT_WAIT_S
|
||||
delay_s = 0.25
|
||||
waited = False
|
||||
while True:
|
||||
# A failed handshake closes the socket on the client's side, so
|
||||
# a refused attempt leaves nothing behind to clean up.
|
||||
client = DuplexClient(
|
||||
self._url,
|
||||
model=model,
|
||||
config=config,
|
||||
session_id=session_id,
|
||||
reconnect=None,
|
||||
heartbeat_interval_s=None,
|
||||
handshake_timeout_s=timeout_s,
|
||||
)
|
||||
await self._client.__aenter__()
|
||||
try:
|
||||
await client.__aenter__()
|
||||
except DuplexProtocolError as exc:
|
||||
if exc.code != "resource_exhausted" or time.monotonic() >= deadline:
|
||||
raise
|
||||
if not waited:
|
||||
logger.info("Seed-TTS Realtime TTS: no free duplex session (%s); waiting for a slot", exc)
|
||||
waited = True
|
||||
await asyncio.sleep(delay_s)
|
||||
delay_s = min(delay_s * 2.0, 2.0)
|
||||
continue
|
||||
self._client = client
|
||||
break
|
||||
self._consume_task = asyncio.create_task(self.events.consume(self._client))
|
||||
|
||||
async def send(self, event: dict[str, object]) -> None:
|
||||
@@ -2387,6 +2466,32 @@ class _RealtimeTTSProbe:
|
||||
assert self._client is not None
|
||||
await acknowledge_collected_playback(self._client, self.events)
|
||||
|
||||
async def stream_silence(
|
||||
self,
|
||||
*,
|
||||
seconds: float,
|
||||
chunk_ms: int = 200,
|
||||
until: Callable[[], bool] | None = None,
|
||||
) -> float:
|
||||
"""Append silent PCM16 units so a model-native session has units to speak on.
|
||||
|
||||
A duplex model generates per audio unit. The target text rides the
|
||||
session context (``duplex_initial_user_text``), so the audio only has
|
||||
to advance the clock; silence keeps it from adding content of its own.
|
||||
Streams in real time for at most ``seconds``, stopping as soon as
|
||||
``until`` holds, and returns the seconds actually appended.
|
||||
"""
|
||||
assert self._client is not None
|
||||
input_format = self._client.config.input_audio
|
||||
chunk = bytes(max(input_format.byte_count(chunk_ms), input_format.bytes_per_sample))
|
||||
chunk_s = input_format.duration_ms(len(chunk)) / 1000.0
|
||||
streamed_s = 0.0
|
||||
while streamed_s < seconds and not (until is not None and until()):
|
||||
await self._client.append_audio(chunk, is_speech=False)
|
||||
streamed_s += chunk_s
|
||||
await asyncio.sleep(chunk_s)
|
||||
return streamed_s
|
||||
|
||||
async def close_session(self, *, timeout_s: float = 20.0) -> None:
|
||||
assert self._client is not None
|
||||
await self._client.close(timeout_s=timeout_s)
|
||||
@@ -2423,7 +2528,21 @@ async def async_request_openai_realtime_duplex(
|
||||
if not turn_prompts:
|
||||
turn_prompts = [("", request_func_input.prompt)]
|
||||
session_id = f"seed-tts-{request_func_input.request_id or uuid.uuid4().hex}"
|
||||
silence_seconds = float(getattr(request_func_input, "seed_tts_silence_seconds", 0.0) or _SEED_TTS_SILENCE_SECONDS)
|
||||
turn_metrics: list[dict[str, object]] = []
|
||||
turn_timings: list[dict[str, object]] = []
|
||||
turn_pcm_bytes: list[bytes] = []
|
||||
turn_transcripts: list[str] = []
|
||||
measurement_origin = {
|
||||
"ttft": "first silence append client send to first non-empty text delta",
|
||||
"ttfp": "first silence append client send to first audio packet",
|
||||
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||
}
|
||||
try:
|
||||
# One session per utterance. A model-native duplex session takes its
|
||||
# text once, in the session context (``duplex_initial_user_text``), so
|
||||
# a session cannot be re-seeded for a second target text.
|
||||
for request_index, (utterance_id, target_text) in enumerate(turn_prompts):
|
||||
async with _RealtimeTTSProbe(_realtime_websocket_url(request_func_input.api_url)) as client:
|
||||
await client.configure(
|
||||
request_func_input.model_name or request_func_input.model,
|
||||
@@ -2433,59 +2552,46 @@ async def async_request_openai_realtime_duplex(
|
||||
"seed_tts_system_prompt",
|
||||
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
|
||||
),
|
||||
native_duplex=False,
|
||||
auto_response=False,
|
||||
extra_body=speech_extra,
|
||||
session_id=session_id,
|
||||
auto_response=True,
|
||||
extra_body={
|
||||
**speech_extra,
|
||||
"duplex_initial_user_text": target_text,
|
||||
"force_listen_count": 0,
|
||||
},
|
||||
timeout_s=120.0,
|
||||
)
|
||||
turn_metrics: list[dict[str, object]] = []
|
||||
turn_timings: list[dict[str, object]] = []
|
||||
turn_pcm_bytes: list[bytes] = []
|
||||
turn_transcripts: list[str] = []
|
||||
measurement_origin = {
|
||||
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||
"ttfp": "conversation.item.create client send to first audio packet",
|
||||
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||
}
|
||||
for request_index, (utterance_id, target_text) in enumerate(turn_prompts):
|
||||
response_offset = len(client.events.response_ids)
|
||||
done_before = client.events.count("response.done")
|
||||
errors_before = len(client.events.errors())
|
||||
turn_started_at_s = time.monotonic()
|
||||
await client.send(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": target_text}],
|
||||
},
|
||||
}
|
||||
)
|
||||
await client.send({"type": "response.create"})
|
||||
await wait_for_condition(
|
||||
lambda: (
|
||||
|
||||
def turn_settled() -> bool:
|
||||
return (
|
||||
client.events.count("response.done") > done_before
|
||||
or len(client.events.errors()) > errors_before
|
||||
),
|
||||
)
|
||||
|
||||
turn_started_at_s = time.monotonic()
|
||||
silence_s = await client.stream_silence(seconds=silence_seconds, until=turn_settled)
|
||||
if silence_s > _SEED_TTS_PROMPT_RESPONSE_S:
|
||||
logger.warning(
|
||||
"Seed-TTS Realtime TTS turn %d: the model took %.1fs of silence to settle its response",
|
||||
request_index,
|
||||
silence_s,
|
||||
)
|
||||
try:
|
||||
await wait_for_condition(
|
||||
turn_settled,
|
||||
timeout_s=180.0,
|
||||
label=f"Seed-TTS Realtime TTS turn {request_index} response.done",
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise RuntimeError(
|
||||
_seed_tts_turn_stall_report(client.events, response_offset, request_index, silence_s)
|
||||
) from exc
|
||||
errors = client.events.errors()
|
||||
if len(errors) > errors_before:
|
||||
raise RuntimeError(f"Seed-TTS Realtime TTS server error: {errors[-1]}")
|
||||
new_audio_response_ids = [
|
||||
response_id
|
||||
for response_id in client.events.response_ids[response_offset:]
|
||||
if client.events.audio_bytes(response_id)
|
||||
]
|
||||
if len(new_audio_response_ids) != 1:
|
||||
raise RuntimeError(
|
||||
f"Seed-TTS Realtime TTS turn {request_index} expected one audio response, "
|
||||
f"got {len(new_audio_response_ids)}"
|
||||
)
|
||||
response_id = new_audio_response_ids[0]
|
||||
response_id = _seed_tts_turn_response_id(client.events, response_offset, request_index)
|
||||
timing = client.events.timing_summary(
|
||||
after_s=turn_started_at_s,
|
||||
input_committed_at_s=turn_started_at_s,
|
||||
@@ -2518,12 +2624,12 @@ async def async_request_openai_realtime_duplex(
|
||||
)
|
||||
turn_transcripts.append(client.events.response_text(response_id))
|
||||
await client.acknowledge_playback()
|
||||
await client.close_session(timeout_s=30.0)
|
||||
request_finished_at = time.perf_counter()
|
||||
session_metrics = summarize_session_request_metrics(
|
||||
turn_metrics,
|
||||
session_id=session_id,
|
||||
)
|
||||
await client.close_session(timeout_s=30.0)
|
||||
|
||||
output.generated_text = " ".join(filter(None, turn_transcripts))
|
||||
from vllm_omni.clients.duplex import metric_mean
|
||||
@@ -2531,10 +2637,8 @@ async def async_request_openai_realtime_duplex(
|
||||
output.ttft = (metric_mean(session_metrics.get("ttft_ms")) or 0.0) / 1000.0
|
||||
output.audio_ttfp = (metric_mean(session_metrics.get("ttfp_ms")) or 0.0) / 1000.0
|
||||
output.audio_rtf = metric_mean(session_metrics.get("rtf")) or 0.0
|
||||
output.audio_duration = (
|
||||
sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
|
||||
)
|
||||
output.audio_frames = int(output.audio_duration * client.events.output_sample_rate_hz)
|
||||
output.audio_duration = sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
|
||||
output.audio_frames = int(output.audio_duration * _SEED_TTS_OUTPUT_SAMPLE_RATE_HZ)
|
||||
output.latency = request_finished_at - output.start_time
|
||||
output.tts_turn_pcm_bytes = turn_pcm_bytes
|
||||
output.tts_output_pcm_bytes = b"".join(turn_pcm_bytes)
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Client-side libraries for vLLM-Omni serving APIs.
|
||||
|
||||
Modules under this package speak to a running vLLM-Omni server over the
|
||||
network. They are for applications and tests; server runtime code
|
||||
``vllm_omni.clients.duplex`` speaks to a running vLLM-Omni server over the
|
||||
network; ``vllm_omni.clients.inline_duplex`` drives an in-process
|
||||
``DuplexOmni`` handed in by the caller with the same client API. Modules here
|
||||
import no server runtime code at module load, and server runtime code
|
||||
(``vllm_omni.engine``, ``vllm_omni.entrypoints``, ``vllm_omni.model_executor``)
|
||||
must never import from here.
|
||||
"""
|
||||
+216
-149
@@ -2,9 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Public asynchronous client for the vLLM-Omni full-duplex Realtime API.
|
||||
|
||||
Connects to ``/v1/realtime?duplex=1`` (the normative duplex contract) and
|
||||
provides typed events, response demultiplexing, incremental playback
|
||||
acking, and transparent session resume on transport drops.
|
||||
:class:`DuplexClientBase` holds the transport-agnostic session client (typed
|
||||
events, response demultiplexing, incremental playback acking, the session
|
||||
handshake). :class:`DuplexClient` connects it to ``/v1/realtime?duplex=1``
|
||||
(the normative duplex contract) over WebSocket with transparent session
|
||||
resume on transport drops; :class:`vllm_omni.clients.inline_duplex.InlineDuplexClient`
|
||||
drives an in-process :class:`~vllm_omni.entrypoints.duplex_omni.DuplexOmni`
|
||||
with exactly the same usage.
|
||||
|
||||
Example::
|
||||
|
||||
@@ -35,10 +39,10 @@ import math
|
||||
import random
|
||||
import time
|
||||
import wave
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -53,6 +57,7 @@ __all__ = [
|
||||
"AudioDelta",
|
||||
"ConnectionResumed",
|
||||
"DuplexClient",
|
||||
"DuplexClientBase",
|
||||
"DuplexClientError",
|
||||
"DuplexConnectionError",
|
||||
"DuplexEvent",
|
||||
@@ -229,7 +234,7 @@ class SessionConfig:
|
||||
idle_timeout_s: float | None = None
|
||||
extra_body: dict[str, object] = field(default_factory=dict)
|
||||
|
||||
def to_session_payload(self, *, model: str, session_id: str | None = None) -> dict[str, object]:
|
||||
def to_session_payload(self, *, model: str) -> dict[str, object]:
|
||||
payload: dict[str, object] = {
|
||||
"model": model,
|
||||
"modalities": list(self.modalities),
|
||||
@@ -246,8 +251,6 @@ class SessionConfig:
|
||||
},
|
||||
"turn_detection": self.turn_detection,
|
||||
}
|
||||
if session_id:
|
||||
payload["session_id"] = session_id
|
||||
if self.instructions is not None:
|
||||
payload["instructions"] = self.instructions
|
||||
if self.ref_audio is not None:
|
||||
@@ -368,11 +371,6 @@ class SessionCreated(DuplexEvent):
|
||||
value = self.raw.get("resume_token")
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
@property
|
||||
def incarnation(self) -> int:
|
||||
value = self.raw.get("incarnation")
|
||||
return value if isinstance(value, int) else 0
|
||||
|
||||
|
||||
class SessionResumed(SessionCreated):
|
||||
pass
|
||||
@@ -598,79 +596,84 @@ class ResponseHandle:
|
||||
# Client
|
||||
|
||||
|
||||
class WebSocketTransport(Protocol):
|
||||
class WebSocketTransport(ABC):
|
||||
"""Minimal WebSocket surface the client needs (satisfied by ``websockets``)."""
|
||||
|
||||
@abstractmethod
|
||||
async def send(self, data: str) -> None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def recv(self) -> str | bytes: ...
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None: ...
|
||||
|
||||
|
||||
ConnectFn = Callable[[str], Awaitable["WebSocketTransport"]]
|
||||
|
||||
|
||||
class DuplexClient:
|
||||
"""Async client for one duplex session over ``/v1/realtime?duplex=1``.
|
||||
class DuplexClientBase(ABC):
|
||||
"""Transport-agnostic duplex session client.
|
||||
|
||||
``session_id`` only names the session this client creates: entering the
|
||||
client always performs the ``session.update`` handshake. Resuming an
|
||||
existing session is supported only as automatic reconnect within the same
|
||||
client instance (see ``ReconnectPolicy``); taking over a session from a
|
||||
new client requires the wire-level ``session.resume`` handshake
|
||||
(``resume_token``, ``incarnation``, ``last_received_server_event_seq``),
|
||||
which this client does not expose.
|
||||
Everything a caller touches (typed events, response demux, input helpers,
|
||||
playback acks, the ``session.update`` handshake) lives here. A subclass
|
||||
only supplies the transport: :meth:`_open` establishes it and must deliver
|
||||
the server's ``session.created`` payload through :meth:`_dispatch`,
|
||||
:meth:`_send_command` delivers one client event, :meth:`_teardown`
|
||||
releases the transport.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
model: str,
|
||||
config: SessionConfig | None = None,
|
||||
session_id: str | None = None,
|
||||
reconnect: ReconnectPolicy | None = ReconnectPolicy(),
|
||||
heartbeat_interval_s: float | None = 30.0,
|
||||
handshake_timeout_s: float = 30.0,
|
||||
connect: ConnectFn | None = None,
|
||||
) -> None:
|
||||
self.url = url
|
||||
self.model = model
|
||||
self.config = config or SessionConfig()
|
||||
self.session_id = session_id
|
||||
#: Allocated by the server; ``None`` until ``session.created`` arrives.
|
||||
self.session_id: str | None = None
|
||||
self.session_info: dict[str, object] = {}
|
||||
self.incarnation = 0
|
||||
self.resume_token: str | None = None
|
||||
self._reconnect = reconnect
|
||||
self._heartbeat_interval_s = heartbeat_interval_s
|
||||
self._handshake_timeout_s = handshake_timeout_s
|
||||
self._connect_fn: ConnectFn = connect or self._default_connect
|
||||
self._ws: WebSocketTransport | None = None
|
||||
self._reader_task: asyncio.Task[None] | None = None
|
||||
self._heartbeat_task: asyncio.Task[None] | None = None
|
||||
self._subscribers: list[asyncio.Queue[DuplexEvent | _ClosedMarker]] = []
|
||||
self._response_queue: asyncio.Queue[ResponseHandle | _ClosedMarker | _ErrorMarker] = asyncio.Queue(
|
||||
maxsize=_MAX_BUFFERED_EVENTS
|
||||
)
|
||||
self._responses: dict[str, ResponseHandle] = {}
|
||||
self._last_server_event_seq: int | None = None
|
||||
self._input_audio_end_ms = 0.0
|
||||
self._closing = False
|
||||
self._closed = asyncio.Event()
|
||||
self._closed_marker: _ClosedMarker | None = None
|
||||
|
||||
# -- transport hooks ------------------------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
async def _open(self) -> None:
|
||||
"""Establish the transport and start delivering server events via ``_dispatch``.
|
||||
|
||||
Called once from :meth:`__aenter__`; the base then sends the
|
||||
``session.update`` handshake and waits for ``session.created``.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def _send_command(self, payload: dict[str, object]) -> None:
|
||||
"""Deliver one client event (already stamped with ``event_id``)."""
|
||||
|
||||
@abstractmethod
|
||||
async def _teardown(self) -> None:
|
||||
"""Release the transport; must be idempotent."""
|
||||
|
||||
# -- lifecycle ----------------------------------------------------------
|
||||
|
||||
async def __aenter__(self) -> DuplexClient:
|
||||
self._ws = await self._connect_fn(self._target_url())
|
||||
async def __aenter__(self) -> DuplexClientBase:
|
||||
handshake_queue = self._add_subscriber()
|
||||
try:
|
||||
self._reader_task = asyncio.create_task(self._read_loop(), name="duplex-client-reader")
|
||||
await self._open()
|
||||
await self.send(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": self.config.to_session_payload(model=self.model, session_id=self.session_id),
|
||||
"session": self.config.to_session_payload(model=self.model),
|
||||
}
|
||||
)
|
||||
created = await self._wait_on_queue(
|
||||
@@ -679,14 +682,12 @@ class DuplexClient:
|
||||
timeout_s=self._handshake_timeout_s,
|
||||
)
|
||||
except BaseException:
|
||||
await self._teardown()
|
||||
await self._teardown_quietly()
|
||||
raise
|
||||
finally:
|
||||
self._remove_subscriber(handshake_queue)
|
||||
assert isinstance(created, SessionCreated)
|
||||
self._adopt_session(created)
|
||||
if self._heartbeat_interval_s is not None:
|
||||
self._heartbeat_task = asyncio.create_task(self._heartbeat_loop(), name="duplex-client-heartbeat")
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback) -> None:
|
||||
@@ -706,7 +707,7 @@ class DuplexClient:
|
||||
except DuplexClientError:
|
||||
pass
|
||||
finally:
|
||||
await self._teardown()
|
||||
await self._teardown_quietly()
|
||||
|
||||
async def close(self, *, timeout_s: float = 20.0) -> None:
|
||||
"""Send ``session.close`` and wait for the server to confirm."""
|
||||
@@ -723,6 +724,12 @@ class DuplexClient:
|
||||
except asyncio.TimeoutError:
|
||||
self._finalize("close timed out", expected=True)
|
||||
|
||||
async def _teardown_quietly(self) -> None:
|
||||
self._closing = True
|
||||
if not self._closed.is_set():
|
||||
self._finalize("closed", expected=True)
|
||||
await self._teardown()
|
||||
|
||||
# -- input ----------------------------------------------------------------
|
||||
|
||||
async def append_audio(
|
||||
@@ -925,53 +932,25 @@ class DuplexClient:
|
||||
payload = dict(event)
|
||||
payload.setdefault("event_id", f"evt_{uuid4().hex}")
|
||||
try:
|
||||
await self._ws.send(json.dumps(payload))
|
||||
await self._send_command(payload)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except DuplexClientError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DuplexConnectionError(f"send failed: {exc}") from exc
|
||||
return payload["event_id"]
|
||||
|
||||
# -- internals ---------------------------------------------------------------
|
||||
|
||||
def _target_url(self) -> str:
|
||||
parts = urlsplit(self.url)
|
||||
path = parts.path if parts.path not in ("", "/") else "/v1/realtime"
|
||||
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
query.setdefault("duplex", "1")
|
||||
query.setdefault("model", self.model)
|
||||
# The client always opens with an explicit session.update (and resumes
|
||||
# with an explicit session.resume). Autostart would race that handshake:
|
||||
# with a model query param, the server creates a bare default session
|
||||
# before reading the first client event, silently dropping ref_audio
|
||||
# and the model-specific extra_body. Force it off even when the caller's
|
||||
# URL carries autostart=1 — this client cannot operate over it.
|
||||
query["autostart"] = "0"
|
||||
return urlunsplit((parts.scheme or "ws", parts.netloc, path, urlencode(query), parts.fragment))
|
||||
|
||||
async def _default_connect(self, url: str) -> WebSocketTransport:
|
||||
try:
|
||||
import websockets
|
||||
except ImportError as exc: # pragma: no cover - websockets is a pinned dep
|
||||
raise DuplexConnectionError("The duplex client requires the 'websockets' package") from exc
|
||||
try:
|
||||
return await websockets.connect(url, max_size=_MAX_FRAME_BYTES)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DuplexConnectionError(f"connect to {url} failed: {exc}") from exc
|
||||
|
||||
def _adopt_session(self, event: SessionCreated) -> None:
|
||||
# A resume activation carries no nested session object; keep the
|
||||
# session_info captured at the original handshake in that case.
|
||||
# session.created and session.resumed both carry the engine's current
|
||||
# session object; keep the last one seen.
|
||||
if event.session:
|
||||
self.session_info = event.session
|
||||
session_id = event.session.get("session_id") or event.session.get("id") or event.session_id
|
||||
if isinstance(session_id, str) and session_id:
|
||||
self.session_id = session_id
|
||||
self.incarnation = event.incarnation
|
||||
if event.resume_token:
|
||||
self.resume_token = event.resume_token
|
||||
|
||||
def _announce(self, handle: ResponseHandle) -> None:
|
||||
"""Deliver a handle to responses() exactly once, decision known."""
|
||||
@@ -1013,36 +992,18 @@ class DuplexClient:
|
||||
if item.type in types:
|
||||
return item
|
||||
|
||||
async def _read_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
while True:
|
||||
raw = await self._ws.recv()
|
||||
if isinstance(raw, (bytes, bytearray)):
|
||||
continue
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(data, dict):
|
||||
await self._dispatch(data)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if self._closing or self._closed.is_set():
|
||||
self._finalize("closed", expected=True)
|
||||
return
|
||||
if await self._attempt_resume():
|
||||
continue
|
||||
self._finalize(f"connection lost: {exc}", expected=False)
|
||||
return
|
||||
def _accept_event(self, data: dict[str, object]) -> bool:
|
||||
"""Transport hook: return False to drop ``data`` before dispatch (duplicates)."""
|
||||
del data
|
||||
return True
|
||||
|
||||
async def _after_dispatch(self, event: DuplexEvent, data: dict[str, object]) -> None:
|
||||
"""Transport hook run after an event was delivered to subscribers."""
|
||||
del event, data
|
||||
|
||||
async def _dispatch(self, data: dict[str, object]) -> None:
|
||||
seq = data.get("server_event_seq")
|
||||
if isinstance(seq, int):
|
||||
if self._last_server_event_seq is not None and seq <= self._last_server_event_seq:
|
||||
return # duplicate delivered by resume replay
|
||||
self._last_server_event_seq = seq
|
||||
if not self._accept_event(data):
|
||||
return
|
||||
event = wrap_event(data)
|
||||
|
||||
if isinstance(event, SessionResumed):
|
||||
@@ -1109,14 +1070,130 @@ class DuplexClient:
|
||||
elif isinstance(event, SessionExpired):
|
||||
reason = data.get("reason")
|
||||
self._finalize(f"expired: {reason}" if reason else "expired", expected=False)
|
||||
elif event.type == "session.resync_required":
|
||||
|
||||
await self._after_dispatch(event, data)
|
||||
|
||||
def _finalize(self, reason: str, *, expected: bool) -> None:
|
||||
if self._closed.is_set():
|
||||
return
|
||||
marker = _ClosedMarker(reason=reason, expected=expected)
|
||||
self._closed_marker = marker
|
||||
self._closed.set()
|
||||
for handle in self._responses.values():
|
||||
handle._finish(None)
|
||||
self._responses.clear()
|
||||
for queue in (*self._subscribers, self._response_queue):
|
||||
_put_drop_oldest(queue, marker)
|
||||
|
||||
|
||||
class DuplexClient(DuplexClientBase):
|
||||
"""Async client for one duplex session over ``/v1/realtime?duplex=1``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
model: str,
|
||||
config: SessionConfig | None = None,
|
||||
reconnect: ReconnectPolicy | None = ReconnectPolicy(),
|
||||
heartbeat_interval_s: float | None = 30.0,
|
||||
handshake_timeout_s: float = 30.0,
|
||||
connect: ConnectFn | None = None,
|
||||
) -> None:
|
||||
super().__init__(model=model, config=config, handshake_timeout_s=handshake_timeout_s)
|
||||
self.url = url
|
||||
self.resume_token: str | None = None
|
||||
self._reconnect = reconnect
|
||||
self._heartbeat_interval_s = heartbeat_interval_s
|
||||
self._connect_fn: ConnectFn = connect or self._default_connect
|
||||
self._ws: WebSocketTransport | None = None
|
||||
self._reader_task: asyncio.Task[None] | None = None
|
||||
self._heartbeat_task: asyncio.Task[None] | None = None
|
||||
self._last_server_event_seq: int | None = None
|
||||
|
||||
# -- transport hooks ------------------------------------------------------
|
||||
|
||||
async def _open(self) -> None:
|
||||
self._ws = await self._connect_fn(self._target_url())
|
||||
self._reader_task = asyncio.create_task(self._read_loop(), name="duplex-client-reader")
|
||||
|
||||
async def __aenter__(self) -> DuplexClient:
|
||||
await super().__aenter__()
|
||||
if self._heartbeat_interval_s is not None:
|
||||
self._heartbeat_task = asyncio.create_task(self._heartbeat_loop(), name="duplex-client-heartbeat")
|
||||
return self
|
||||
|
||||
async def _send_command(self, payload: dict[str, object]) -> None:
|
||||
if self._ws is None:
|
||||
raise DuplexConnectionError("send failed: transport is not connected")
|
||||
await self._ws.send(json.dumps(payload))
|
||||
|
||||
async def _teardown(self) -> None:
|
||||
for task in (self._heartbeat_task, self._reader_task):
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
if self._ws is not None:
|
||||
try:
|
||||
await self._ws.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# -- internals ---------------------------------------------------------------
|
||||
|
||||
def _target_url(self) -> str:
|
||||
parts = urlsplit(self.url)
|
||||
path = parts.path if parts.path not in ("", "/") else "/v1/realtime"
|
||||
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
||||
query.setdefault("duplex", "1")
|
||||
query.setdefault("model", self.model)
|
||||
# The client always opens with an explicit session.update (and resumes
|
||||
# with an explicit session.resume). Autostart would race that handshake:
|
||||
# with a model query param, the server creates a bare default session
|
||||
# before reading the first client event, silently dropping ref_audio
|
||||
# and the model-specific extra_body. Force it off even when the caller's
|
||||
# URL carries autostart=1 — this client cannot operate over it.
|
||||
query["autostart"] = "0"
|
||||
return urlunsplit((parts.scheme or "ws", parts.netloc, path, urlencode(query), parts.fragment))
|
||||
|
||||
async def _default_connect(self, url: str) -> WebSocketTransport:
|
||||
try:
|
||||
import websockets
|
||||
except ImportError as exc: # pragma: no cover - websockets is a pinned dep
|
||||
raise DuplexConnectionError("The duplex client requires the 'websockets' package") from exc
|
||||
try:
|
||||
return await websockets.connect(url, max_size=_MAX_FRAME_BYTES)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DuplexConnectionError(f"connect to {url} failed: {exc}") from exc
|
||||
|
||||
def _adopt_session(self, event: SessionCreated) -> None:
|
||||
super()._adopt_session(event)
|
||||
if event.resume_token:
|
||||
self.resume_token = event.resume_token
|
||||
|
||||
def _accept_event(self, data: dict[str, object]) -> bool:
|
||||
seq = data.get("server_event_seq")
|
||||
if isinstance(seq, int):
|
||||
if self._last_server_event_seq is not None and seq <= self._last_server_event_seq:
|
||||
return False # duplicate delivered by resume replay
|
||||
self._last_server_event_seq = seq
|
||||
return True
|
||||
|
||||
async def _after_dispatch(self, event: DuplexEvent, data: dict[str, object]) -> None:
|
||||
if event.type == "session.resync_required":
|
||||
# The server stopped journaling this session's events, so any
|
||||
# retained resume credential will be refused on the next
|
||||
# session.resume. Drop it now: a later transport failure then
|
||||
# finalizes immediately instead of burning reconnect attempts.
|
||||
self.resume_token = None
|
||||
|
||||
if isinstance(seq, int) and not self._closed.is_set():
|
||||
seq = data.get("server_event_seq")
|
||||
if isinstance(seq, int) and not self._closed.is_set() and self._ws is not None:
|
||||
try:
|
||||
await self._ws.send(json.dumps({"type": "session.event_ack", "server_event_seq": seq}))
|
||||
except asyncio.CancelledError:
|
||||
@@ -1124,6 +1201,30 @@ class DuplexClient:
|
||||
except Exception:
|
||||
pass # the reader will observe the transport failure itself
|
||||
|
||||
async def _read_loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
while True:
|
||||
raw = await self._ws.recv()
|
||||
if isinstance(raw, (bytes, bytearray)):
|
||||
continue
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(data, dict):
|
||||
await self._dispatch(data)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if self._closing or self._closed.is_set():
|
||||
self._finalize("closed", expected=True)
|
||||
return
|
||||
if await self._attempt_resume():
|
||||
continue
|
||||
self._finalize(f"connection lost: {exc}", expected=False)
|
||||
return
|
||||
|
||||
async def _attempt_resume(self) -> bool:
|
||||
policy = self._reconnect
|
||||
if policy is None or not self.resume_token or not self.session_id:
|
||||
@@ -1142,7 +1243,6 @@ class DuplexClient:
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": self.session_id,
|
||||
"incarnation": self.incarnation,
|
||||
"resume_token": self.resume_token,
|
||||
"last_received_server_event_seq": self._last_server_event_seq or 0,
|
||||
}
|
||||
@@ -1162,7 +1262,10 @@ class DuplexClient:
|
||||
continue
|
||||
event_type = data.get("type")
|
||||
if event_type == "session.resumed":
|
||||
previous = self._ws
|
||||
self._ws = ws
|
||||
if previous is not None and previous is not ws:
|
||||
await self._close_quietly(previous)
|
||||
await self._dispatch({"type": "connection.resumed", "attempt": attempt})
|
||||
await self._dispatch(data)
|
||||
for entry in pending:
|
||||
@@ -1212,35 +1315,6 @@ class DuplexClient:
|
||||
except DuplexClientError:
|
||||
return
|
||||
|
||||
def _finalize(self, reason: str, *, expected: bool) -> None:
|
||||
if self._closed.is_set():
|
||||
return
|
||||
marker = _ClosedMarker(reason=reason, expected=expected)
|
||||
self._closed_marker = marker
|
||||
self._closed.set()
|
||||
for handle in self._responses.values():
|
||||
handle._finish(None)
|
||||
self._responses.clear()
|
||||
for queue in (*self._subscribers, self._response_queue):
|
||||
_put_drop_oldest(queue, marker)
|
||||
|
||||
async def _teardown(self) -> None:
|
||||
self._closing = True
|
||||
if not self._closed.is_set():
|
||||
self._finalize("closed", expected=True)
|
||||
for task in (self._heartbeat_task, self._reader_task):
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
if self._ws is not None:
|
||||
try:
|
||||
await self._ws.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test/benchmark collector
|
||||
@@ -1368,7 +1442,7 @@ class EventCollector:
|
||||
self.response_ids: list[str] = []
|
||||
self.output_sample_rate_hz = 24_000
|
||||
|
||||
async def consume(self, client: DuplexClient) -> None:
|
||||
async def consume(self, client: DuplexClientBase) -> None:
|
||||
"""Subscribe to ``client`` and collect until the session ends."""
|
||||
try:
|
||||
async for event in client.events():
|
||||
@@ -1795,15 +1869,12 @@ def build_realtime_url(
|
||||
model: str | None,
|
||||
*,
|
||||
autostart: bool | None = None,
|
||||
native_duplex: bool | None = None,
|
||||
session_id: str | None = None,
|
||||
extra_query: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Add explicit duplex query parameters to a Realtime URL.
|
||||
|
||||
:class:`DuplexClient` builds its own URL; this helper is for drivers that
|
||||
speak the wire protocol directly. ``native_duplex`` sets the per-session
|
||||
model-native opt-in query flag; other model-specific query flags ride in
|
||||
speak the wire protocol directly. Model-specific query flags ride in
|
||||
``extra_query``. ``http(s)`` URLs are rewritten to ``ws(s)``.
|
||||
"""
|
||||
parts = urlsplit(url)
|
||||
@@ -1815,14 +1886,10 @@ def build_realtime_url(
|
||||
query.setdefault("duplex", "1")
|
||||
if model:
|
||||
query.setdefault("model", model)
|
||||
if native_duplex is not None:
|
||||
query["native_duplex"] = "1" if native_duplex else "0"
|
||||
for key, value in (extra_query or {}).items():
|
||||
query.setdefault(key, value)
|
||||
if autostart is not None:
|
||||
query.setdefault("autostart", "1" if autostart else "0")
|
||||
if session_id:
|
||||
query.setdefault("session_id", session_id)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment))
|
||||
|
||||
|
||||
@@ -1880,7 +1947,7 @@ def summarize_session_request_metrics(
|
||||
return summary
|
||||
|
||||
|
||||
async def acknowledge_collected_playback(client: DuplexClient, collector: EventCollector) -> None:
|
||||
async def acknowledge_collected_playback(client: DuplexClientBase, collector: EventCollector) -> None:
|
||||
"""Ack playback of every collected response's audio (probe shorthand).
|
||||
|
||||
A response that has produced no audio yet is still acked (0 ms) unless it
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""In-process duplex client: the :class:`DuplexClient` usage over a ``DuplexOmni``.
|
||||
|
||||
Example::
|
||||
|
||||
from vllm_omni.clients.inline_duplex import InlineDuplexClient
|
||||
from vllm_omni.clients.minicpmo_4_5 import create_duplex_session_config
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni
|
||||
|
||||
omni = DuplexOmni(model="openbmb/MiniCPM-o-4_5", trust_remote_code=True)
|
||||
cfg = create_duplex_session_config(ref_audio=audio_data_url(wav_bytes))
|
||||
async with InlineDuplexClient(omni, model="openbmb/MiniCPM-o-4_5", config=cfg) as client:
|
||||
await client.stream_pcm(pcm16)
|
||||
await client.commit()
|
||||
async for response in client.responses():
|
||||
async for chunk in response.audio():
|
||||
play(chunk)
|
||||
break
|
||||
|
||||
There is no transport to drop, so this client has no reconnect, resume,
|
||||
heartbeat or event-acknowledgement machinery. The ``DuplexOmni`` instance is
|
||||
passed in by the caller; this module never imports server runtime code at
|
||||
import time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm_omni.clients.duplex import DuplexClientBase, DuplexConnectionError, SessionConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
|
||||
from vllm_omni.entrypoints.duplex_omni import DuplexOmni, DuplexSessionHandle
|
||||
|
||||
__all__ = ["InlineDuplexClient"]
|
||||
|
||||
#: Acknowledgements are a websocket-journal concern; heartbeats are lease touches and go through.
|
||||
_TRANSPORT_ONLY_EVENTS = frozenset({"session.event_ack"})
|
||||
|
||||
|
||||
class InlineDuplexClient(DuplexClientBase):
|
||||
"""Async client for one duplex session on an in-process :class:`DuplexOmni`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
omni: DuplexOmni,
|
||||
*,
|
||||
model: str,
|
||||
config: SessionConfig | None = None,
|
||||
handshake_timeout_s: float = 30.0,
|
||||
) -> None:
|
||||
super().__init__(model=model, config=config, handshake_timeout_s=handshake_timeout_s)
|
||||
self.omni = omni
|
||||
self._handle: DuplexSessionHandle | None = None
|
||||
self._pump_task: asyncio.Task[None] | None = None
|
||||
self._handshake_pending = False
|
||||
|
||||
# -- transport hooks ------------------------------------------------------
|
||||
|
||||
async def _open(self) -> None:
|
||||
# The handle's first event is ``session.created``; the pump delivers
|
||||
# it through ``_dispatch`` where the base handshake is waiting.
|
||||
self._handle = await self.omni.open_session(self.config.to_session_payload(model=self.model))
|
||||
# The base class follows up with the ``session.update`` handshake; the
|
||||
# open above already carried that session object, so it is not resent.
|
||||
self._handshake_pending = True
|
||||
self._pump_task = asyncio.create_task(self._pump_events(), name="inline-duplex-client-pump")
|
||||
|
||||
def _input_defaults(self) -> RealtimeInputDefaults:
|
||||
"""Audio format / sample-rate / VAD defaults declared by this session's object."""
|
||||
from vllm_omni.engine.duplex.realtime_commands import RealtimeInputDefaults
|
||||
|
||||
session_payload = self.session_info or self.config.to_session_payload(model=self.model)
|
||||
return RealtimeInputDefaults().with_session_payload(session_payload)
|
||||
|
||||
async def _send_command(self, payload: dict[str, object]) -> None:
|
||||
handle = self._handle
|
||||
if handle is None or handle.closed:
|
||||
raise DuplexConnectionError("send failed: session is not open")
|
||||
event_type = payload.get("type")
|
||||
if event_type == "session.update" and self._handshake_pending:
|
||||
self._handshake_pending = False
|
||||
return
|
||||
if event_type == "session.close":
|
||||
await handle.close()
|
||||
return
|
||||
if event_type in _TRANSPORT_ONLY_EVENTS:
|
||||
return
|
||||
# Lazy import keeps ``vllm_omni.clients`` free of runtime imports at
|
||||
# module load; the caller already holds a live DuplexOmni.
|
||||
from vllm_omni.engine.duplex.commands import DuplexCommandError, command_from_realtime
|
||||
from vllm_omni.engine.duplex.events import error_event
|
||||
|
||||
try:
|
||||
command = command_from_realtime(payload, defaults=self._input_defaults())
|
||||
except DuplexCommandError as exc:
|
||||
await self._dispatch(
|
||||
error_event(exc.code, str(exc), event_id=exc.event_id or payload.get("event_id")).to_realtime()
|
||||
)
|
||||
return
|
||||
await handle.submit(command)
|
||||
|
||||
async def _teardown(self) -> None:
|
||||
pump = self._pump_task
|
||||
if pump is not None and not pump.done():
|
||||
pump.cancel()
|
||||
try:
|
||||
await pump
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
handle = self._handle
|
||||
if handle is not None and not handle.closed:
|
||||
try:
|
||||
await handle.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# -- internals ---------------------------------------------------------------
|
||||
|
||||
async def _pump_events(self) -> None:
|
||||
handle = self._handle
|
||||
assert handle is not None
|
||||
reason = "closed"
|
||||
try:
|
||||
async for event in handle.events():
|
||||
await self._dispatch(event.to_realtime())
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
reason = f"session failed: {exc}"
|
||||
self._finalize(reason, expected=False)
|
||||
return
|
||||
self._finalize(reason, expected=True)
|
||||
@@ -13,7 +13,7 @@ __all__ = ["create_duplex_session_config"]
|
||||
|
||||
def create_duplex_session_config(*, ref_audio: str | None = None, **overrides: object) -> SessionConfig:
|
||||
"""Session preset matching the MiniCPM-o 4.5 native duplex deployment."""
|
||||
extra_body: dict[str, object] = {"native_duplex": True, "force_listen_count": 0}
|
||||
extra_body: dict[str, object] = {"force_listen_count": 0}
|
||||
extra_body.update(overrides.pop("extra_body", {}))
|
||||
config = SessionConfig(
|
||||
ref_audio=ref_audio,
|
||||
|
||||
@@ -309,14 +309,17 @@ class PipelineConfig:
|
||||
diffusers_class_name: str | None = None
|
||||
diffusers_class_aliases: tuple[str, ...] = ()
|
||||
endpoint_restrictions: tuple[EndpointRestriction, ...] = ()
|
||||
# Optional model-owned duplex planner loaded by the stable engine runtime.
|
||||
# Dotted path of the model's ``DuplexModelPlugin``. A pipeline is a duplex
|
||||
# model iff this is set: ``vllm-omni serve`` then always runs it through
|
||||
# ``DuplexOmni`` (every served surface runs on a duplex session) and the engine hosts one
|
||||
# ``DuplexOrchestrator`` with the plugin loaded.
|
||||
duplex_plugin: str | None = None
|
||||
# Legacy duplex wiring of the models that are not ported to the plugin
|
||||
# framework yet (PersonaPlex, Nemotron VoiceChat). Nothing reads them: a
|
||||
# pipeline that only declares these is served turn-based. Each field goes
|
||||
# away with the follow-up PR that ports its model to ``duplex_plugin``.
|
||||
duplex_runtime_extension: str | None = None
|
||||
# Optional model-owned Serving adapter loaded only when the Realtime duplex
|
||||
# endpoint is enabled. Generic OpenAI modules must not select a model.
|
||||
duplex_serving_adapter: str | None = None
|
||||
# Explicitly enable the stable duplex control mechanism. This is separate
|
||||
# from the optional model extension because turn-commit-only deployments
|
||||
# do not require a model planner.
|
||||
duplex_control_enabled: bool = False
|
||||
# Bundled deploy defaults for this concrete pipeline topology. The file is
|
||||
# loaded from vllm_omni/deploy; None uses DeployConfig defaults.
|
||||
@@ -510,6 +513,12 @@ class DuplexSessionRuntimeConfig:
|
||||
max_pending_input_bytes_per_session: int = 16 * 1024 * 1024
|
||||
max_pending_turns_per_session: int = 4
|
||||
max_sessions: int = 1
|
||||
# Unread by the plugin framework. It used to bound the per-session
|
||||
# completed-append table that made a retried append RPC submit once; the
|
||||
# framework carries appends as one-way commands on the runner's ordered
|
||||
# mailbox, so there is no client-visible retry to deduplicate. The field
|
||||
# stays so the deploy configs of the models that are not ported yet still
|
||||
# parse, and goes away with the PR that ports the last of them.
|
||||
completed_append_cache_size: int = 256
|
||||
server_vad_model_path: str | None = None
|
||||
# Startup warmup: run this many silent 80 ms-style frames through a
|
||||
|
||||
Loaded 100 of 182 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user